├── tests ├── __init__.py ├── mispevent_testfiles │ ├── simple.json │ ├── sighting.json │ ├── event.json │ ├── event_tags.json │ ├── attribute.json │ ├── attribute_del.json │ ├── malware.json │ ├── test_object_template │ │ └── definition.json │ ├── event_obj_tag.json │ ├── proposals.json │ ├── misp_custom_obj.json │ ├── event_obj_def_param.json │ ├── def_param.json │ ├── event_obj_attr_tag.json │ └── shadow.json ├── new_misp_event.json ├── search_index_result.json ├── misp_event.json ├── sharing_groups.json └── 57c4445b-c548-4654-af0b-4be3950d210f.json ├── examples ├── __init__.py ├── profiles │ ├── __init__.py │ ├── weekly_report.py │ └── daily_report.py ├── feed-generator │ ├── output │ │ └── empty │ ├── README.md │ └── settings.default.py ├── get_network_activity.event_id ├── freetext.txt ├── sighting.json ├── user_sample.json ├── feed-generator-from-redis │ ├── install.sh │ ├── server.py │ ├── ObjectConstructor │ │ └── CowrieMISPObject.py │ ├── settings.default.py │ ├── README.md │ ├── MISPItemToRedis.py │ └── fromredis.py ├── keys.py.sample ├── cache_all.py ├── test_sign.py ├── ioc-2-misp │ ├── README.md │ ├── taxonomy.csv │ └── keys.py.sample ├── tags.py ├── stats.py ├── users_list.py ├── situational-awareness │ ├── test_attribute_treemap.html │ ├── style2.css │ ├── style.css │ ├── bokeh_tools.py │ ├── attribute_treemap.py │ ├── date_tools.py │ ├── pygal_tools.py │ ├── tag_scatter.py │ ├── tags_count.py │ ├── README.md │ ├── tag_search.py │ └── tags_to_graphs.py ├── freetext.py ├── sharing_groups.py ├── fetch_events_feed.py ├── sighting.py ├── warninglists.py ├── get_attachment.py ├── events │ ├── create_dummy_event.py │ ├── create_massive_dummy_events.py │ ├── dummy │ ├── README.md │ └── tools.py ├── lookup.py ├── up.py ├── delete_user.py ├── suricata.py ├── edit_user.py ├── add_user.py ├── add_named_attribute.py ├── add_user_json.py ├── add_feed.py ├── del.py ├── openioc_to_misp.py ├── yara.py ├── edit_user_json.py ├── suricata_search │ └── README.md ├── tagstatistics.py ├── add_email_object.py ├── create_events.py ├── add_sbsignature.py ├── get.py ├── addtag.py ├── last.py ├── searchall.py ├── get_csv.py ├── search_attributes_yara.py ├── add_generic_object.py ├── misp2clamav.py ├── add_file_object.py ├── graphdb │ └── make_neo4j.py ├── addtag2.py ├── search.py ├── upload.py ├── generate_file_objects.py ├── misp2cef.py ├── copy_list.py ├── add_fail2ban_object.py ├── yara_dump.py ├── et2misp.py └── asciidoc_generator.py ├── docs ├── source │ ├── README.md │ ├── index.rst │ ├── tools.rst │ └── modules.rst └── tutorial │ └── install_notebook.sh ├── setup.cfg ├── .gitmodules ├── .gitignore ├── MANIFEST.in ├── .travis.yml ├── pymisp ├── tools │ ├── fail2banobject.py │ ├── sbsignatureobject.py │ ├── asnobject.py │ ├── domainipobject.py │ ├── geolocationobject.py │ ├── load_warninglists.py │ ├── __init__.py │ ├── ext_lookups.py │ ├── stix.py │ ├── genericgenerator.py │ ├── neo4j.py │ ├── abstractgenerator.py │ ├── emailobject.py │ ├── vtreportobject.py │ ├── fileobject.py │ ├── machoobject.py │ ├── elfobject.py │ └── create_misp_object.py ├── exceptions.py └── __init__.py ├── LICENSE ├── setup.py └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/profiles/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/source/README.md: -------------------------------------------------------------------------------- 1 | ../../README.md -------------------------------------------------------------------------------- /examples/feed-generator/output/empty: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/get_network_activity.event_id: -------------------------------------------------------------------------------- 1 | 2 2 | 1 3 | 3 4 | 4 -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/simple.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | } 4 | } 5 | -------------------------------------------------------------------------------- /examples/freetext.txt: -------------------------------------------------------------------------------- 1 | 8.8.8.8 2 | 3 | google.fr 4 | 5 | https://gmail.com 6 | -------------------------------------------------------------------------------- /docs/tutorial/install_notebook.sh: -------------------------------------------------------------------------------- 1 | pip3 install --upgrade pip 2 | pip3 install jupyter 3 | -------------------------------------------------------------------------------- /examples/sighting.json: -------------------------------------------------------------------------------- 1 | {"values":["www.google.com", "8.8.8.8"], "timestamp":1460558710} 2 | 3 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/sighting.json: -------------------------------------------------------------------------------- 1 | { 2 | "timestamp": 11111111, 3 | "type": "bar", 4 | "value": "1" 5 | } 6 | -------------------------------------------------------------------------------- /examples/user_sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "email":"maaiil@domain.lu", 3 | "org_id":1, 4 | "role_id":1, 5 | "autoalert":1 6 | } 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "pymisp/data/misp-objects"] 2 | path = pymisp/data/misp-objects 3 | url = https://github.com/MISP/misp-objects 4 | -------------------------------------------------------------------------------- /examples/feed-generator-from-redis/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | virtualenv -p python3 serv-env 3 | . ./serv-env/bin/activate 4 | pip3 install -U flask Flask-AutoIndex redis 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pem 3 | *.pyc 4 | examples/keys.py 5 | examples/cudeso.py 6 | examples/feed-generator/output/*.json 7 | build/* 8 | dist/* 9 | pymisp.egg-info/* 10 | .idea 11 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/event.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "analysis": "1", 4 | "date": "2017-12-31", 5 | "distribution": "1", 6 | "info": "This is a test", 7 | "published": true, 8 | "threat_level_id": "1" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /examples/keys.py.sample: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | misp_url = 'https:///' 5 | misp_key = 'Your MISP auth key' # The MISP auth key can be found on the MISP web interface under the automation section 6 | misp_verifycert = True 7 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft docs 2 | graft examples 3 | graft tests 4 | include CHANGELOG.txt 5 | include LICENSE 6 | include pymisp/data/*.json 7 | include pymisp/data/misp-objects/*.json 8 | include pymisp/data/misp-objects/objects/*/definition.json 9 | include pymisp/data/misp-objects/relationships/definition.json 10 | include README.md 11 | -------------------------------------------------------------------------------- /examples/cache_all.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from keys import misp_url, misp_key, misp_verifycert 5 | from pymisp import PyMISP 6 | 7 | 8 | def init(url, key): 9 | return PyMISP(url, key, misp_verifycert, 'json') 10 | 11 | 12 | if __name__ == '__main__': 13 | misp = init(misp_url, misp_key) 14 | misp.cache_all_feeds() -------------------------------------------------------------------------------- /examples/feed-generator-from-redis/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os.path 4 | from flask import Flask 5 | from flask_autoindex import AutoIndex 6 | from settings import outputdir 7 | 8 | app = Flask(__name__) 9 | AutoIndex(app, browse_root=os.path.join(os.path.curdir, outputdir)) 10 | 11 | if __name__ == '__main__': 12 | app.run(host='0.0.0.0') 13 | -------------------------------------------------------------------------------- /examples/feed-generator/README.md: -------------------------------------------------------------------------------- 1 | # What 2 | 3 | This python script can be used to generate a MISP feed based on an existing MISP instance. 4 | 5 | # Installation 6 | 7 | ```` 8 | git clone https://github.com/CIRCL/PyMISP 9 | cd examples/feed-generator 10 | cp settings-default.py settings.py 11 | vi settings.py #adjust your settings 12 | python3 generate.py 13 | ```` 14 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/event_tags.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "Tag": [ 4 | { 5 | "name": "bar" 6 | }, 7 | { 8 | "name": "baz" 9 | }, 10 | { 11 | "name": "foo" 12 | } 13 | ], 14 | "analysis": "1", 15 | "date": "2017-12-31", 16 | "distribution": "1", 17 | "info": "This is a test", 18 | "threat_level_id": "1" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. PyMISP documentation master file, created by 2 | sphinx-quickstart on Fri Aug 26 11:39:17 2016. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to PyMISP's documentation! 7 | ================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 4 13 | 14 | README 15 | modules 16 | tools 17 | 18 | Indices and tables 19 | ================== 20 | 21 | * :ref:`genindex` 22 | * :ref:`modindex` 23 | * :ref:`search` 24 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/attribute.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "Attribute": [ 4 | { 5 | "Tag": [ 6 | { 7 | "name": "osint" 8 | } 9 | ], 10 | "category": "Payload delivery", 11 | "disable_correlation": false, 12 | "to_ids": true, 13 | "type": "filename", 14 | "value": "bar.exe" 15 | } 16 | ], 17 | "analysis": "1", 18 | "date": "2017-12-31", 19 | "distribution": "1", 20 | "info": "This is a test", 21 | "threat_level_id": "1" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /examples/test_sign.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | 6 | from pymisp import mispevent 7 | 8 | 9 | if __name__ == '__main__': 10 | parser = argparse.ArgumentParser(description='Sign & verify a MISP event.') 11 | parser.add_argument("-i", "--input", required=True, help="Json file") 12 | parser.add_argument("-u", "--uid", required=True, help="GPG UID") 13 | args = parser.parse_args() 14 | 15 | me = mispevent.MISPEvent() 16 | me.load(args.input) 17 | 18 | me.sign(args.uid) 19 | me.verify(args.uid) 20 | -------------------------------------------------------------------------------- /examples/ioc-2-misp/README.md: -------------------------------------------------------------------------------- 1 | ### Description 2 | 3 | Python script for ioc import to misp 4 | 5 | ### requires 6 | 7 | > python 2.7 8 | > PyMISP 9 | > BeautifulSoup (apt-get install python-bs4 python-lxml) 10 | 11 | ### Usage 12 | 13 | ```bash 14 | python ioc2misp.py -i myioc -t "tag:mytag='sample','tag:other='foo'" 15 | ``` 16 | 17 | ```bash 18 | time find /iocsample -type f|while read line ;do python ioc2misp.py -i ${line};done 19 | ``` 20 | 21 | ### Conf 22 | 23 | * rename keys.py.sample as keys.py 24 | * add your url and api key in keys.py 25 | * use command in terminal 26 | -------------------------------------------------------------------------------- /examples/profiles/weekly_report.py: -------------------------------------------------------------------------------- 1 | types_to_attach = ['ip-dst', 'url', 'domain', 'md5'] 2 | objects_to_attach = ['domain-ip', 'file'] 3 | 4 | headers = """ 5 | :toc: right 6 | :toclevels: 1 7 | :toc-title: Weekly Report 8 | :icons: font 9 | :sectanchors: 10 | :sectlinks: 11 | = Weekly report by {org_name} 12 | {date} 13 | 14 | :icons: font 15 | 16 | """ 17 | 18 | event_level_tags = """ 19 | """ 20 | 21 | attributes = """ 22 | === Indicator(s) of compromise 23 | 24 | {list_attributes} 25 | 26 | """ 27 | 28 | title = """ 29 | == ({internal_id}) {title} 30 | 31 | {summary} 32 | 33 | """ 34 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/attribute_del.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "Attribute": [ 4 | { 5 | "Tag": [ 6 | { 7 | "name": "osint" 8 | } 9 | ], 10 | "category": "Payload delivery", 11 | "deleted": true, 12 | "disable_correlation": false, 13 | "id": "42", 14 | "to_ids": true, 15 | "type": "filename", 16 | "value": "bar.exe" 17 | } 18 | ], 19 | "analysis": "1", 20 | "date": "2017-12-31", 21 | "distribution": "1", 22 | "info": "This is a test", 23 | "threat_level_id": "1" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/malware.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "Attribute": [ 4 | { 5 | "category": "Payload delivery", 6 | "data": "ewogICJFdmVudCI6IHsKICB9Cn0K", 7 | "disable_correlation": false, 8 | "encrypt": true, 9 | "malware_filename": "bar.exe", 10 | "to_ids": true, 11 | "type": "malware-sample", 12 | "value": "bar.exe|7637beddacbeac59d44469b2b120b9e6" 13 | } 14 | ], 15 | "analysis": "1", 16 | "date": "2017-12-31", 17 | "distribution": "1", 18 | "info": "This is a test", 19 | "threat_level_id": "1" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /examples/tags.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | import json 8 | 9 | 10 | def init(url, key): 11 | return PyMISP(url, key, True, 'json', True) 12 | 13 | 14 | def get_tags(m): 15 | result = m.get_all_tags(True) 16 | r = result 17 | print(json.dumps(r) + '\n') 18 | 19 | 20 | if __name__ == '__main__': 21 | parser = argparse.ArgumentParser(description='Get tags from MISP instance.') 22 | 23 | args = parser.parse_args() 24 | 25 | misp = init(misp_url, misp_key) 26 | 27 | get_tags(misp) 28 | -------------------------------------------------------------------------------- /examples/ioc-2-misp/taxonomy.csv: -------------------------------------------------------------------------------- 1 | link,value,keep,taxonomy,comment 2 | classification,TLP AMBER,1,tlp:amber, 3 | classification,TLP GREEN,1,tlp:green, 4 | confidential,TLP-AMBER,1,tlp:amber, 5 | confidential,TLP GREEN,1,tlp:green, 6 | confidential,TLP-GREEN,1,tlp:green, 7 | confidential,TLP RED,1,tlp:red, 8 | exportable,Yes,0,, 9 | family,APT,1,malware_classification:malware-category='APT', 10 | family,APT3,1,malware_classification:malware-category='APT3',https://github.com/fireeye/iocs/tree/master/APT3 11 | license,Apache 2.0,0,, 12 | threatcategory,APT3,1,malware_classification:malware-category='APT3',https://github.com/fireeye/iocs/tree/master/APT3 13 | -------------------------------------------------------------------------------- /examples/profiles/daily_report.py: -------------------------------------------------------------------------------- 1 | types_to_attach = ['ip-dst', 'url', 'domain'] 2 | objects_to_attach = ['domain-ip'] 3 | 4 | headers = """ 5 | :toc: right 6 | :toclevels: 1 7 | :toc-title: Daily Report 8 | :icons: font 9 | :sectanchors: 10 | :sectlinks: 11 | = Daily report by {org_name} 12 | {date} 13 | 14 | :icons: font 15 | 16 | """ 17 | 18 | event_level_tags = """ 19 | IMPORTANT: This event is classified TLP:{value}. 20 | 21 | {expanded} 22 | 23 | """ 24 | 25 | attributes = """ 26 | === Indicator(s) of compromise 27 | 28 | {list_attributes} 29 | 30 | """ 31 | 32 | title = """ 33 | == ({internal_id}) {title} 34 | 35 | {summary} 36 | 37 | """ 38 | -------------------------------------------------------------------------------- /examples/stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | 8 | 9 | def init(url, key): 10 | return PyMISP(url, key, misp_verifycert, 'json') 11 | 12 | if __name__ == '__main__': 13 | parser = argparse.ArgumentParser(description='Output attributes statistics from a MISP instance.') 14 | args = parser.parse_args() 15 | 16 | misp = init(misp_url, misp_key) 17 | 18 | print (misp.get_attributes_statistics(misp, percentage=True)) 19 | print (misp.get_attributes_statistics(context='category', percentage=True)) 20 | -------------------------------------------------------------------------------- /examples/users_list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 9 | try: 10 | input = raw_input 11 | except NameError: 12 | pass 13 | 14 | 15 | def init(url, key): 16 | return PyMISP(url, key, True, 'json') 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Get a list of the sharing groups from the MISP instance.') 20 | 21 | misp = init(misp_url, misp_key) 22 | 23 | users_list = misp.get_users_list() 24 | print (users_list) 25 | -------------------------------------------------------------------------------- /examples/situational-awareness/test_attribute_treemap.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 14 | 15 | 16 | 17 | 18 | 19 |
20 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /examples/freetext.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | from io import open 9 | 10 | 11 | if __name__ == '__main__': 12 | parser = argparse.ArgumentParser(description="Update a MISP event.") 13 | parser.add_argument("-e", "--event", required=True, help="Event ID to update.") 14 | parser.add_argument("-i", "--input", required=True, help="Input file") 15 | 16 | args = parser.parse_args() 17 | 18 | pymisp = PyMISP(misp_url, misp_key) 19 | 20 | with open(args.input, 'r') as f: 21 | result = pymisp.freetext(args.event, f.read()) 22 | print(result) 23 | -------------------------------------------------------------------------------- /examples/sharing_groups.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 9 | try: 10 | input = raw_input 11 | except NameError: 12 | pass 13 | 14 | 15 | def init(url, key): 16 | return PyMISP(url, key, True, 'json') 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Get a list of the sharing groups from the MISP instance.') 20 | 21 | misp = init(misp_url, misp_key) 22 | 23 | sharing_groups = misp.get_sharing_groups() 24 | print sharing_groups 25 | 26 | -------------------------------------------------------------------------------- /examples/situational-awareness/style2.css: -------------------------------------------------------------------------------- 1 | body 2 | { 3 | /*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/ 4 | font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; 5 | } 6 | 7 | h1 8 | { 9 | font-size: 16px; 10 | width: 290px; 11 | text-align:center; 12 | } 13 | 14 | /*** Stats Tables ***/ 15 | 16 | table 17 | { 18 | border-collapse: collapse; 19 | border-spacing: 0; 20 | table-layout: fixed; 21 | width: 6000px; 22 | border: 1px solid #cbcbcb; 23 | } 24 | 25 | tbody 26 | { 27 | font-size:12px; 28 | } 29 | 30 | td 31 | { 32 | border-left: 1px solid #cbcbcb; 33 | border-width: 0 0 0 1px; 34 | margin: 0; 35 | padding: 0.5em 1em; 36 | } 37 | 38 | table tr td:first-child 39 | { 40 | font-weight: bold; 41 | } 42 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/test_object_template/definition.json: -------------------------------------------------------------------------------- 1 | { 2 | "requiredOneOf": [ 3 | "member1", 4 | "member2" 5 | ], 6 | "required": [ 7 | "member3" 8 | ], 9 | "attributes": { 10 | "member1": { 11 | "description": "FirstMember", 12 | "misp-attribute": "text" 13 | }, 14 | "member2": { 15 | "description": "SecondMember", 16 | "misp-attribute": "text", 17 | "multiple": true 18 | }, 19 | "member3": { 20 | "description": "Thirdmember", 21 | "misp-attribute": "text" 22 | } 23 | }, 24 | "version": 1, 25 | "description": "TestTemplate.", 26 | "meta-category": "file", 27 | "uuid": "4ec55cc6-9e49-4c64-b794-03c25c1a6589", 28 | "name": "test_object_template" 29 | } 30 | -------------------------------------------------------------------------------- /examples/fetch_events_feed.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from keys import misp_url, misp_key 5 | import argparse 6 | from pymisp import PyMISP 7 | 8 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 9 | try: 10 | input = raw_input 11 | except NameError: 12 | pass 13 | 14 | def init(url, key): 15 | return PyMISP(url, key, False, 'json', debug=False) 16 | 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Fetch all events from a feed.') 20 | parser.add_argument("-f", "--feed", required=True, help="feed's ID to be fetched.") 21 | args = parser.parse_args() 22 | 23 | misp = init(misp_url, misp_key) 24 | misp.fetch_feed(args.feed) 25 | -------------------------------------------------------------------------------- /examples/sighting.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 9 | try: 10 | input = raw_input 11 | except NameError: 12 | pass 13 | 14 | 15 | def init(url, key): 16 | return PyMISP(url, key, True, 'json') 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Add sighting.') 20 | parser.add_argument("-f", "--json_file", required=True, help="The name of the json file describing the attribute you want to add sighting to.") 21 | args = parser.parse_args() 22 | 23 | misp = init(misp_url, misp_key) 24 | 25 | misp.sighting_per_json(args.json_file) 26 | -------------------------------------------------------------------------------- /examples/warninglists.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from pymisp.tools import load_warninglists 6 | import argparse 7 | from keys import misp_url, misp_key 8 | 9 | 10 | if __name__ == '__main__': 11 | 12 | parser = argparse.ArgumentParser(description='Load the warninglists.') 13 | parser.add_argument("-p", "--package", action='store_true', help="from the PyMISPWarninglists package.") 14 | parser.add_argument("-r", "--remote", action='store_true', help="from the MISP instance.") 15 | 16 | args = parser.parse_args() 17 | 18 | if args.package: 19 | print(load_warninglists.from_package()) 20 | elif args.remote: 21 | pm = PyMISP(misp_url, misp_key) 22 | print(load_warninglists.from_instance(pm)) 23 | -------------------------------------------------------------------------------- /examples/get_attachment.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | 8 | 9 | def init(url, key): 10 | return PyMISP(url, key, misp_verifycert, 'json') 11 | 12 | 13 | if __name__ == '__main__': 14 | parser = argparse.ArgumentParser(description='Get an attachment.') 15 | parser.add_argument("-a", "--attribute", type=int, help="Attribute ID to download.") 16 | args = parser.parse_args() 17 | 18 | misp = init(misp_url, misp_key) 19 | 20 | with open('foo', 'wb') as f: 21 | out = misp.get_attachment(args.attribute) 22 | if isinstance(out, dict): 23 | # Fails 24 | print(out) 25 | else: 26 | f.write(out) 27 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | cache: pip 4 | 5 | addons: 6 | apt: 7 | sources: [ 'ubuntu-toolchain-r-test' ] 8 | packages: 9 | - libstdc++6 10 | - libfuzzy-dev 11 | 12 | python: 13 | - "2.7" 14 | - "3.5-dev" 15 | - "3.6" 16 | - "3.6-dev" 17 | 18 | install: 19 | - pip install -U nose pip setuptools 20 | - pip install coveralls codecov requests-mock 21 | - pip install git+https://github.com/kbandla/pydeep.git 22 | - pip install .[fileobjects,neo,openioc,virustotal] 23 | - pushd tests 24 | - git clone https://github.com/viper-framework/viper-test-files.git 25 | - popd 26 | 27 | script: 28 | - nosetests --with-coverage --cover-package=pymisp,tests --cover-tests tests/test_*.py 29 | 30 | after_success: 31 | - codecov 32 | - coveralls 33 | -------------------------------------------------------------------------------- /examples/events/create_dummy_event.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | import tools 8 | 9 | def init(url, key): 10 | return PyMISP(url, key, misp_verifycert, 'json') 11 | 12 | if __name__ == '__main__': 13 | parser = argparse.ArgumentParser(description='Create a given number of event containing an domain|ip attribute and an attachment each.') 14 | parser.add_argument("-l", "--limit", type=int, help="Number of events to create (default 1)") 15 | args = parser.parse_args() 16 | 17 | misp = init(misp_url, misp_key) 18 | 19 | if args.limit is None: 20 | args.limit = 1 21 | 22 | for i in range(args.limit): 23 | tools.create_dummy_event(misp) 24 | -------------------------------------------------------------------------------- /pymisp/tools/fail2banobject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from .abstractgenerator import AbstractMISPObjectGenerator 5 | import logging 6 | 7 | logger = logging.getLogger('pymisp') 8 | 9 | 10 | class Fail2BanObject(AbstractMISPObjectGenerator): 11 | 12 | def __init__(self, parameters, strict=True, standalone=True, **kwargs): 13 | super(Fail2BanObject, self).__init__('fail2ban', strict=strict, standalone=standalone, **kwargs) 14 | self._parameters = parameters 15 | self.generate_attributes() 16 | 17 | def generate_attributes(self): 18 | timestamp = self._sanitize_timestamp(self._parameters.pop('processing-timestamp', None)) 19 | self._parameters['processing-timestamp'] = timestamp 20 | return super(Fail2BanObject, self).generate_attributes() 21 | -------------------------------------------------------------------------------- /examples/lookup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp.tools import ext_lookups 5 | import argparse 6 | 7 | 8 | if __name__ == '__main__': 9 | 10 | parser = argparse.ArgumentParser(description='Search is galaxies or taxonomies.') 11 | parser.add_argument("-q", "--query", help="Query.") 12 | 13 | args = parser.parse_args() 14 | 15 | tag_gal = ext_lookups.revert_tag_from_galaxies(args.query) 16 | tag_tax = ext_lookups.revert_tag_from_taxonomies(args.query) 17 | 18 | found_tax = ext_lookups.search_taxonomies(args.query) 19 | found_gal = ext_lookups.search_galaxies(args.query) 20 | 21 | if tag_gal: 22 | print(tag_gal) 23 | if tag_tax: 24 | print(tag_tax) 25 | if found_tax: 26 | print(found_tax) 27 | if found_gal: 28 | print(found_gal) 29 | -------------------------------------------------------------------------------- /examples/up.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | from io import open 9 | 10 | def init(url, key): 11 | return PyMISP(url, key, True, 'json', debug=True) 12 | 13 | def up_event(m, event, content): 14 | with open(content, 'r') as f: 15 | result = m.update_event(event, f.read()) 16 | print(result) 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description="Update a MISP event.") 20 | parser.add_argument("-e", "--event", required=True, help="Event ID to update.") 21 | parser.add_argument("-i", "--input", required=True, help="Input file") 22 | 23 | args = parser.parse_args() 24 | 25 | misp = init(misp_url, misp_key) 26 | 27 | up_event(misp, args.event, args.input) 28 | -------------------------------------------------------------------------------- /pymisp/tools/sbsignatureobject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from .abstractgenerator import AbstractMISPObjectGenerator 5 | 6 | 7 | class SBSignatureObject(AbstractMISPObjectGenerator): 8 | ''' 9 | Sandbox Analyzer 10 | ''' 11 | def __init__(self, software, report, standalone=True, **kwargs): 12 | super(SBSignatureObject, self).__init__("sb-signature", **kwargs) 13 | self._software = software 14 | self._report = report 15 | self.generate_attributes() 16 | 17 | def generate_attributes(self): 18 | ''' Parse the report for relevant attributes ''' 19 | self.add_attribute("software", value=self._software) 20 | for (signature_name, description) in self._report: 21 | self.add_attribute("signature", value=signature_name, comment=description) 22 | -------------------------------------------------------------------------------- /examples/situational-awareness/style.css: -------------------------------------------------------------------------------- 1 | body 2 | { 3 | /*font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;*/ 4 | font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; 5 | } 6 | 7 | h1 8 | { 9 | font-size: 16px; 10 | width: 290px; 11 | text-align:center; 12 | } 13 | 14 | /*** Stats Tables ***/ 15 | 16 | table 17 | { 18 | border-collapse: collapse; 19 | border-spacing: 0; 20 | border: 1px solid #cbcbcb; 21 | } 22 | 23 | tbody 24 | { 25 | font-size:12px; 26 | } 27 | 28 | table td 29 | { 30 | border-left: 1px solid #cbcbcb; 31 | border-width: 0 0 0 1px; 32 | width: 500px; 33 | margin: 0; 34 | padding: 0.5em 1em; 35 | } 36 | 37 | .test 38 | { 39 | width: 500px; 40 | } 41 | 42 | table tr:nth-child(2n-1) td 43 | { 44 | background-color: #f2f2f2; 45 | } 46 | 47 | table tr td:first-child 48 | { 49 | font-weight: bold; 50 | } 51 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/event_obj_tag.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "Object": [ 4 | { 5 | "Attribute": [ 6 | { 7 | "category": "Payload delivery", 8 | "disable_correlation": false, 9 | "object_relation": "filename", 10 | "to_ids": true, 11 | "type": "filename", 12 | "value": "bar" 13 | } 14 | ], 15 | "Tag": [ 16 | { 17 | "name": "osint" 18 | } 19 | ], 20 | "description": "File object describing a file with meta-information", 21 | "distribution": 5, 22 | "meta-category": "file", 23 | "name": "file", 24 | "sharing_group_id": 0, 25 | "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", 26 | "template_version": 9, 27 | "uuid": "a" 28 | } 29 | ] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /examples/delete_user.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 9 | try: 10 | input = raw_input 11 | except NameError: 12 | pass 13 | 14 | 15 | def init(url, key): 16 | return PyMISP(url, key, True, 'json') 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Delete the user with the given id. Keep in mind that disabling users (by setting the disabled flag via an edit) is always prefered to keep user associations to events intact.') 20 | parser.add_argument("-i", "--user_id", help="The id of the user you want to delete.") 21 | args = parser.parse_args() 22 | 23 | misp = init(misp_url, misp_key) 24 | 25 | print(misp.delete_user(args.user_id)) 26 | -------------------------------------------------------------------------------- /examples/suricata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | 9 | def init(url, key): 10 | return PyMISP(url, key, True) 11 | 12 | 13 | def fetch(m, all_events, event): 14 | if all_events: 15 | print(misp.download_all_suricata().text) 16 | else: 17 | print(misp.download_suricata_rule_event(event).text) 18 | 19 | if __name__ == '__main__': 20 | parser = argparse.ArgumentParser(description='Download Suricata events.') 21 | parser.add_argument("-a", "--all", action='store_true', help="Download all suricata rules available.") 22 | parser.add_argument("-e", "--event", help="Download suricata rules from one event.") 23 | 24 | args = parser.parse_args() 25 | 26 | misp = init(misp_url, misp_key) 27 | 28 | fetch(misp, args.all, args.event) 29 | -------------------------------------------------------------------------------- /examples/edit_user.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 9 | try: 10 | input = raw_input 11 | except NameError: 12 | pass 13 | 14 | 15 | def init(url, key): 16 | return PyMISP(url, key, True, 'json') 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Edit the email of the user designed by the user_id.') 20 | parser.add_argument("-i", "--user_id", required=True, help="The name of the json file describing the user you want to modify.") 21 | parser.add_argument("-e", "--email", help="Email linked to the account.") 22 | args = parser.parse_args() 23 | 24 | misp = init(misp_url, misp_key) 25 | 26 | print(misp.edit_user(args.user_id, email=args.email)) 27 | -------------------------------------------------------------------------------- /pymisp/tools/asnobject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from .abstractgenerator import AbstractMISPObjectGenerator 5 | import logging 6 | 7 | logger = logging.getLogger('pymisp') 8 | 9 | 10 | class ASNObject(AbstractMISPObjectGenerator): 11 | 12 | def __init__(self, parameters, strict=True, standalone=True, **kwargs): 13 | super(ASNObject, self).__init__('asn', strict=strict, standalone=standalone, **kwargs) 14 | self._parameters = parameters 15 | self.generate_attributes() 16 | 17 | def generate_attributes(self): 18 | first = self._sanitize_timestamp(self._parameters.pop('first-seen', None)) 19 | self._parameters['first-seen'] = first 20 | last = self._sanitize_timestamp(self._parameters.pop('last-seen', None)) 21 | self._parameters['last-seen'] = last 22 | return super(ASNObject, self).generate_attributes() 23 | -------------------------------------------------------------------------------- /examples/events/create_massive_dummy_events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import url, key 6 | import argparse 7 | import tools 8 | 9 | 10 | if __name__ == '__main__': 11 | parser = argparse.ArgumentParser(description='Create a given number of event containing a given number of attributes eachh.') 12 | parser.add_argument("-l", "--limit", type=int, help="Number of events to create (default 1)") 13 | parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)") 14 | args = parser.parse_args() 15 | 16 | misp = PyMISP(url, key, True, 'json') 17 | 18 | if args.limit is None: 19 | args.limit = 1 20 | if args.attribute is None: 21 | args.attribute = 3000 22 | 23 | for i in range(args.limit): 24 | tools.create_massive_dummy_events(misp, args.attribute) 25 | -------------------------------------------------------------------------------- /pymisp/tools/domainipobject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from .abstractgenerator import AbstractMISPObjectGenerator 5 | import logging 6 | 7 | logger = logging.getLogger('pymisp') 8 | 9 | 10 | class DomainIPObject(AbstractMISPObjectGenerator): 11 | 12 | def __init__(self, parameters, strict=True, standalone=True, **kwargs): 13 | super(DomainIPObject, self).__init__('domain-ip', strict=strict, standalone=standalone, **kwargs) 14 | self._parameters = parameters 15 | self.generate_attributes() 16 | 17 | def generate_attributes(self): 18 | first = self._sanitize_timestamp(self._parameters.pop('first-seen', None)) 19 | self._parameters['first-seen'] = first 20 | last = self._sanitize_timestamp(self._parameters.pop('last-seen', None)) 21 | self._parameters['last-seen'] = last 22 | return super(DomainIPObject, self).generate_attributes() 23 | -------------------------------------------------------------------------------- /pymisp/tools/geolocationobject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from .abstractgenerator import AbstractMISPObjectGenerator 5 | import logging 6 | 7 | logger = logging.getLogger('pymisp') 8 | 9 | 10 | class GeolocationObject(AbstractMISPObjectGenerator): 11 | 12 | def __init__(self, parameters, strict=True, standalone=True, **kwargs): 13 | super(GeolocationObject, self).__init__('asn', strict=strict, standalone=standalone, **kwargs) 14 | self._parameters = parameters 15 | self.generate_attributes() 16 | 17 | def generate_attributes(self): 18 | first = self._sanitize_timestamp(self._parameters.pop('first-seen', None)) 19 | self._parameters['first-seen'] = first 20 | last = self._sanitize_timestamp(self._parameters.pop('last-seen', None)) 21 | self._parameters['last-seen'] = last 22 | return super(GeolocationObject, self).generate_attributes() 23 | -------------------------------------------------------------------------------- /pymisp/tools/load_warninglists.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | try: 5 | from pymispwarninglists import WarningLists 6 | has_pymispwarninglists = True 7 | except ImportError: 8 | has_pymispwarninglists = False 9 | 10 | 11 | def from_instance(pymisp_instance, slow_search=False): 12 | """Load the warnindlist from an existing MISP instance 13 | :pymisp_instance: Already instantialized PyMISP instance.""" 14 | 15 | warninglists_index = pymisp_instance.get_warninglists()['Warninglists'] 16 | all_warningslists = [] 17 | for warninglist in warninglists_index: 18 | wl = pymisp_instance.get_warninglist(warninglist['Warninglist']['id'])['Warninglist'] 19 | wl['list'] = wl.pop('WarninglistEntry') 20 | all_warningslists.append(wl) 21 | 22 | return WarningLists(slow_search, all_warningslists) 23 | 24 | 25 | def from_package(slow_search=False): 26 | return WarningLists(slow_search) 27 | -------------------------------------------------------------------------------- /examples/add_user.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 9 | try: 10 | input = raw_input 11 | except NameError: 12 | pass 13 | 14 | 15 | def init(url, key): 16 | return PyMISP(url, key, True, 'json') 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Add a new user by setting the mandory fields.') 20 | parser.add_argument("-e", "--email", required=True, help="Email linked to the account.") 21 | parser.add_argument("-o", "--org_id", required=True, help="Organisation linked to the user.") 22 | parser.add_argument("-r", "--role_id", required=True, help="Role linked to the user.") 23 | args = parser.parse_args() 24 | 25 | misp = init(misp_url, misp_key) 26 | 27 | print (misp.add_user(args.email, args.org_id, args.role_id)) 28 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/proposals.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "Attribute": [ 4 | { 5 | "ShadowAttribute": [ 6 | { 7 | "category": "Payload delivery", 8 | "disable_correlation": false, 9 | "to_ids": true, 10 | "type": "filename", 11 | "value": "bar.pdf" 12 | } 13 | ], 14 | "category": "Payload delivery", 15 | "disable_correlation": false, 16 | "to_ids": true, 17 | "type": "filename", 18 | "value": "bar.exe" 19 | } 20 | ], 21 | "ShadowAttribute": [ 22 | { 23 | "category": "Payload delivery", 24 | "disable_correlation": false, 25 | "to_ids": true, 26 | "type": "filename", 27 | "value": "baz.jpg" 28 | } 29 | ], 30 | "analysis": "1", 31 | "date": "2017-12-31", 32 | "distribution": "1", 33 | "info": "This is a test", 34 | "threat_level_id": "1" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /examples/add_named_attribute.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 9 | try: 10 | input = raw_input 11 | except NameError: 12 | pass 13 | 14 | 15 | def init(url, key): 16 | return PyMISP(url, key, True, 'json', debug=True) 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Add an attribute to an event') 20 | parser.add_argument("-e", "--event", help="The id, uuid or json of the event to update.") 21 | parser.add_argument("-t", "--type", help="The type of the added attribute") 22 | parser.add_argument("-v", "--value", help="The value of the attribute") 23 | args = parser.parse_args() 24 | 25 | misp = init(misp_url, misp_key) 26 | 27 | event = misp.add_named_attribute(args.event, args.type, args.value) 28 | print(event) 29 | -------------------------------------------------------------------------------- /examples/add_user_json.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 9 | try: 10 | input = raw_input 11 | except NameError: 12 | pass 13 | 14 | 15 | def init(url, key): 16 | return PyMISP(url, key, True, 'json') 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Add the user described in the given json. If no file is provided, returns a json listing all the fields used to describe a user.') 20 | parser.add_argument("-f", "--json_file", help="The name of the json file describing the user you want to create.") 21 | args = parser.parse_args() 22 | 23 | misp = init(misp_url, misp_key) 24 | 25 | if args.json_file is None: 26 | print (misp.get_add_user_fields_list()) 27 | else: 28 | print(misp.add_user_json(args.json_file)) 29 | -------------------------------------------------------------------------------- /examples/add_feed.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | 8 | if __name__ == '__main__': 9 | parser = argparse.ArgumentParser(description='Add a feed') 10 | parser.add_argument("-f", "--format", required=True, choices=['misp', 'csv', 'freetext'], help="Feed source format") 11 | parser.add_argument("-u", "--url", required=True, help="URL, or local path") 12 | parser.add_argument("-n", "--name", required=True, help="Name of the feed") 13 | parser.add_argument("-i", "--input", required=True, choices=['local', 'network'], help="URL, or local path") 14 | parser.add_argument("-p", "--provider", required=True, help="Provider name") 15 | args = parser.parse_args() 16 | 17 | pm = PyMISP(misp_url, misp_key, misp_verifycert, debug=True) 18 | response = pm.add_feed(args.format, args.url, args.name, args.input, args.provider) 19 | print(response) 20 | -------------------------------------------------------------------------------- /pymisp/tools/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from .vtreportobject import VTReportObject # noqa 4 | from .neo4j import Neo4j # noqa 5 | from .fileobject import FileObject # noqa 6 | from .peobject import PEObject, PESectionObject # noqa 7 | from .elfobject import ELFObject, ELFSectionObject # noqa 8 | from .machoobject import MachOObject, MachOSectionObject # noqa 9 | from .create_misp_object import make_binary_objects # noqa 10 | from .abstractgenerator import AbstractMISPObjectGenerator # noqa 11 | from .genericgenerator import GenericObjectGenerator # noqa 12 | from .openioc import load_openioc, load_openioc_file # noqa 13 | from .sbsignatureobject import SBSignatureObject # noqa 14 | from .fail2banobject import Fail2BanObject # noqa 15 | from .domainipobject import DomainIPObject # noqa 16 | from .asnobject import ASNObject # noqa 17 | from .geolocationobject import GeolocationObject # noqa 18 | 19 | if sys.version_info >= (3, 6): 20 | from .emailobject import EMailObject # noqa 21 | -------------------------------------------------------------------------------- /examples/del.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key,misp_verifycert 6 | import argparse 7 | 8 | 9 | # Usage for pipe masters: ./last.py -l 5h | jq . 10 | 11 | 12 | def init(url, key): 13 | return PyMISP(url, key, misp_verifycert, 'json', debug=True) 14 | 15 | 16 | def del_event(m, eventid): 17 | result = m.delete_event(eventid) 18 | print(result) 19 | 20 | def del_attr(m, attrid): 21 | result = m.delete_attribute(attrid) 22 | print(result) 23 | 24 | if __name__ == '__main__': 25 | parser = argparse.ArgumentParser(description='Delete an event from a MISP instance.') 26 | parser.add_argument("-e", "--event", help="Event ID to delete.") 27 | parser.add_argument("-a", "--attribute", help="Attribute ID to delete.") 28 | 29 | args = parser.parse_args() 30 | 31 | misp = init(misp_url, misp_key) 32 | 33 | if args.event: 34 | del_event(misp, args.event) 35 | else: 36 | del_attr(misp, args.attribute) 37 | -------------------------------------------------------------------------------- /tests/new_misp_event.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "uuid": "57c06bb1-625c-4d34-9b9f-4066950d210f", 4 | "orgc_id": "1", 5 | "publish_timestamp": "0", 6 | "RelatedEvent": [], 7 | "org_id": "1", 8 | "Org": { 9 | "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f", 10 | "name": "CIRCL", 11 | "id": "1" 12 | }, 13 | "attribute_count": null, 14 | "distribution": "0", 15 | "sharing_group_id": "0", 16 | "threat_level_id": "1", 17 | "locked": false, 18 | "Attribute": [], 19 | "published": false, 20 | "ShadowAttribute": [], 21 | "date": "2016-08-26", 22 | "info": "This is a test", 23 | "timestamp": "1472228273", 24 | "Orgc": { 25 | "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f", 26 | "name": "CIRCL", 27 | "id": "1" 28 | }, 29 | "id": "594", 30 | "proposal_email_lock": false, 31 | "analysis": "0" 32 | } 33 | } 34 | 35 | -------------------------------------------------------------------------------- /examples/openioc_to_misp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | 6 | from pymisp import PyMISP 7 | from keys import misp_url, misp_key, misp_verifycert 8 | from pymisp.tools import load_openioc_file 9 | 10 | 11 | if __name__ == '__main__': 12 | parser = argparse.ArgumentParser(description='Convert an OpenIOC file to a MISPEvent. Optionnaly send it to MISP.') 13 | parser.add_argument("-i", "--input", required=True, help="Input file") 14 | group = parser.add_mutually_exclusive_group(required=True) 15 | group.add_argument("-o", "--output", help="Output file") 16 | group.add_argument("-m", "--misp", action='store_true', help="Create new event on MISP") 17 | 18 | args = parser.parse_args() 19 | 20 | misp_event = load_openioc_file(args.input) 21 | 22 | if args.misp: 23 | pymisp = PyMISP(misp_url, misp_key, misp_verifycert, debug=True) 24 | pymisp.add_event(misp_event) 25 | else: 26 | with open(args.output, 'w') as f: 27 | f.write(misp_event.to_json()) 28 | -------------------------------------------------------------------------------- /examples/yara.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key,misp_verifycert 6 | import argparse 7 | import os 8 | 9 | 10 | def init(url, key): 11 | return PyMISP(url, key, misp_verifycert, 'json') 12 | 13 | 14 | def get_yara(m, event_id, out=None): 15 | ok, rules = m.get_yara(event_id) 16 | if not ok: 17 | print(rules) 18 | elif out is None: 19 | print(rules) 20 | else: 21 | with open(out, 'w') as f: 22 | f.write(rules) 23 | 24 | 25 | if __name__ == '__main__': 26 | parser = argparse.ArgumentParser(description='Get yara rules from an event.') 27 | parser.add_argument("-e", "--event", required=True, help="Event ID.") 28 | parser.add_argument("-o", "--output", help="Output file") 29 | 30 | args = parser.parse_args() 31 | 32 | if args.output is not None and os.path.exists(args.output): 33 | print('Output file already exists, abord.') 34 | exit(0) 35 | 36 | misp = init(misp_url, misp_key) 37 | 38 | get_yara(misp, args.event, args.output) 39 | -------------------------------------------------------------------------------- /pymisp/tools/ext_lookups.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | try: 5 | from pymispgalaxies import Clusters 6 | has_pymispgalaxies = True 7 | except ImportError: 8 | has_pymispgalaxies = False 9 | 10 | try: 11 | from pytaxonomies import Taxonomies 12 | has_pymispgalaxies = True 13 | except ImportError: 14 | has_pymispgalaxies = False 15 | 16 | 17 | def revert_tag_from_galaxies(tag): 18 | clusters = Clusters() 19 | try: 20 | return clusters.revert_machinetag(tag) 21 | except Exception: 22 | return [] 23 | 24 | 25 | def revert_tag_from_taxonomies(tag): 26 | taxonomies = Taxonomies() 27 | try: 28 | return taxonomies.revert_machinetag(tag) 29 | except Exception: 30 | return [] 31 | 32 | 33 | def search_taxonomies(query): 34 | taxonomies = Taxonomies() 35 | found = taxonomies.search(query) 36 | if not found: 37 | found = taxonomies.search(query, expanded=True) 38 | return found 39 | 40 | 41 | def search_galaxies(query): 42 | clusters = Clusters() 43 | return clusters.search(query) 44 | -------------------------------------------------------------------------------- /examples/edit_user_json.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 9 | try: 10 | input = raw_input 11 | except NameError: 12 | pass 13 | 14 | 15 | def init(url, key): 16 | return PyMISP(url, key, True, 'json') 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Edit the user designed by the user_id. If no file is provided, returns a json listing all the fields used to describe a user.') 20 | parser.add_argument("-i", "--user_id", required=True, help="The name of the json file describing the user you want to modify.") 21 | parser.add_argument("-f", "--json_file", help="The name of the json file describing your modifications.") 22 | args = parser.parse_args() 23 | 24 | misp = init(misp_url, misp_key) 25 | 26 | if args.json_file is None: 27 | print (misp.get_edit_user_fields_list(args.user_id)) 28 | else: 29 | print(misp.edit_user_json(args.json_file, args.user_id)) 30 | -------------------------------------------------------------------------------- /examples/suricata_search/README.md: -------------------------------------------------------------------------------- 1 | # Description 2 | Get all attributes, from a MISP (https://github.com/MISP) instance, that can be converted into Suricata rules, given a *parameter* and a *term* to search 3 | 4 | **requires** 5 | * PyMISP (https://github.com/CIRCL/PyMISP/) 6 | * python 2.7 or python3 (suggested) 7 | 8 | 9 | # Usage 10 | * **suricata_search.py -p tags -s 'APT' -o misp_ids.rules -t 5** 11 | - search for 'APT' tag 12 | - use 5 threads while generating IDS rules 13 | - dump results to misp_ids.rules 14 | 15 | * **suricata_search.py -p tags -s 'APT' -o misp_ids.rules -ne 411 357 343** 16 | - same as above, but skip events ID 411,357 and 343 17 | 18 | * **suricata_search.py -p tags -s 'circl:incident-classification="malware", tlp:green' -o misp_ids.rules** 19 | - search for multiple tags 'circl:incident-classification="malware", tlp:green' 20 | 21 | * **suricata_search.py -p categories -s 'Artifacts dropped' -t 20 -o artifacts_dropped.rules** 22 | - search for category 'Artifacts dropped' 23 | - use 20 threads while generating IDS rules 24 | - dump results to artifacts_dropped.rules -------------------------------------------------------------------------------- /examples/tagstatistics.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | import json 8 | 9 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 10 | try: 11 | input = raw_input 12 | except NameError: 13 | pass 14 | 15 | 16 | def init(url, key): 17 | return PyMISP(url, key, misp_verifycert, 'json') 18 | 19 | if __name__ == '__main__': 20 | parser = argparse.ArgumentParser(description='Get statistics from tags.') 21 | parser.add_argument("-p", "--percentage", action='store_true', default=None, help="An optional field, if set, it will return the results in percentages, otherwise it returns exact count.") 22 | parser.add_argument("-n", "--namesort", action='store_true', default=None, help="An optional field, if set, values are sort by the namespace, otherwise the sorting will happen on the value.") 23 | args = parser.parse_args() 24 | 25 | misp = init(misp_url, misp_key) 26 | 27 | stats = misp.get_tags_statistics(args.percentage, args.namesort) 28 | print(json.dumps(stats)) 29 | -------------------------------------------------------------------------------- /examples/add_email_object.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from pymisp.tools import EMailObject 6 | import traceback 7 | from keys import misp_url, misp_key, misp_verifycert 8 | import glob 9 | import argparse 10 | 11 | 12 | if __name__ == '__main__': 13 | parser = argparse.ArgumentParser(description='Extract indicators out of binaries and add MISP objects to a MISP instance.') 14 | parser.add_argument("-e", "--event", required=True, help="Event ID to update.") 15 | parser.add_argument("-p", "--path", required=True, help="Path to process (expanded using glob).") 16 | args = parser.parse_args() 17 | 18 | pymisp = PyMISP(misp_url, misp_key, misp_verifycert, debug=True) 19 | 20 | for f in glob.glob(args.path): 21 | try: 22 | eo = EMailObject(f) 23 | except Exception as e: 24 | traceback.print_exc() 25 | continue 26 | 27 | if eo: 28 | template_id = pymisp.get_object_template_id(eo.template_uuid) 29 | response = pymisp.add_object(args.event, template_id, eo) 30 | for ref in eo.ObjectReference: 31 | r = pymisp.add_object_reference(ref) 32 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/misp_custom_obj.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "Object": [ 4 | { 5 | "Attribute": [ 6 | { 7 | "category": "Other", 8 | "disable_correlation": false, 9 | "object_relation": "member3", 10 | "to_ids": false, 11 | "type": "text", 12 | "value": "foo" 13 | }, 14 | { 15 | "category": "Other", 16 | "disable_correlation": false, 17 | "object_relation": "member1", 18 | "to_ids": false, 19 | "type": "text", 20 | "value": "bar" 21 | } 22 | ], 23 | "description": "TestTemplate.", 24 | "distribution": 5, 25 | "meta-category": "file", 26 | "misp_objects_path_custom": "tests/mispevent_testfiles", 27 | "name": "test_object_template", 28 | "sharing_group_id": 0, 29 | "template_uuid": "4ec55cc6-9e49-4c64-b794-03c25c1a6589", 30 | "template_version": 1, 31 | "uuid": "a" 32 | } 33 | ], 34 | "analysis": "1", 35 | "date": "2017-12-31", 36 | "distribution": "1", 37 | "info": "This is a test", 38 | "threat_level_id": "1" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /pymisp/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | class PyMISPError(Exception): 5 | def __init__(self, message): 6 | super(PyMISPError, self).__init__(message) 7 | self.message = message 8 | 9 | 10 | class NewEventError(PyMISPError): 11 | pass 12 | 13 | 14 | class UpdateEventError(PyMISPError): 15 | pass 16 | 17 | 18 | class NewAttributeError(PyMISPError): 19 | pass 20 | 21 | 22 | class UpdateAttributeError(PyMISPError): 23 | pass 24 | 25 | 26 | class SearchError(PyMISPError): 27 | pass 28 | 29 | 30 | class MissingDependency(PyMISPError): 31 | pass 32 | 33 | 34 | class NoURL(PyMISPError): 35 | pass 36 | 37 | 38 | class NoKey(PyMISPError): 39 | pass 40 | 41 | 42 | class MISPObjectException(PyMISPError): 43 | pass 44 | 45 | 46 | class InvalidMISPObject(MISPObjectException): 47 | """Exception raised when an object doesn't respect the contrains in the definition""" 48 | pass 49 | 50 | 51 | class UnknownMISPObjectTemplate(MISPObjectException): 52 | """Exception raised when the template is unknown""" 53 | pass 54 | 55 | 56 | class PyMISPInvalidFormat(PyMISPError): 57 | pass 58 | 59 | 60 | class MISPServerError(PyMISPError): 61 | pass 62 | -------------------------------------------------------------------------------- /examples/create_events.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key 6 | import argparse 7 | 8 | # For python2 & 3 compat, a bit dirty, but it seems to be the least bad one 9 | try: 10 | input = raw_input 11 | except NameError: 12 | pass 13 | 14 | 15 | def init(url, key): 16 | return PyMISP(url, key, True, 'json', debug=True) 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Create an event on MISP.') 20 | parser.add_argument("-d", "--distrib", type=int, help="The distribution setting used for the attributes and for the newly created event, if relevant. [0-3].") 21 | parser.add_argument("-i", "--info", help="Used to populate the event info field if no event ID supplied.") 22 | parser.add_argument("-a", "--analysis", type=int, help="The analysis level of the newly created event, if applicable. [0-2]") 23 | parser.add_argument("-t", "--threat", type=int, help="The threat level ID of the newly created event, if applicable. [1-4]") 24 | args = parser.parse_args() 25 | 26 | misp = init(misp_url, misp_key) 27 | 28 | event = misp.new_event(args.distrib, args.threat, args.analysis, args.info) 29 | print(event) 30 | -------------------------------------------------------------------------------- /docs/source/tools.rst: -------------------------------------------------------------------------------- 1 | pymisp - Tools 2 | ============== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | .. automodule:: pymisp.tools 8 | :members: 9 | 10 | File Object 11 | ----------- 12 | 13 | .. autoclass:: FileObject 14 | :members: 15 | :inherited-members: 16 | 17 | ELF Object 18 | ---------- 19 | 20 | .. autoclass:: ELFObject 21 | :members: 22 | :inherited-members: 23 | 24 | .. autoclass:: ELFSectionObject 25 | :members: 26 | :inherited-members: 27 | 28 | PE Object 29 | --------- 30 | 31 | .. autoclass:: PEObject 32 | :members: 33 | :inherited-members: 34 | 35 | .. autoclass:: PESectionObject 36 | :members: 37 | :inherited-members: 38 | 39 | Mach-O Object 40 | ------------- 41 | 42 | .. autoclass:: MachOObject 43 | :members: 44 | :inherited-members: 45 | 46 | .. autoclass:: MachOSectionObject 47 | :members: 48 | :inherited-members: 49 | 50 | VT Report Object 51 | ---------------- 52 | 53 | .. autoclass:: VTReportObject 54 | :members: 55 | :inherited-members: 56 | 57 | STIX 58 | ---- 59 | 60 | .. automodule:: pymisp.tools.stix 61 | :members: 62 | 63 | OpenIOC 64 | -------- 65 | 66 | .. automethod:: pymisp.tools.load_openioc 67 | 68 | .. automethod:: pymisp.tools.load_openioc_file 69 | 70 | -------------------------------------------------------------------------------- /examples/situational-awareness/bokeh_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from bokeh.plotting import figure, output_file, show, ColumnDataSource 5 | from bokeh.models import HoverTool 6 | import date_tools 7 | 8 | 9 | def tagsDistributionScatterPlot(NbTags, dates, plotname='Tags Distribution Plot'): 10 | 11 | output_file(plotname + ".html") 12 | 13 | counts = {} 14 | glyphs = {} 15 | desc = {} 16 | hover = HoverTool() 17 | plot = figure(plot_width=800, plot_height=800, x_axis_type="datetime", x_axis_label='Date', y_axis_label='Number of tags', tools=[hover]) 18 | 19 | for name in NbTags.keys(): 20 | desc[name] = [] 21 | for date in dates[name]: 22 | desc[name].append(date_tools.datetimeToString(date, "%Y-%m-%d")) 23 | counts[name] = plot.circle(dates[name], NbTags[name], legend="Number of events with y tags", source=ColumnDataSource( 24 | data=dict( 25 | desc=desc[name] 26 | ) 27 | )) 28 | glyphs[name] = counts[name].glyph 29 | glyphs[name].size = int(name) * 2 30 | hover.tooltips = [("date", "@desc")] 31 | if int(name) != 0: 32 | glyphs[name].fill_alpha = 1/int(name) 33 | show(plot) 34 | -------------------------------------------------------------------------------- /examples/feed-generator-from-redis/ObjectConstructor/CowrieMISPObject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import time 4 | 5 | from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator 6 | 7 | 8 | class CowrieMISPObject(AbstractMISPObjectGenerator): 9 | def __init__(self, dico_val, **kargs): 10 | self._dico_val = dico_val 11 | self.name = "cowrie" 12 | 13 | # Enforce attribute date with timestamp 14 | super(CowrieMISPObject, self).__init__('cowrie', 15 | default_attributes_parameters={'timestamp': int(time.time())}, 16 | **kargs) 17 | self.generate_attributes() 18 | 19 | def generate_attributes(self): 20 | skip_list = ['time', 'duration', 'isError', 'ttylog'] 21 | for object_relation, value in self._dico_val.items(): 22 | if object_relation in skip_list or 'log_' in object_relation: 23 | continue 24 | 25 | if object_relation == 'timestamp': 26 | # Date already in ISO format, removing trailing Z 27 | value = value.rstrip('Z') 28 | 29 | if isinstance(value, dict): 30 | self.add_attribute(object_relation, **value) 31 | else: 32 | self.add_attribute(object_relation, value=value) 33 | -------------------------------------------------------------------------------- /examples/add_sbsignature.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pymisp import PyMISP 3 | from keys import misp_url, misp_key, misp_verifycert 4 | from pymisp.tools import SBSignatureObject 5 | 6 | pymisp = PyMISP(misp_url, misp_key, misp_verifycert) 7 | a = json.loads('{"signatures":[{"new_data":[],"confidence":100,"families":[],"severity":1,"weight":0,"description":"AttemptstoconnecttoadeadIP:Port(2uniquetimes)","alert":false,"references":[],"data":[{"IP":"95.101.39.58:80(Europe)"},{"IP":"192.35.177.64:80(UnitedStates)"}],"name":"dead_connect"},{"new_data":[],"confidence":30,"families":[],"severity":2,"weight":1,"description":"PerformssomeHTTPrequests","alert":false,"references":[],"data":[{"url":"http://cert.int-x3.letsencrypt.org/"},{"url":"http://apps.identrust.com/roots/dstrootcax3.p7c"}],"name":"network_http"},{"new_data":[],"confidence":100,"families":[],"severity":2,"weight":1,"description":"Theofficefilehasaunconventionalcodepage:ANSICyrillic;Cyrillic(Windows)","alert":false,"references":[],"data":[],"name":"office_code_page"}]}') 8 | a = [(x['name'], x['description']) for x in a["signatures"]] 9 | 10 | 11 | b = SBSignatureObject(a) 12 | 13 | 14 | template_id = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == 'sb-signature'][0] 15 | 16 | pymisp.add_object(234111, template_id, b) 17 | -------------------------------------------------------------------------------- /examples/get.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | import os 8 | import json 9 | 10 | 11 | # Usage for pipe masters: ./last.py -l 5h | jq . 12 | 13 | proxies = { 14 | 'http': 'http://127.0.0.1:8123', 15 | 'https': 'http://127.0.0.1:8123', 16 | } 17 | 18 | proxies = None 19 | 20 | 21 | def init(url, key): 22 | return PyMISP(url, key, misp_verifycert, 'json', proxies=proxies) 23 | 24 | 25 | def get_event(m, event, out=None): 26 | result = m.get_event(event) 27 | if out is None: 28 | print(json.dumps(result) + '\n') 29 | else: 30 | with open(out, 'w') as f: 31 | f.write(json.dumps(result) + '\n') 32 | 33 | if __name__ == '__main__': 34 | 35 | parser = argparse.ArgumentParser(description='Get an event from a MISP instance.') 36 | parser.add_argument("-e", "--event", required=True, help="Event ID to get.") 37 | parser.add_argument("-o", "--output", help="Output file") 38 | 39 | args = parser.parse_args() 40 | 41 | if args.output is not None and os.path.exists(args.output): 42 | print('Output file already exists, abort.') 43 | exit(0) 44 | 45 | misp = init(misp_url, misp_key) 46 | 47 | get_event(misp, args.event, args.output) 48 | -------------------------------------------------------------------------------- /examples/addtag.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | import os 8 | import json 9 | 10 | 11 | def init(url, key): 12 | return PyMISP(url, key, misp_verifycert, 'json') 13 | 14 | result = m.get_event(event) 15 | 16 | if __name__ == '__main__': 17 | parser = argparse.ArgumentParser(description='Get an event from a MISP instance.') 18 | parser.add_argument("-e", "--event", required=True, help="Event ID to get.") 19 | parser.add_argument("-a", "--attribute", help="Attribute ID to modify. A little dirty for now, argument need to be included in event") 20 | parser.add_argument("-t", "--tag", required=True, type=int, help="Attribute ID to modify.") 21 | parser.add_argument("-m", "--modify_attribute", action='store_true', help="If set, the tag will be add to the attribute, otherwise to the event.") 22 | 23 | args = parser.parse_args() 24 | 25 | misp = init(misp_url, misp_key) 26 | 27 | event = misp.get_event(args.event) 28 | if args.modify_attribute: 29 | for temp in event['Event']['Attribute']: 30 | if temp['id'] == args.attribute: 31 | attribute = temp 32 | break 33 | 34 | misp.add_tag(attribute, args.tag, attribute=True) 35 | else: 36 | misp.add_tag(event['Event'], args.tag) 37 | -------------------------------------------------------------------------------- /examples/last.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | import os 8 | import json 9 | 10 | 11 | # Usage for pipe masters: ./last.py -l 5h | jq . 12 | 13 | 14 | def init(url, key): 15 | return PyMISP(url, key, misp_verifycert, 'json') 16 | 17 | 18 | def download_last(m, last, out=None): 19 | result = m.download_last(last) 20 | if out is None: 21 | if 'response' in result: 22 | print(json.dumps(result['response'])) 23 | else: 24 | print('No results for that time period') 25 | exit(0) 26 | else: 27 | with open(out, 'w') as f: 28 | f.write(json.dumps(result['response'])) 29 | 30 | if __name__ == '__main__': 31 | parser = argparse.ArgumentParser(description='Download latest events from a MISP instance.') 32 | parser.add_argument("-l", "--last", required=True, help="can be defined in days, hours, minutes (for example 5d or 12h or 30m).") 33 | parser.add_argument("-o", "--output", help="Output file") 34 | 35 | args = parser.parse_args() 36 | 37 | if args.output is not None and os.path.exists(args.output): 38 | print('Output file already exists, abord.') 39 | exit(0) 40 | 41 | misp = init(misp_url, misp_key) 42 | 43 | download_last(misp, args.last, args.output) 44 | -------------------------------------------------------------------------------- /examples/searchall.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key,misp_verifycert 6 | import argparse 7 | import os 8 | import json 9 | 10 | 11 | def init(url, key): 12 | return PyMISP(url, key, misp_verifycert, 'json') 13 | 14 | 15 | def searchall(m, search, quiet, url, out=None): 16 | result = m.search_all(search) 17 | if quiet: 18 | for e in result['response']: 19 | print('{}{}{}\n'.format(url, '/events/view/', e['Event']['id'])) 20 | elif out is None: 21 | print(json.dumps(result['response'])) 22 | else: 23 | with open(out, 'w') as f: 24 | f.write(json.dumps(result['response'])) 25 | 26 | 27 | if __name__ == '__main__': 28 | parser = argparse.ArgumentParser(description='Get all the events matching a value.') 29 | parser.add_argument("-s", "--search", required=True, help="String to search.") 30 | parser.add_argument("-q", "--quiet", action='store_true', help="Only display URLs to MISP") 31 | parser.add_argument("-o", "--output", help="Output file") 32 | 33 | args = parser.parse_args() 34 | 35 | if args.output is not None and os.path.exists(args.output): 36 | print('Output file already exists, abord.') 37 | exit(0) 38 | 39 | misp = init(misp_url, misp_key) 40 | 41 | searchall(misp, args.search, args.quiet, misp_url, args.output) 42 | -------------------------------------------------------------------------------- /docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | pymisp 2 | ====== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | .. automodule:: pymisp 8 | :members: 9 | 10 | 11 | PyMISP 12 | ------ 13 | 14 | .. autoclass:: PyMISP 15 | :members: 16 | 17 | MISPAbstract 18 | ------------ 19 | 20 | .. autoclass:: AbstractMISP 21 | :members: 22 | 23 | MISPEncode 24 | ---------- 25 | 26 | .. autoclass:: MISPEncode 27 | :members: 28 | 29 | MISPEvent 30 | --------- 31 | 32 | .. autoclass:: MISPEvent 33 | :members: 34 | :inherited-members: 35 | 36 | MISPAttribute 37 | ------------- 38 | 39 | .. autoclass:: MISPAttribute 40 | :members: 41 | :inherited-members: 42 | 43 | MISPObject 44 | ---------- 45 | 46 | .. autoclass:: MISPObject 47 | :members: 48 | :inherited-members: 49 | 50 | MISPObjectAttribute 51 | ------------------- 52 | 53 | .. autoclass:: MISPObjectAttribute 54 | :members: 55 | :inherited-members: 56 | 57 | MISPObjectReference 58 | ------------------- 59 | 60 | .. autoclass:: MISPObjectReference 61 | :members: 62 | :inherited-members: 63 | 64 | MISPTag 65 | ------- 66 | 67 | .. autoclass:: MISPTag 68 | :members: 69 | :inherited-members: 70 | 71 | MISPUser 72 | -------- 73 | 74 | .. autoclass:: MISPUser 75 | :members: 76 | :inherited-members: 77 | 78 | 79 | MISPOrganisation 80 | ---------------- 81 | 82 | .. autoclass:: MISPOrganisation 83 | :members: 84 | :inherited-members: 85 | 86 | -------------------------------------------------------------------------------- /pymisp/tools/stix.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | try: 4 | from misp_stix_converter.converters.buildMISPAttribute import buildEvent 5 | from misp_stix_converter.converters import convert 6 | from misp_stix_converter.converters.convert import MISPtoSTIX 7 | has_misp_stix_converter = True 8 | except ImportError: 9 | has_misp_stix_converter = False 10 | 11 | 12 | def load_stix(stix, distribution=3, threat_level_id=2, analysis=0): 13 | '''Returns a MISPEvent object from a STIX package''' 14 | if not has_misp_stix_converter: 15 | raise Exception('You need to install misp_stix_converter: pip install git+https://github.com/MISP/MISP-STIX-Converter.git') 16 | stix = convert.load_stix(stix) 17 | return buildEvent(stix, distribution=distribution, 18 | threat_level_id=threat_level_id, analysis=analysis) 19 | 20 | 21 | def make_stix_package(misp_event, to_json=False, to_xml=False): 22 | '''Returns a STIXPackage from a MISPEvent. 23 | 24 | Optionally can return the package in json or xml. 25 | 26 | ''' 27 | if not has_misp_stix_converter: 28 | raise Exception('You need to install misp_stix_converter: pip install git+https://github.com/MISP/MISP-STIX-Converter.git') 29 | package = MISPtoSTIX(misp_event) 30 | if to_json: 31 | return package.to_json() 32 | elif to_xml: 33 | return package.to_xml() 34 | else: 35 | return package 36 | -------------------------------------------------------------------------------- /examples/get_csv.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | 6 | from pymisp import PyMISP 7 | from keys import misp_url, misp_key, misp_verifycert 8 | 9 | 10 | if __name__ == '__main__': 11 | parser = argparse.ArgumentParser(description='Get MISP stuff as CSV.') 12 | parser.add_argument("-e", "--event_id", help="Event ID to fetch. Without it, it will fetch the whole database.") 13 | parser.add_argument("-a", "--attribute", nargs='+', help="Attribute column names") 14 | parser.add_argument("-o", "--object_attribute", nargs='+', help="Object attribute column names") 15 | parser.add_argument("-t", "--misp_types", nargs='+', help="MISP types to fetch (ip-src, hostname, ...)") 16 | parser.add_argument("-c", "--context", action='store_true', help="Add event level context (tags...)") 17 | parser.add_argument("-i", "--ignore", action='store_true', help="Returns the attributes even if the event isn't published, or the attribute doesn't have the to_ids flag") 18 | parser.add_argument("-f", "--outfile", help="Output file to write the CSV.") 19 | 20 | args = parser.parse_args() 21 | pymisp = PyMISP(misp_url, misp_key, misp_verifycert, debug=True) 22 | response = pymisp.get_csv(args.event_id, args.attribute, args.object_attribute, args.misp_types, args.context, args.ignore) 23 | 24 | if args.outfile: 25 | with open(args.outfile, 'w') as f: 26 | f.write(response) 27 | else: 28 | print(response) 29 | -------------------------------------------------------------------------------- /examples/search_attributes_yara.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # Example of specifying special attribute type in your search: here yara attribute 4 | 5 | from pymisp import PyMISP 6 | from keys import misp_url, misp_key,misp_verifycert 7 | import argparse 8 | import os 9 | import json 10 | 11 | def init(url, key): 12 | return PyMISP(url, key, misp_verifycert, 'json') 13 | 14 | def search(m, quiet, url, out=None, custom_type_attribute="yara"): 15 | controller='attributes' 16 | result = m.search(controller, type_attribute = custom_type_attribute) 17 | if quiet: 18 | for e in result['response']: 19 | print('{}{}{}\n'.format(url, '/events/view/', e['Event']['id'])) 20 | elif out is None: 21 | print(json.dumps(result['response'])) 22 | else: 23 | with open(out, 'w') as f: 24 | f.write(json.dumps(result['response'])) 25 | 26 | 27 | if __name__ == '__main__': 28 | parser = argparse.ArgumentParser(description='Get all the events matching a value for a given param.') 29 | parser.add_argument("-q", "--quiet", action='store_true', help="Only display URLs to MISP") 30 | parser.add_argument("-o", "--output", help="Output file") 31 | 32 | args = parser.parse_args() 33 | 34 | if args.output is not None and os.path.exists(args.output): 35 | print('Output file already exists, abort.') 36 | exit(0) 37 | 38 | misp = init(misp_url, misp_key) 39 | 40 | search(misp, args.quiet, misp_url, args.output) 41 | -------------------------------------------------------------------------------- /examples/add_generic_object.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import json 5 | from pymisp import PyMISP 6 | from pymisp.tools import GenericObjectGenerator 7 | from keys import misp_url, misp_key, misp_verifycert 8 | import argparse 9 | 10 | """ 11 | Sample usage: 12 | ./add_generic_object.py -e 5065 -t email -l '[{"to": "undisclosed@ppp.com"}, {"to": "second.to@mail.com"}]' 13 | """ 14 | 15 | if __name__ == '__main__': 16 | parser = argparse.ArgumentParser(description='Create a MISP Object selectable by type starting from a dictionary') 17 | parser.add_argument("-e", "--event", required=True, help="Event ID to update") 18 | parser.add_argument("-t", "--type", required=True, help="Type of the generic object") 19 | parser.add_argument("-l", "--attr_list", required=True, help="List of attributes") 20 | args = parser.parse_args() 21 | 22 | pymisp = PyMISP(misp_url, misp_key, misp_verifycert) 23 | try: 24 | template_id = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == args.type][0] 25 | except IndexError: 26 | valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) 27 | print ("Template for type %s not found! Valid types are: %s" % (args.type, valid_types)) 28 | exit() 29 | 30 | misp_object = GenericObjectGenerator(args.type.replace("|", "-")) 31 | misp_object.generate_attributes(json.loads(args.attr_list)) 32 | r = pymisp.add_object(args.event, template_id, misp_object) 33 | -------------------------------------------------------------------------------- /examples/feed-generator/settings.default.py: -------------------------------------------------------------------------------- 1 | # Your MISP's URL 2 | url = '' 3 | 4 | # The auth key to the MISP user that you wish to use. Make sure that the 5 | # user has auth_key access 6 | key = '' 7 | 8 | # Should the certificate be validated? 9 | ssl = False 10 | 11 | # The output dir for the feed. This will drop a lot of files, so make 12 | # sure that you use a directory dedicated to the feed 13 | outputdir = 'output' 14 | 15 | # The filters to be used for by the feed. You can use any filter that 16 | # you can use on the event index, such as organisation, tags, etc. 17 | # It uses the same joining and condition rules as the API parameters 18 | # For example: 19 | # filters = {'tag':'tlp:white|feed-export|!privint','org':'CIRCL', 'published':1} 20 | # the above would generate a feed for all published events created by CIRCL, 21 | # tagged tlp:white and/or feed-export but exclude anything tagged privint 22 | filters = {'published':'true'} 23 | 24 | 25 | # By default all attributes will be included in the feed generation 26 | # Remove the levels that you do not wish to include in the feed 27 | # Use this to further narrow down what gets exported, for example: 28 | # Setting this to ['3', '5'] will exclude any attributes from the feed that 29 | # are not exportable to all or inherit the event 30 | # 31 | # The levels are as follows: 32 | # 0: Your Organisation Only 33 | # 1: This Community Only 34 | # 2: Connected Communities 35 | # 3: All 36 | # 4: Sharing Group 37 | # 5: Inherit Event 38 | valid_attribute_distribution_levels = ['0', '1', '2', '3', '4', '5'] 39 | 40 | -------------------------------------------------------------------------------- /examples/feed-generator-from-redis/settings.default.py: -------------------------------------------------------------------------------- 1 | """ REDIS RELATED """ 2 | # Your redis server 3 | host='127.0.0.1' 4 | port=6379 5 | db=0 6 | ## The keynames to POP element from 7 | #keyname_pop='misp_feed_generator_key' 8 | keyname_pop=['cowrie'] 9 | 10 | # OTHERS 11 | ## How frequent the event should be written on disk 12 | flushing_interval=5*60 13 | ## The redis list keyname in which to put items that generated an error 14 | keyname_error='feed-generation-error' 15 | 16 | """ FEED GENERATOR CONFIGURATION """ 17 | 18 | # The output dir for the feed. This will drop a lot of files, so make 19 | # sure that you use a directory dedicated to the feed 20 | outputdir = 'output' 21 | 22 | # Event meta data 23 | ## Required 24 | ### The organisation id that generated this feed 25 | org_name='myOrg' 26 | ### Your organisation UUID 27 | org_uuid='' 28 | ### The daily event name to be used in MISP. 29 | ### (e.g. honeypot_1, will produce each day an event of the form honeypot_1 dd-mm-yyyy) 30 | daily_event_name='PyMISP default event name' 31 | 32 | ## Optional 33 | analysis=0 34 | threat_level_id=3 35 | published=False 36 | Tag=[ 37 | { 38 | "colour": "#ffffff", 39 | "name": "tlp:white" 40 | }, 41 | { 42 | "colour": "#ff00ff", 43 | "name": "my:custom:feed" 44 | } 45 | ] 46 | 47 | # MISP Object constructor 48 | from ObjectConstructor.CowrieMISPObject import CowrieMISPObject 49 | from pymisp.tools import GenericObjectGenerator 50 | 51 | constructor_dict = { 52 | 'cowrie': CowrieMISPObject, 53 | 'generic': GenericObjectGenerator 54 | } 55 | 56 | # Others 57 | ## Redis pooling time 58 | sleep=60 59 | -------------------------------------------------------------------------------- /examples/situational-awareness/attribute_treemap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | import tools 8 | import pygal_tools 9 | 10 | if __name__ == '__main__': 11 | parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py of searchall.py) and create a treemap epresenting the distribution of attributes in this sample.') 12 | parser.add_argument("-f", "--function", required=True, help='The parameter can be either set to "last" or "searchall". If the parameter is not valid, "last" will be the default setting.') 13 | parser.add_argument("-a", "--argument", required=True, help='if function is "last", time can be defined in days, hours, minutes (for example 5d or 12h or 30m). Otherwise, this argument is the string to search') 14 | 15 | args = parser.parse_args() 16 | 17 | misp = PyMISP(misp_url, misp_key, misp_verifycert, 'json') 18 | 19 | if args.function == "searchall": 20 | result = misp.search_all(args.argument) 21 | else: 22 | result = misp.download_last(args.argument) 23 | 24 | if 'response' in result: 25 | events = tools.eventsListBuildFromArray(result) 26 | attributes = tools.attributesListBuild(events) 27 | temp = tools.getNbAttributePerEventCategoryType(attributes) 28 | temp = temp.groupby(level=['category', 'type']).sum() 29 | pygal_tools.createTreemap(temp, 'Attributes Distribution', 'attribute_treemap.svg', 'attribute_table.html') 30 | else: 31 | print ('There is no event answering the research criteria') 32 | -------------------------------------------------------------------------------- /pymisp/tools/genericgenerator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from .abstractgenerator import AbstractMISPObjectGenerator 5 | 6 | 7 | class GenericObjectGenerator(AbstractMISPObjectGenerator): 8 | 9 | def generate_attributes(self, attributes): 10 | """Generates MISPObjectAttributes from a list of dictionaries. 11 | Each entry if the list must be in one of the two following formats: 12 | * {: } 13 | * {: {'value'=, 'type'=, ]} 14 | 15 | Note: Any missing parameter will default to the pre-defined value from the Object template. 16 | If the object template isn't known by PyMISP, you *must* pass a type key/value, or it will fail. 17 | 18 | Example: 19 | [{'analysis_submitted_at': '2018-06-15T06:40:27'}, 20 | {'threat_score': {value=95, to_ids=False}}, 21 | {'permalink': 'https://panacea.threatgrid.com/mask/samples/2e445ef5389d8b'}, 22 | {'heuristic_raw_score': 7.8385159793597}, {'heuristic_score': 96}, 23 | {'original_filename': 'juice.exe'}, {'id': '2e445ef5389d8b'}] 24 | """ 25 | for attribute in attributes: 26 | for object_relation, value in attribute.items(): 27 | if isinstance(value, dict): 28 | self.add_attribute(object_relation, **value) 29 | else: 30 | # In this case, we need a valid template, as all the other parameters will be pre-set. 31 | self.add_attribute(object_relation, value=value) 32 | -------------------------------------------------------------------------------- /examples/misp2clamav.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # vim: tabstop=4 shiftwidth=4 expandtab 4 | # 5 | # Export file hashes from MISP to ClamAV hdb file 6 | 7 | import sys 8 | from pymisp import PyMISP, MISPAttribute 9 | from keys import misp_url, misp_key 10 | 11 | 12 | def init_misp(): 13 | global mymisp 14 | mymisp = PyMISP(misp_url, misp_key) 15 | 16 | 17 | def echeck(r): 18 | if r.get('errors'): 19 | if r.get('message') == 'No matches.': 20 | return 21 | else: 22 | print(r['errors']) 23 | sys.exit(1) 24 | 25 | 26 | def find_hashes(htype): 27 | r = mymisp.search(controller='attributes', type_attribute=htype) 28 | echeck(r) 29 | if not r.get('response'): 30 | return 31 | for a in r['response']['Attribute']: 32 | attribute = MISPAttribute(mymisp.describe_types) 33 | attribute.from_dict(**a) 34 | if '|' in attribute.type and '|' in attribute.value: 35 | c, value = attribute.value.split('|') 36 | comment = '{} - {}'.format(attribute.comment, c) 37 | else: 38 | comment = attribute.comment 39 | value = attribute.value 40 | mhash = value.replace(':', ';') 41 | mfile = 'MISP event {} {}'.format(a['event_id'], comment.replace(':', ';').replace('\r', '').replace('\n', '')) 42 | print('{}:*:{}:73'.format(mhash, mfile)) 43 | 44 | 45 | if __name__ == '__main__': 46 | init_misp() 47 | find_hashes('md5') 48 | find_hashes('sha1') 49 | find_hashes('sha256') 50 | find_hashes('filename|md5') 51 | find_hashes('filename|sha1') 52 | find_hashes('filename|sha256') 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013, 2014 Raphaël Vinot 2 | Copyright (c) 2013, 2014 Alexandre Dulaunoy 3 | Copyright (c) 2013, 2014 CIRCL - Computer Incident Response Center Luxembourg 4 | (c/o smile, security made in Lëtzebuerg, Groupement 5 | d'Intérêt Economique) 6 | Copyright (c) 2014 Koen Van Impe 7 | 8 | All rights reserved. 9 | 10 | Redistribution and use in source and binary forms, with or without modification, 11 | are permitted provided that the following conditions are met: 12 | 13 | 1. Redistributions of source code must retain the above copyright notice, 14 | this list of conditions and the following disclaimer. 15 | 2. Redistributions in binary form must reproduce the above copyright notice, 16 | this list of conditions and the following disclaimer in the documentation 17 | and/or other materials provided with the distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 23 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 26 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 27 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 28 | OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /examples/add_file_object.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from pymisp.tools import make_binary_objects 6 | import traceback 7 | from keys import misp_url, misp_key, misp_verifycert 8 | import glob 9 | import argparse 10 | 11 | if __name__ == '__main__': 12 | parser = argparse.ArgumentParser(description='Extract indicators out of binaries and add MISP objects to a MISP instance.') 13 | parser.add_argument("-e", "--event", required=True, help="Event ID to update.") 14 | parser.add_argument("-p", "--path", required=True, help="Path to process (expanded using glob).") 15 | args = parser.parse_args() 16 | 17 | pymisp = PyMISP(misp_url, misp_key, misp_verifycert) 18 | 19 | for f in glob.glob(args.path): 20 | try: 21 | fo, peo, seos = make_binary_objects(f) 22 | except Exception as e: 23 | traceback.print_exc() 24 | continue 25 | 26 | if seos: 27 | for s in seos: 28 | template_id = pymisp.get_object_template_id(s.template_uuid) 29 | r = pymisp.add_object(args.event, template_id, s) 30 | 31 | if peo: 32 | template_id = pymisp.get_object_template_id(peo.template_uuid) 33 | r = pymisp.add_object(args.event, template_id, peo) 34 | for ref in peo.ObjectReference: 35 | r = pymisp.add_object_reference(ref) 36 | 37 | if fo: 38 | template_id = pymisp.get_object_template_id(fo.template_uuid) 39 | response = pymisp.add_object(args.event, template_id, fo) 40 | for ref in fo.ObjectReference: 41 | r = pymisp.add_object_reference(ref) 42 | -------------------------------------------------------------------------------- /examples/events/dummy: -------------------------------------------------------------------------------- 1 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 2 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 3 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 4 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 5 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 6 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 7 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 8 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 9 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 10 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 11 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 12 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 13 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 14 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 15 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 16 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 17 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 18 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 19 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 20 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 21 | DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY DUMMY 22 | -------------------------------------------------------------------------------- /examples/graphdb/make_neo4j.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from pymisp import Neo4j 6 | from pymisp import MISPEvent 7 | from keys import misp_url, misp_key 8 | import argparse 9 | 10 | """ 11 | Sample Neo4J query: 12 | 13 | 14 | MATCH ()-[r:has]->(n) 15 | WITH n, count(r) as rel_cnt 16 | WHERE rel_cnt > 5 17 | MATCH (m)-[r:has]->(n) 18 | RETURN m, n LIMIT 200; 19 | """ 20 | 21 | if __name__ == '__main__': 22 | parser = argparse.ArgumentParser(description='Get all the events matching a value.') 23 | parser.add_argument("-s", "--search", required=True, help="String to search.") 24 | parser.add_argument("--host", default='localhost:7474', help="Host where neo4j is running.") 25 | parser.add_argument("-u", "--user", default='neo4j', help="User on neo4j.") 26 | parser.add_argument("-p", "--password", default='neo4j', help="Password on neo4j.") 27 | parser.add_argument("-d", "--deleteall", action="store_true", default=False, help="Delete all nodes from the database") 28 | args = parser.parse_args() 29 | 30 | neo4j = Neo4j(args.host, args.user, args.password) 31 | if args.deleteall: 32 | neo4j.del_all() 33 | misp = PyMISP(misp_url, misp_key) 34 | result = misp.search_all(args.search) 35 | for json_event in result['response']: 36 | if not json_event['Event']: 37 | print(json_event) 38 | continue 39 | print('Importing', json_event['Event']['info'], json_event['Event']['id']) 40 | try: 41 | misp_event = MISPEvent() 42 | misp_event.load(json_event) 43 | neo4j.import_event(misp_event) 44 | except: 45 | print('broken') 46 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/event_obj_def_param.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "Object": [ 4 | { 5 | "Attribute": [ 6 | { 7 | "Tag": [ 8 | { 9 | "name": "blah" 10 | } 11 | ], 12 | "category": "Payload delivery", 13 | "disable_correlation": true, 14 | "object_relation": "filename", 15 | "to_ids": true, 16 | "type": "filename", 17 | "value": "bar" 18 | } 19 | ], 20 | "description": "File object describing a file with meta-information", 21 | "distribution": 5, 22 | "meta-category": "file", 23 | "name": "file", 24 | "sharing_group_id": 0, 25 | "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", 26 | "template_version": 13, 27 | "uuid": "a" 28 | }, 29 | { 30 | "Attribute": [ 31 | { 32 | "Tag": [ 33 | { 34 | "name": "blah" 35 | } 36 | ], 37 | "category": "Payload delivery", 38 | "disable_correlation": true, 39 | "object_relation": "filename", 40 | "to_ids": true, 41 | "type": "filename", 42 | "value": "baz" 43 | } 44 | ], 45 | "description": "File object describing a file with meta-information", 46 | "distribution": 5, 47 | "meta-category": "file", 48 | "name": "file", 49 | "sharing_group_id": 0, 50 | "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", 51 | "template_version": 13, 52 | "uuid": "b" 53 | } 54 | ] 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /tests/search_index_result.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "3", 4 | "org": "", 5 | "date": "2016-12-01", 6 | "info": "Another random Event", 7 | "published": false, 8 | "uuid": "5758ebf5-c898-48e6-9fe9-5665c0a83866", 9 | "attribute_count": "2", 10 | "analysis": "0", 11 | "orgc": "", 12 | "timestamp": "1465681801", 13 | "distribution": "3", 14 | "proposal_email_lock": false, 15 | "locked": false, 16 | "threat_level_id": "1", 17 | "publish_timestamp": "0", 18 | "sharing_group_id": "0", 19 | "org_id": "1", 20 | "orgc_id": "1", 21 | "Org": { 22 | "id": "1", 23 | "name": "ORGNAME" 24 | }, 25 | "Orgc": { 26 | "id": "1", 27 | "name": "ORGNAME" 28 | }, 29 | "EventTag": [ 30 | { 31 | "id": "9760", 32 | "event_id": "6028", 33 | "tag_id": "4", 34 | "Tag": { 35 | "id": "4", 36 | "name": "TLP:GREEN", 37 | "colour": "#33822d", 38 | "exportable": true 39 | } 40 | }, 41 | { 42 | "id": "9801", 43 | "event_id": "3", 44 | "tag_id": "1", 45 | "Tag": { 46 | "id": "1", 47 | "name": "for_intelmq_processing", 48 | "colour": "#00ad1c", 49 | "exportable": true 50 | } 51 | }, 52 | { 53 | "id": "9803", 54 | "event_id": "3", 55 | "tag_id": "6", 56 | "Tag": { 57 | "id": "6", 58 | "name": "ecsirt:malicious-code=\"ransomware\"", 59 | "colour": "#005a5a", 60 | "exportable": true 61 | } 62 | } 63 | ], 64 | "SharingGroup": { 65 | "id": null, 66 | "name": null 67 | } 68 | } 69 | ] 70 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/def_param.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "Object": [ 4 | { 5 | "Attribute": [ 6 | { 7 | "category": "Attribution", 8 | "disable_correlation": false, 9 | "object_relation": "registrar", 10 | "to_ids": false, 11 | "type": "whois-registrar", 12 | "value": "registar.example.com" 13 | }, 14 | { 15 | "category": "Network activity", 16 | "disable_correlation": false, 17 | "object_relation": "domain", 18 | "to_ids": true, 19 | "type": "domain", 20 | "value": "domain.example.com" 21 | }, 22 | { 23 | "category": "Network activity", 24 | "disable_correlation": true, 25 | "object_relation": "nameserver", 26 | "to_ids": false, 27 | "type": "hostname", 28 | "value": "ns1.example.com" 29 | }, 30 | { 31 | "category": "External analysis", 32 | "disable_correlation": false, 33 | "object_relation": "nameserver", 34 | "to_ids": true, 35 | "type": "hostname", 36 | "value": "ns2.example.com" 37 | } 38 | ], 39 | "description": "Whois records information for a domain name or an IP address.", 40 | "distribution": 5, 41 | "meta-category": "network", 42 | "name": "whois", 43 | "sharing_group_id": 0, 44 | "template_uuid": "429faea1-34ff-47af-8a00-7c62d3be5a6a", 45 | "template_version": 10, 46 | "uuid": "a" 47 | } 48 | ], 49 | "analysis": "1", 50 | "date": "2017-12-31", 51 | "distribution": "1", 52 | "info": "This is a test", 53 | "threat_level_id": "1" 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /examples/addtag2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from pymisp import PyMISP 4 | from keys import misp_url, misp_key, misp_verifycert 5 | import argparse 6 | import os 7 | import json 8 | 9 | 10 | def init(url, key): 11 | return PyMISP(url, key, misp_verifycert, 'json') 12 | 13 | result = m.get_event(event) 14 | 15 | if __name__ == '__main__': 16 | parser = argparse.ArgumentParser(description='Tag something.') 17 | parser.add_argument("-u", "--uuid", help="UUID to tag.") 18 | parser.add_argument("-e", "--event", help="Event ID to tag.") 19 | parser.add_argument("-a", "--attribute", help="Attribute ID to tag") 20 | parser.add_argument("-t", "--tag", required=True, help="Attribute ID to modify.") 21 | args = parser.parse_args() 22 | 23 | if not args.event and not args.uuid and not args.attribute: 24 | print("Please provide at least one of the following : uuid, eventID or attribute ID, see --help") 25 | exit() 26 | 27 | misp = init(misp_url, misp_key) 28 | 29 | event = misp.get_event(args.event) 30 | 31 | if args.event and not args.attribute: 32 | result = misp.search(eventid=args.event) 33 | data = result['response'] 34 | for event in data: 35 | uuid = event['Event']['uuid'] 36 | 37 | if args.attribute: 38 | if not args.event: 39 | print("Please provide event ID also") 40 | exit() 41 | result = misp.search(eventid=args.event) 42 | data = result['response'] 43 | for event in data: 44 | for attribute in event['Event']['Attribute']: 45 | if attribute["id"] == args.attribute: 46 | uuid = attribute["uuid"] 47 | 48 | if args.uuid: 49 | uuid = args.uuid 50 | 51 | print("UUID tagged: %s"%uuid) 52 | misp.tag(uuid, args.tag) 53 | -------------------------------------------------------------------------------- /examples/search.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key,misp_verifycert 6 | import argparse 7 | import os 8 | import json 9 | 10 | 11 | def init(url, key): 12 | return PyMISP(url, key, misp_verifycert, 'json') 13 | 14 | 15 | def search(m, quiet, url, controller, out=None, **kwargs): 16 | result = m.search(controller, **kwargs) 17 | if quiet: 18 | for e in result['response']: 19 | print('{}{}{}\n'.format(url, '/events/view/', e['Event']['id'])) 20 | elif out is None: 21 | print(json.dumps(result['response'])) 22 | else: 23 | with open(out, 'w') as f: 24 | f.write(json.dumps(result['response'])) 25 | 26 | 27 | if __name__ == '__main__': 28 | parser = argparse.ArgumentParser(description='Get all the events matching a value for a given param.') 29 | parser.add_argument("-p", "--param", required=True, help="Parameter to search (e.g. category, org, etc.)") 30 | parser.add_argument("-s", "--search", required=True, help="String to search.") 31 | parser.add_argument("-a", "--attributes", action='store_true', help="Search attributes instead of events") 32 | parser.add_argument("-q", "--quiet", action='store_true', help="Only display URLs to MISP") 33 | parser.add_argument("-o", "--output", help="Output file") 34 | 35 | args = parser.parse_args() 36 | 37 | if args.output is not None and os.path.exists(args.output): 38 | print('Output file already exists, abort.') 39 | exit(0) 40 | 41 | misp = init(misp_url, misp_key) 42 | kwargs = {args.param: args.search} 43 | 44 | if args.attributes: 45 | controller='attributes' 46 | else: 47 | controller='events' 48 | 49 | search(misp, args.quiet, misp_url, controller, args.output, **kwargs) 50 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/event_obj_attr_tag.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "Object": [ 4 | { 5 | "Attribute": [ 6 | { 7 | "Tag": [ 8 | { 9 | "name": "blah" 10 | } 11 | ], 12 | "category": "Payload delivery", 13 | "disable_correlation": true, 14 | "object_relation": "filename", 15 | "to_ids": true, 16 | "type": "filename", 17 | "value": "bar" 18 | } 19 | ], 20 | "ObjectReference": [ 21 | { 22 | "comment": "foo", 23 | "object_uuid": "a", 24 | "referenced_uuid": "b", 25 | "relationship_type": "baz" 26 | } 27 | ], 28 | "description": "File object describing a file with meta-information", 29 | "distribution": 5, 30 | "meta-category": "file", 31 | "name": "file", 32 | "sharing_group_id": 0, 33 | "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", 34 | "template_version": 13, 35 | "uuid": "a" 36 | }, 37 | { 38 | "Attribute": [ 39 | { 40 | "category": "Network activity", 41 | "disable_correlation": false, 42 | "object_relation": "url", 43 | "to_ids": true, 44 | "type": "url", 45 | "value": "https://www.circl.lu" 46 | } 47 | ], 48 | "description": "url object describes an url along with its normalized field (like extracted using faup parsing library) and its metadata.", 49 | "distribution": 5, 50 | "meta-category": "network", 51 | "name": "url", 52 | "sharing_group_id": 0, 53 | "template_uuid": "60efb77b-40b5-4c46-871b-ed1ed999fce5", 54 | "template_version": 6, 55 | "uuid": "b" 56 | } 57 | ] 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | from setuptools import setup 4 | import pymisp 5 | 6 | 7 | setup( 8 | name='pymisp', 9 | version=pymisp.__version__, 10 | author='Raphaël Vinot', 11 | author_email='raphael.vinot@circl.lu', 12 | maintainer='Raphaël Vinot', 13 | url='https://github.com/MISP/PyMISP', 14 | description='Python API for MISP.', 15 | packages=['pymisp', 'pymisp.tools'], 16 | classifiers=[ 17 | 'License :: OSI Approved :: BSD License', 18 | 'Development Status :: 5 - Production/Stable', 19 | 'Environment :: Console', 20 | 'Operating System :: POSIX :: Linux', 21 | 'Intended Audience :: Science/Research', 22 | 'Intended Audience :: Telecommunications Industry', 23 | 'Intended Audience :: Information Technology', 24 | 'Programming Language :: Python :: 2.7', 25 | 'Programming Language :: Python :: 3', 26 | 'Topic :: Security', 27 | 'Topic :: Internet', 28 | ], 29 | test_suite="tests.test_offline", 30 | install_requires=['six', 'requests', 'python-dateutil', 'jsonschema', 'setuptools>=36.4', 'python-dateutil'], 31 | extras_require={'fileobjects': ['lief>=0.8', 'python-magic'], 32 | 'neo': ['py2neo'], 33 | 'openioc': ['beautifulsoup4'], 34 | 'virustotal': ['validators'], 35 | 'warninglists': ['pymispwarninglists']}, 36 | tests_require=[ 37 | 'jsonschema', 38 | 'python-magic', 39 | 'requests-mock', 40 | 'six' 41 | ], 42 | include_package_data=True, 43 | package_data={'pymisp': ['data/*.json', 44 | 'data/misp-objects/schema_objects.json', 45 | 'data/misp-objects/schema_relationships.json', 46 | 'data/misp-objects/objects/*/definition.json', 47 | 'data/misp-objects/relationships/definition.json']}, 48 | ) 49 | -------------------------------------------------------------------------------- /pymisp/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '2.4.93' 2 | import logging 3 | import functools 4 | import warnings 5 | import sys 6 | 7 | FORMAT = "%(levelname)s [%(filename)s:%(lineno)s - %(funcName)s() ] %(message)s" 8 | formatter = logging.Formatter(FORMAT) 9 | default_handler = logging.StreamHandler() 10 | default_handler.setFormatter(formatter) 11 | 12 | logger = logging.getLogger(__name__) 13 | logger.addHandler(default_handler) 14 | logger.setLevel(logging.WARNING) 15 | 16 | 17 | def deprecated(func): 18 | '''This is a decorator which can be used to mark functions 19 | as deprecated. It will result in a warning being emitted 20 | when the function is used.''' 21 | 22 | @functools.wraps(func) 23 | def new_func(*args, **kwargs): 24 | warnings.showwarning( 25 | "Call to deprecated function {}.".format(func.__name__), 26 | category=DeprecationWarning, 27 | filename=func.__code__.co_filename, 28 | lineno=func.__code__.co_firstlineno + 1 29 | ) 30 | return func(*args, **kwargs) 31 | return new_func 32 | 33 | 34 | try: 35 | from .exceptions import PyMISPError, NewEventError, NewAttributeError, MissingDependency, NoURL, NoKey, InvalidMISPObject, UnknownMISPObjectTemplate, PyMISPInvalidFormat, MISPServerError # noqa 36 | from .api import PyMISP # noqa 37 | from .abstract import AbstractMISP, MISPEncode, MISPTag, Distribution, ThreatLevel, Analysis # noqa 38 | from .mispevent import MISPEvent, MISPAttribute, MISPObjectReference, MISPObjectAttribute, MISPObject, MISPUser, MISPOrganisation, MISPSighting # noqa 39 | from .tools import AbstractMISPObjectGenerator # noqa 40 | from .tools import Neo4j # noqa 41 | from .tools import stix # noqa 42 | from .tools import openioc # noqa 43 | from .tools import load_warninglists # noqa 44 | from .tools import ext_lookups # noqa 45 | if sys.version_info >= (3, 6): 46 | from .aping import ExpandedPyMISP # noqa 47 | logger.debug('pymisp loaded properly') 48 | except ImportError as e: 49 | logger.warning('Unable to load pymisp properly: {}'.format(e)) 50 | -------------------------------------------------------------------------------- /examples/situational-awareness/date_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from datetime import datetime 5 | from datetime import timedelta 6 | from dateutil.parser import parse 7 | 8 | 9 | class DateError(Exception): 10 | def __init__(self, value): 11 | self.value = value 12 | 13 | def __str__(self): 14 | return repr(self.value) 15 | 16 | 17 | # ############### Date Tools ################ 18 | 19 | def dateInRange(datetimeTested, begin=None, end=None): 20 | if begin is None: 21 | begin = datetime(1970, 1, 1) 22 | if end is None: 23 | end = datetime.now() 24 | return begin <= datetimeTested <= end 25 | 26 | 27 | def toDatetime(date): 28 | return parse(date) 29 | 30 | 31 | def datetimeToString(datetime, formatstring): 32 | return datetime.strftime(formatstring) 33 | 34 | 35 | def checkDateConsistancy(begindate, enddate, lastdate): 36 | if begindate is not None and enddate is not None: 37 | if begindate > enddate: 38 | raise DateError('begindate ({}) cannot be after enddate ({})'.format(begindate, enddate)) 39 | 40 | if enddate is not None: 41 | if toDatetime(enddate) < lastdate: 42 | raise DateError('enddate ({}) cannot be before lastdate ({})'.format(enddate, lastdate)) 43 | 44 | if begindate is not None: 45 | if toDatetime(begindate) > datetime.now(): 46 | raise DateError('begindate ({}) cannot be after today ({})'.format(begindate, datetime.now().date())) 47 | 48 | 49 | def setBegindate(begindate, lastdate): 50 | return max(begindate, lastdate) 51 | 52 | 53 | def setEnddate(enddate): 54 | return min(enddate, datetime.now()) 55 | 56 | 57 | def getLastdate(last): 58 | return (datetime.now() - timedelta(days=int(last))).replace(hour=0, minute=0, second=0, microsecond=0) 59 | 60 | 61 | def getNDaysBefore(date, days): 62 | return (date - timedelta(days=days)).replace(hour=0, minute=0, second=0, microsecond=0) 63 | 64 | 65 | def getToday(): 66 | return (datetime.now()).replace(hour=0, minute=0, second=0, microsecond=0) 67 | 68 | 69 | def days_between(date_1, date_2): 70 | return abs((date_2 - date_1).days) 71 | -------------------------------------------------------------------------------- /pymisp/tools/neo4j.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import glob 4 | import os 5 | from .. import MISPEvent 6 | 7 | try: 8 | from py2neo import authenticate, Graph, Node, Relationship 9 | has_py2neo = True 10 | except ImportError: 11 | has_py2neo = False 12 | 13 | 14 | class Neo4j(): 15 | 16 | def __init__(self, host='localhost:7474', username='neo4j', password='neo4j'): 17 | if not has_py2neo: 18 | raise Exception('py2neo is required, please install: pip install py2neo') 19 | authenticate(host, username, password) 20 | self.graph = Graph("http://{}/db/data/".format(host)) 21 | 22 | def load_events_directory(self, directory): 23 | self.events = [] 24 | for path in glob.glob(os.path.join(directory, '*.json')): 25 | e = MISPEvent() 26 | e.load(path) 27 | self.import_event(e) 28 | 29 | def del_all(self): 30 | self.graph.delete_all() 31 | 32 | def import_event(self, event): 33 | tx = self.graph.begin() 34 | event_node = Node('Event', uuid=event.uuid, name=event.info) 35 | # event_node['distribution'] = event.distribution 36 | # event_node['threat_level_id'] = event.threat_level_id 37 | # event_node['analysis'] = event.analysis 38 | # event_node['published'] = event.published 39 | # event_node['date'] = event.date.isoformat() 40 | tx.create(event_node) 41 | for a in event.attributes: 42 | attr_node = Node('Attribute', a.type, uuid=a.uuid) 43 | attr_node['category'] = a.category 44 | attr_node['name'] = a.value 45 | # attr_node['to_ids'] = a.to_ids 46 | # attr_node['comment'] = a.comment 47 | # attr_node['distribution'] = a.distribution 48 | tx.create(attr_node) 49 | member_rel = Relationship(event_node, "is member", attr_node) 50 | tx.create(member_rel) 51 | val = Node('Value', name=a.value) 52 | ev = Relationship(event_node, "has", val) 53 | av = Relationship(attr_node, "is", val) 54 | s = val | ev | av 55 | tx.merge(s) 56 | # tx.graph.push(s) 57 | tx.commit() 58 | -------------------------------------------------------------------------------- /tests/misp_event.json: -------------------------------------------------------------------------------- 1 | { 2 | "Attribute": [ 3 | { 4 | "ShadowAttribute": [], 5 | "category": "Payload delivery", 6 | "comment": "", 7 | "deleted": false, 8 | "distribution": "5", 9 | "event_id": "2", 10 | "id": "7", 11 | "sharing_group_id": "0", 12 | "timestamp": "1465681304", 13 | "to_ids": false, 14 | "type": "url", 15 | "uuid": "575c8598-f1f0-4c16-a94a-0612c0a83866", 16 | "value": "http://fake.website.com/malware/is/here" 17 | }, 18 | { 19 | "ShadowAttribute": [], 20 | "category": "Payload type", 21 | "comment": "", 22 | "deleted": false, 23 | "distribution": "5", 24 | "event_id": "2", 25 | "id": "6", 26 | "sharing_group_id": "0", 27 | "timestamp": "1465681801", 28 | "to_ids": false, 29 | "type": "text", 30 | "uuid": "575c8549-9010-4555-8b37-057ac0a83866", 31 | "value": "Locky" 32 | } 33 | ], 34 | "Org": { 35 | "id": "1", 36 | "name": "ORGNAME", 37 | "uuid": "57586e9a-4a64-4f79-9009-4dc1c0a83866" 38 | }, 39 | "Orgc": { 40 | "id": "1", 41 | "name": "ORGNAME", 42 | "uuid": "57586e9a-4a64-4f79-9009-4dc1c0a83866" 43 | }, 44 | "RelatedEvent": [], 45 | "ShadowAttribute": [], 46 | "Tag": [ 47 | { 48 | "colour": "#005a5a", 49 | "exportable": true, 50 | "id": "6", 51 | "name": "ecsirt:malicious-code=\"ransomware\"" 52 | }, 53 | { 54 | "colour": "#142bf7", 55 | "exportable": true, 56 | "id": "1", 57 | "name": "for_intelmq_processing" 58 | } 59 | ], 60 | "analysis": "0", 61 | "attribute_count": "2", 62 | "date": "2016-06-09", 63 | "distribution": "0", 64 | "id": "2", 65 | "info": "A Random Event", 66 | "locked": false, 67 | "org_id": "1", 68 | "orgc_id": "1", 69 | "proposal_email_lock": false, 70 | "publish_timestamp": "0", 71 | "published": false, 72 | "sharing_group_id": "0", 73 | "threat_level_id": "1", 74 | "timestamp": "1465681801", 75 | "uuid": "5758ebf5-c898-48e6-9fe9-5665c0a83866" 76 | } 77 | -------------------------------------------------------------------------------- /examples/upload.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | import os 8 | import glob 9 | 10 | 11 | def init(url, key): 12 | return PyMISP(url, key, misp_verifycert, 'json') 13 | 14 | 15 | def upload_files(m, eid, paths, distrib, ids, categ, comment, info, analysis, threat): 16 | out = m.upload_samplelist(paths, eid, distrib, ids, categ, comment, info, analysis, threat) 17 | print(out) 18 | 19 | if __name__ == '__main__': 20 | parser = argparse.ArgumentParser(description='Send malware sample to MISP.') 21 | parser.add_argument("-u", "--upload", type=str, required=True, help="File or directory of files to upload.") 22 | parser.add_argument("-e", "--event", type=int, help="Not supplying an event ID will cause MISP to create a single new event for all of the POSTed malware samples.") 23 | parser.add_argument("-d", "--distrib", type=int, help="The distribution setting used for the attributes and for the newly created event, if relevant. [0-3].") 24 | parser.add_argument("-ids", action='store_true', help="You can flag all attributes created during the transaction to be marked as \"to_ids\" or not.") 25 | parser.add_argument("-c", "--categ", help="The category that will be assigned to the uploaded samples. Valid options are: Payload delivery, Artifacts dropped, Payload Installation, External Analysis.") 26 | parser.add_argument("-i", "--info", help="Used to populate the event info field if no event ID supplied.") 27 | parser.add_argument("-a", "--analysis", type=int, help="The analysis level of the newly created event, if applicatble. [0-2]") 28 | parser.add_argument("-t", "--threat", type=int, help="The threat level ID of the newly created event, if applicatble. [1-4]") 29 | parser.add_argument("-co", "--comment", type=str, help="Comment for the uploaded file(s).") 30 | args = parser.parse_args() 31 | 32 | misp = init(misp_url, misp_key) 33 | 34 | files = [] 35 | if os.path.isfile(args.upload): 36 | files = [args.upload] 37 | elif os.path.isdir(args.upload): 38 | files = [f for f in glob.iglob(os.path.join(args.upload + '*'))] 39 | else: 40 | print('invalid file') 41 | exit(0) 42 | 43 | upload_files(misp, args.event, files, args.distrib, args.ids, args.categ, args.comment, args.info, args.analysis, args.threat) 44 | -------------------------------------------------------------------------------- /examples/generate_file_objects.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | import json 6 | 7 | try: 8 | from pymisp import MISPEncode 9 | from pymisp.tools import make_binary_objects 10 | except ImportError: 11 | pass 12 | 13 | 14 | def check(): 15 | missing_dependencies = {'pydeep': False, 'lief': False, 'magic': False, 'pymisp': False} 16 | try: 17 | import pymisp # noqa 18 | except ImportError: 19 | missing_dependencies['pymisp'] = 'Please install pydeep: pip install pymisp' 20 | try: 21 | import pydeep # noqa 22 | except ImportError: 23 | missing_dependencies['pydeep'] = 'Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git' 24 | try: 25 | import lief # noqa 26 | except ImportError: 27 | missing_dependencies['lief'] = 'Please install lief, documentation here: https://github.com/lief-project/LIEF' 28 | try: 29 | import magic # noqa 30 | except ImportError: 31 | missing_dependencies['magic'] = 'Please install python-magic: pip install python-magic.' 32 | return json.dumps(missing_dependencies) 33 | 34 | 35 | def make_objects(path): 36 | to_return = {'objects': [], 'references': []} 37 | fo, peo, seos = make_binary_objects(path) 38 | 39 | if seos: 40 | for s in seos: 41 | to_return['objects'].append(s) 42 | if s.ObjectReference: 43 | to_return['references'] += s.ObjectReference 44 | 45 | if peo: 46 | to_return['objects'].append(peo) 47 | if peo.ObjectReference: 48 | to_return['references'] += peo.ObjectReference 49 | 50 | if fo: 51 | to_return['objects'].append(fo) 52 | if fo.ObjectReference: 53 | to_return['references'] += fo.ObjectReference 54 | return json.dumps(to_return, cls=MISPEncode) 55 | 56 | if __name__ == '__main__': 57 | parser = argparse.ArgumentParser(description='Extract indicators out of binaries and returns MISP objects.') 58 | group = parser.add_mutually_exclusive_group() 59 | group.add_argument("-p", "--path", help="Path to process.") 60 | group.add_argument("-c", "--check", action='store_true', help="Check the dependencies.") 61 | args = parser.parse_args() 62 | 63 | if args.check: 64 | print(check()) 65 | if args.path: 66 | obj = make_objects(args.path) 67 | print(obj) 68 | -------------------------------------------------------------------------------- /examples/situational-awareness/pygal_tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import pygal 5 | from pygal.style import Style 6 | import pandas 7 | import random 8 | 9 | 10 | def createTable(colors, categ_types_hash, tablename='attribute_table.html'): 11 | with open(tablename, 'w') as target: 12 | target.write('\n\n\n\n\n') 13 | for categ_name, types in categ_types_hash.items(): 14 | table = pygal.Treemap(pretty_print=True) 15 | target.write('\n

{}

\n'.format(colors[categ_name], categ_name)) 16 | for d in types: 17 | table.add(d['label'], d['value']) 18 | target.write(table.render_table(transpose=True)) 19 | target.write('\n\n') 20 | 21 | 22 | def createTreemap(data, title, treename='attribute_treemap.svg', tablename='attribute_table.html'): 23 | labels_categ = data.index.labels[0] 24 | labels_types = data.index.labels[1] 25 | names_categ = data.index.levels[0] 26 | names_types = data.index.levels[1] 27 | categ_types_hash = {} 28 | for categ_id, type_val, total in zip(labels_categ, labels_types, data): 29 | if not categ_types_hash.get(names_categ[categ_id]): 30 | categ_types_hash[names_categ[categ_id]] = [] 31 | dict_to_print = {'label': names_types[type_val], 'value': total} 32 | categ_types_hash[names_categ[categ_id]].append(dict_to_print) 33 | 34 | colors = {categ: "#%06X" % random.randint(0, 0xFFFFFF) for categ in categ_types_hash.keys()} 35 | style = Style(background='transparent', 36 | plot_background='#FFFFFF', 37 | foreground='#111111', 38 | foreground_strong='#111111', 39 | foreground_subtle='#111111', 40 | opacity='.6', 41 | opacity_hover='.9', 42 | transition='400ms ease-in', 43 | colors=tuple(colors.values())) 44 | 45 | treemap = pygal.Treemap(pretty_print=True, legend_at_bottom=True, style=style) 46 | treemap.title = title 47 | treemap.print_values = True 48 | treemap.print_labels = True 49 | 50 | for categ_name, types in categ_types_hash.items(): 51 | treemap.add(categ_name, types) 52 | 53 | createTable(colors, categ_types_hash) 54 | treemap.render_to_file(treename) 55 | -------------------------------------------------------------------------------- /examples/misp2cef.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Export IOC's from MISP in CEF format 5 | # Based on cef_export.py MISP module by Hannah Ward 6 | 7 | import sys 8 | import datetime 9 | from pymisp import PyMISP, MISPAttribute 10 | from keys import misp_url, misp_key 11 | 12 | cefconfig = {"Default_Severity":1, "Device_Vendor":"MISP", "Device_Product":"MISP", "Device_Version":1} 13 | 14 | cefmapping = {"ip-src":"src", "ip-dst":"dst", "hostname":"dhost", "domain":"destinationDnsDomain", 15 | "md5":"fileHash", "sha1":"fileHash", "sha256":"fileHash", 16 | "filename|md5":"fileHash", "filename|sha1":"fileHash", "filename|sha256":"fileHash", 17 | "url":"request"} 18 | 19 | mispattributes = {'input':list(cefmapping.keys())} 20 | 21 | 22 | def make_cef(event): 23 | for attr in event["Attribute"]: 24 | if attr["to_ids"] and attr["type"] in cefmapping: 25 | if '|' in attr["type"] and '|' in attr["value"]: 26 | value = attr["value"].split('|')[1] 27 | else: 28 | value = attr["value"] 29 | response = "{} host CEF:0|{}|{}|{}|{}|{}|{}|msg={} customerURI={} externalId={} {}={}".format( 30 | datetime.datetime.fromtimestamp(int(attr["timestamp"])).strftime("%b %d %H:%M:%S"), 31 | cefconfig["Device_Vendor"], 32 | cefconfig["Device_Product"], 33 | cefconfig["Device_Version"], 34 | attr["category"], 35 | attr["category"], 36 | cefconfig["Default_Severity"], 37 | event["info"].replace("\\","\\\\").replace("=","\\=").replace('\n','\\n') + "(MISP Event #" + event["id"] + ")", 38 | misp_url + 'events/view/' + event["id"], 39 | attr["uuid"], 40 | cefmapping[attr["type"]], 41 | value, 42 | ) 43 | print(str(bytes(response, 'utf-8'), 'utf-8')) 44 | 45 | 46 | def init_misp(): 47 | global mymisp 48 | mymisp = PyMISP(misp_url, misp_key) 49 | 50 | 51 | def echeck(r): 52 | if r.get('errors'): 53 | if r.get('message') == 'No matches.': 54 | return 55 | else: 56 | print(r['errors']) 57 | sys.exit(1) 58 | 59 | 60 | def find_events(): 61 | r = mymisp.search(controller='events', published=True, to_ids=True) 62 | echeck(r) 63 | if not r.get('response'): 64 | return 65 | for ev in r['response']: 66 | make_cef(ev['Event']) 67 | 68 | 69 | if __name__ == '__main__': 70 | init_misp() 71 | find_events() 72 | -------------------------------------------------------------------------------- /pymisp/tools/abstractgenerator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import abc 5 | import six 6 | from .. import MISPObject 7 | from ..exceptions import InvalidMISPObject 8 | from datetime import datetime, date 9 | from dateutil.parser import parse 10 | 11 | 12 | @six.add_metaclass(abc.ABCMeta) # Remove that line when discarding python2 support. 13 | # Python3 way: class MISPObjectGenerator(metaclass=abc.ABCMeta): 14 | class AbstractMISPObjectGenerator(MISPObject): 15 | 16 | def _detect_epoch(self, timestamp): 17 | try: 18 | tmp = float(timestamp) 19 | if tmp < 30000000: 20 | # Assuming the user doesn't want to report anything before datetime(1970, 12, 14, 6, 20) 21 | # The date is most probably in the format 20180301 22 | return False 23 | return True 24 | except ValueError: 25 | return False 26 | 27 | def _sanitize_timestamp(self, timestamp): 28 | if not timestamp: 29 | return datetime.now() 30 | 31 | if isinstance(timestamp, datetime): 32 | return timestamp 33 | elif isinstance(timestamp, date): 34 | return datetime.combine(timestamp, datetime.min.time()) 35 | elif isinstance(timestamp, dict): 36 | if not isinstance(timestamp['value'], datetime): 37 | timestamp['value'] = parse(timestamp['value']) 38 | return timestamp 39 | elif not isinstance(timestamp, datetime): # Supported: float/int/string 40 | if self._detect_epoch(timestamp): 41 | return datetime.fromtimestamp(float(timestamp)) 42 | return parse(timestamp) 43 | return timestamp 44 | 45 | def generate_attributes(self): 46 | """Contains the logic where all the values of the object are gathered""" 47 | if hasattr(self, '_parameters'): 48 | for object_relation in self._definition['attributes']: 49 | value = self._parameters.pop(object_relation, None) 50 | if not value: 51 | continue 52 | if isinstance(value, dict): 53 | self.add_attribute(object_relation, **value) 54 | else: 55 | # Assume it is the value only 56 | self.add_attribute(object_relation, value=value) 57 | if self._strict and self._known_template and self._parameters: 58 | raise InvalidMISPObject('Some object relations are unknown in the template and could not be attached: {}'.format(', '.join(self._parameters))) 59 | -------------------------------------------------------------------------------- /examples/feed-generator-from-redis/README.md: -------------------------------------------------------------------------------- 1 | # Generic MISP feed generator 2 | ## Description 3 | 4 | - ``generator.py`` exposes a class allowing to generate a MISP feed in real time, where each items can be added on daily generated events. 5 | - ``fromredis.py`` uses ``generator.py`` to generate a MISP feed based on data stored in redis. 6 | - ``server.py`` is a simple script using *Flask_autoindex* to serve data to MISP. 7 | - ``MISPItemToRedis.py`` permits to push (in redis) items to be added in MISP by the ``fromredis.py`` script. 8 | 9 | 10 | ## Installation 11 | 12 | ```` 13 | # Feed generator 14 | git clone https://github.com/CIRCL/PyMISP 15 | cd examples/feed-generator-from-redis 16 | cp settings.default.py settings.py 17 | vi settings.py # adjust your settings 18 | 19 | python3 fromredis.py 20 | 21 | # Serving file to MISP 22 | bash install.sh 23 | . ./serv-env/bin/activate 24 | python3 server.py 25 | ```` 26 | 27 | 28 | ## Usage 29 | 30 | ``` 31 | # Activate virtualenv 32 | . ./serv-env/bin/activate 33 | ``` 34 | 35 | ### Adding items to MISP 36 | 37 | ``` 38 | # create helper object 39 | >>> helper = MISPItemToRedis("redis_list_keyname") 40 | 41 | # push an attribute to redis 42 | >>> helper.push_attribute("ip-src", "8.8.8.8", category="Network activity") 43 | 44 | # push an object to redis 45 | >>> helper.push_object({ "name": "cowrie", "session": "session_id", "username": "admin", "password": "admin", "protocol": "telnet" }) 46 | 47 | # push a sighting to redis 48 | >>> helper.push_sighting(uuid="5a9e9e26-fe40-4726-8563-5585950d210f") 49 | ``` 50 | 51 | ### Generate the feed 52 | 53 | ``` 54 | # Create the FeedGenerator object using the configuration provided in the file settings.py 55 | # It will create daily event in which attributes and object will be added 56 | >>> generator = FeedGenerator() 57 | 58 | # Add an attribute to the daily event 59 | >>> attr_type = "ip-src" 60 | >>> attr_value = "8.8.8.8" 61 | >>> additional_data = {} 62 | >>> generator.add_attribute_to_event(attr_type, attr_value, **additional_data) 63 | 64 | # Add a cowrie object to the daily event 65 | >>> obj_name = "cowrie" 66 | >>> obj_data = { "session": "session_id", "username": "admin", "password": "admin", "protocol": "telnet" } 67 | >>> generator.add_object_to_event(obj_name, **obj_data) 68 | 69 | # Immediatly write the event to the disk (Bypassing the default flushing behavior) 70 | >>> generator.flush_event() 71 | ``` 72 | 73 | ### Consume stored data in redis 74 | 75 | ``` 76 | # Configuration provided in the file settings.py 77 | >>> python3 fromredis.py 78 | ``` 79 | 80 | ### Serve data to MISP 81 | 82 | ``` 83 | >>> python3 server.py 84 | ``` 85 | -------------------------------------------------------------------------------- /examples/situational-awareness/tag_scatter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | import numpy 8 | import tools 9 | import date_tools 10 | import bokeh_tools 11 | 12 | import time 13 | 14 | if __name__ == '__main__': 15 | parser = argparse.ArgumentParser(description='Show the evolution of trend of tags.') 16 | parser.add_argument("-d", "--days", type=int, required=True, help='') 17 | parser.add_argument("-s", "--begindate", required=True, help='format yyyy-mm-dd') 18 | parser.add_argument("-e", "--enddate", required=True, help='format yyyy-mm-dd') 19 | 20 | args = parser.parse_args() 21 | 22 | misp = PyMISP(misp_url, misp_key, misp_verifycert) 23 | 24 | result = misp.search(date_from=args.begindate, date_to=args.enddate, metadata=False) 25 | 26 | # Getting data 27 | 28 | if 'response' in result: 29 | events = tools.eventsListBuildFromArray(result) 30 | NbTags = [] 31 | dates = [] 32 | enddate = date_tools.toDatetime(args.enddate) 33 | begindate = date_tools.toDatetime(args.begindate) 34 | 35 | for i in range(round(date_tools.days_between(enddate, begindate)/args.days)): 36 | begindate = date_tools.getNDaysBefore(enddate, args.days) 37 | eventstemp = tools.selectInRange(events, begindate, enddate) 38 | if eventstemp is not None: 39 | for event in eventstemp.iterrows(): 40 | if 'Tag' in event[1]: 41 | dates.append(enddate) 42 | if isinstance(event[1]['Tag'], list): 43 | NbTags.append(len(event[1]['Tag'])) 44 | else: 45 | NbTags.append(0) 46 | enddate = begindate 47 | 48 | # Prepare plot 49 | 50 | NbTagsPlot = {} 51 | datesPlot = {} 52 | 53 | for i in range(len(NbTags)): 54 | if NbTags[i] == -1: 55 | continue 56 | count = 1 57 | for j in range(i+1, len(NbTags)): 58 | if NbTags[i] == NbTags[j] and dates[i] == dates[j]: 59 | count = count + 1 60 | NbTags[j] = -1 61 | if str(count) in NbTagsPlot: 62 | NbTagsPlot[str(count)].append(NbTags[i]) 63 | datesPlot[str(count)].append(dates[i]) 64 | else: 65 | NbTagsPlot[str(count)] = [NbTags[i]] 66 | datesPlot[str(count)] = [dates[i]] 67 | NbTags[i] = -1 68 | 69 | # Plot 70 | 71 | bokeh_tools.tagsDistributionScatterPlot(NbTagsPlot, datesPlot) 72 | -------------------------------------------------------------------------------- /examples/situational-awareness/tags_count.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | from datetime import datetime 7 | import argparse 8 | import tools 9 | import date_tools 10 | 11 | 12 | def init(url, key): 13 | return PyMISP(url, key, misp_verifycert, 'json') 14 | 15 | # ######### fetch data ########## 16 | 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the repartition of tags in this sample.') 20 | parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7") 21 | parser.add_argument("-b", "--begindate", default='1970-01-01', help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") 22 | parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") 23 | 24 | args = parser.parse_args() 25 | 26 | misp = init(misp_url, misp_key) 27 | 28 | if args.days is None: 29 | args.days = 7 30 | result = misp.search(last='{}d'.format(args.days), metadata=True) 31 | 32 | date_tools.checkDateConsistancy(args.begindate, args.enddate, date_tools.getLastdate(args.days)) 33 | 34 | if args.begindate is None: 35 | args.begindate = date_tools.getLastdate(args.days) 36 | else: 37 | args.begindate = date_tools.setBegindate(date_tools.toDatetime(args.begindate), date_tools.getLastdate(args.days)) 38 | 39 | if args.enddate is None: 40 | args.enddate = datetime.now() 41 | else: 42 | args.enddate = date_tools.setEnddate(date_tools.toDatetime(args.enddate)) 43 | 44 | if 'response' in result: 45 | events = tools.selectInRange(tools.eventsListBuildFromArray(result), begin=args.begindate, end=args.enddate) 46 | tags = tools.tagsListBuild(events) 47 | result = tools.getNbOccurenceTags(tags) 48 | else: 49 | result = 'There is no event during the studied period' 50 | 51 | text = 'Studied pediod: from ' 52 | if args.begindate is None: 53 | text = text + '1970-01-01' 54 | else: 55 | text = text + str(args.begindate.date()) 56 | text = text + ' to ' 57 | if args.enddate is None: 58 | text = text + str(datetime.now().date()) 59 | else: 60 | text = text + str(args.enddate.date()) 61 | 62 | print('\n========================================================') 63 | print(text) 64 | print(result) 65 | -------------------------------------------------------------------------------- /tests/sharing_groups.json: -------------------------------------------------------------------------------- 1 | { 2 | "response": [ 3 | { 4 | "SharingGroup": { 5 | "id": "1", 6 | "name": "PrivateTrustedGroup", 7 | "description": "", 8 | "releasability": "", 9 | "local": true, 10 | "active": true 11 | }, 12 | "Organisation": { 13 | "id": "1", 14 | "name": "CIRCL", 15 | "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" 16 | }, 17 | "SharingGroupOrg": [ 18 | { 19 | "id": "1", 20 | "sharing_group_id": "1", 21 | "org_id": "1", 22 | "extend": true, 23 | "Organisation": { 24 | "name": "CIRCL", 25 | "id": "1", 26 | "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" 27 | } 28 | }, 29 | { 30 | "id": "2", 31 | "sharing_group_id": "1", 32 | "org_id": "2", 33 | "extend": false, 34 | "Organisation": { 35 | "name": "PifPafPoum", 36 | "id": "2", 37 | "uuid": "56bf12a7-c19c-4b98-83e7-d9bb02de0b81" 38 | } 39 | } 40 | ], 41 | "SharingGroupServer": [ 42 | { 43 | "all_orgs": false, 44 | "server_id": "0", 45 | "sharing_group_id": "1", 46 | "Server": [] 47 | } 48 | ], 49 | "editable": true 50 | }, 51 | { 52 | "SharingGroup": { 53 | "id": "2", 54 | "name": "test", 55 | "description": "", 56 | "releasability": "", 57 | "local": true, 58 | "active": true 59 | }, 60 | "Organisation": { 61 | "id": "1", 62 | "name": "CIRCL", 63 | "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" 64 | }, 65 | "SharingGroupOrg": [ 66 | { 67 | "id": "3", 68 | "sharing_group_id": "2", 69 | "org_id": "1", 70 | "extend": true, 71 | "Organisation": { 72 | "name": "CIRCL", 73 | "id": "1", 74 | "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" 75 | } 76 | }, 77 | { 78 | "id": "4", 79 | "sharing_group_id": "2", 80 | "org_id": "2", 81 | "extend": false, 82 | "Organisation": { 83 | "name": "PifPafPoum", 84 | "id": "2", 85 | "uuid": "56bf12a7-c19c-4b98-83e7-d9bb02de0b81" 86 | } 87 | } 88 | ], 89 | "SharingGroupServer": [ 90 | { 91 | "all_orgs": false, 92 | "server_id": "0", 93 | "sharing_group_id": "2", 94 | "Server": [] 95 | } 96 | ], 97 | "editable": true 98 | } 99 | ] 100 | } 101 | -------------------------------------------------------------------------------- /examples/situational-awareness/README.md: -------------------------------------------------------------------------------- 1 | ## Explanation 2 | 3 | * treemap.py is a script that will generate an interactive svg (attribute\_treemap.svg) containing a treepmap representing the distribution of attributes in a sample (data) fetched from the instance using "last" or "searchall" examples. 4 | * It will also generate a html document with a table (attribute\_table.html) containing count for each type of attribute. 5 | * test\_attribute\_treemap.html is a quick page made to visualize both treemap and table at the same time. 6 | 7 | * tags\_count.py is a script that count the number of occurences of every tags in a fetched sample of Events in a given period of time. 8 | * tag\_search.py is a script that count the number of occurences of a given tag in a fetched sample of Events in a given period of time. 9 | * Events will be fetched from _days_ days ago to today. 10 | * _begindate_ is the beginning of the studied period. If it is later than today, an error will be raised. 11 | * _enddate_ is the end of the studied period. If it is earlier than _begindate_, an error will be raised. 12 | * tag\_search.py allows research for multiple tags is possible by separating each tag by the | symbol. 13 | * Partial research is also possible with tag\_search.py. For instance, search for "ransom" will also return tags containin "ransomware". 14 | 15 | * tags\_to\_graphs.py is a script that will generate several plots to visualise tags distribution. 16 | * The studied _period_ can be either the 7, 28 or 360 last days 17 | * _accuracy_ allows to get smallers splits of data instead of the default values 18 | * _order_ define the accuracy of the curve fitting. Default value is 3 19 | * It will generate two plots comparing all the tags: 20 | * tags_repartition_plot that present the raw data 21 | * tags_repartition_trend_plot that present the general evolution for each tag 22 | * Then each taxonomies will be represented in three plots: 23 | * Raw datas: in "plot" folder, named with the name of the corresponding taxonomy 24 | * Trend: in "plot" folder, named _taxonomy_\_trend. general evolution of the data (linear fitting, curve fitting at order 1) 25 | * Curve fitting: in "plotlib" folder, name as the taxonomy it presents. 26 | * In order to visualize the last plots, a html file is also generated automaticaly (might be improved in the future) 27 | 28 | :warning: These scripts are not time optimised 29 | 30 | ## Requierements 31 | 32 | * [Pygal](https://github.com/Kozea/pygal/) 33 | * [Matplotlib](https://github.com/matplotlib/matplotlib) 34 | * [Pandas](https://github.com/pandas-dev/pandas) 35 | * [SciPy](https://github.com/scipy/scipy) 36 | * [PyTaxonomies](https://github.com/MISP/PyTaxonomies) 37 | * [Python3-tk](https://github.com/python-git/python/blob/master/Lib/lib-tk/Tkinter.py) 38 | 39 | -------------------------------------------------------------------------------- /examples/copy_list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | 6 | from pymisp import PyMISP 7 | 8 | from keys import cert, priv 9 | 10 | url_cert = 'https://misp.circl.lu' 11 | url_priv = 'https://misppriv.circl.lu' 12 | cert_cert = 'misp.circl.lu.crt' 13 | cert_priv = 'misppriv.circl.lu.crt' 14 | source = None 15 | destination = None 16 | 17 | 18 | def init(cert_to_priv=True): 19 | global source 20 | global destination 21 | print(cert_to_priv) 22 | if cert_to_priv: 23 | source = PyMISP(url_cert, cert, cert_cert, 'xml') 24 | destination = PyMISP(url_priv, priv, cert_priv, 'xml') 25 | else: 26 | source = PyMISP(url_priv, priv, cert_priv, 'xml') 27 | destination = PyMISP(url_cert, cert, cert_cert, 'xml') 28 | 29 | 30 | def copy_event(event_id): 31 | e = source.get_event(event_id) 32 | return destination.add_event(e) 33 | 34 | 35 | def update_event(event_id, event_to_update): 36 | e = source.get_event(event_id) 37 | return destination.update_event(event_to_update, e) 38 | 39 | 40 | def list_copy(filename): 41 | with open(filename, 'r') as f: 42 | for l in f: 43 | copy(l) 44 | 45 | 46 | def loop_copy(): 47 | while True: 48 | line = sys.stdin.readline() 49 | copy(line) 50 | 51 | 52 | def copy(eventid): 53 | eventid = eventid.strip() 54 | if len(eventid) == 0 or not eventid.isdigit(): 55 | print('empty line or NaN.') 56 | return 57 | eventid = int(eventid) 58 | print(eventid, 'copying...') 59 | r = copy_event(eventid) 60 | if r.status_code >= 400: 61 | loc = r.headers['location'] 62 | if loc is not None: 63 | event_to_update = loc.split('/')[-1] 64 | print('updating', event_to_update) 65 | r = update_event(eventid, event_to_update) 66 | if r.status_code >= 400: 67 | print(r.status_code, r.headers) 68 | else: 69 | print(r.status_code, r.headers) 70 | print(eventid, 'done.') 71 | 72 | 73 | def export_our_org(): 74 | circl = source.search(org='CIRCL') 75 | return circl 76 | 77 | if __name__ == '__main__': 78 | import argparse 79 | parser = argparse.ArgumentParser( 80 | description='Copy the events from one MISP instance to an other.') 81 | parser.add_argument('-f', '--filename', type=str, 82 | help='File containing a list of event id.') 83 | parser.add_argument( 84 | '-l', '--loop', action='store_true', 85 | help='Endless loop: eventid in the terminal and it will be copied.') 86 | parser.add_argument('--priv_to_cert', action='store_false', default=True, 87 | help='Copy from MISP priv to MISP CERT.') 88 | args = parser.parse_args() 89 | init(args.priv_to_cert) 90 | if args.filename is not None: 91 | list_copy(args.filename) 92 | else: 93 | loop_copy() 94 | -------------------------------------------------------------------------------- /examples/events/README.md: -------------------------------------------------------------------------------- 1 | ## Explanation 2 | 3 | This folder contains scripts made to create dummy events in order to test MISP instances. 4 | 5 | * dummy is a containing text only file used as uploaded attachement. 6 | * create\_dummy\_event.py will create a given number of events (default: 1)with a randomly generated domain|ip attribute as well as a copy of dummy file. 7 | * create\_massive\_dummy\_events.py will create a given number of events (default: 1) with a given number of randomly generated attributes(default: 3000). 8 | 9 | ### Tools description 10 | 11 | * randomStringGenerator: generate a random string of a given size, characters used to build the string can be chosen, default are characters from string.ascii\_lowercase and string.digits 12 | * randomIpGenerator: generate a random ip 13 | 14 | * floodtxt: add a generated string as attribute of the given event. The added attributes can be of the following category/type: 15 | - Internal reference/comment 16 | - Internal reference/text 17 | - Internal reference/other 18 | - Payload delivery/email-subject 19 | - Artifact dropped/mutex 20 | - Artifact dropped/filename 21 | * floodip: add a generated ip as attribute of the given event. The added attributes can be of the following category/type: 22 | - Network activity/ip-src 23 | - Network activity/ip.dst 24 | * flooddomain: add a generated domain-like string as attribute of the given event. The added attributes can be of the following category/type: 25 | - Network activity/hostname 26 | - Network activity/domain 27 | * flooddomainip: add a generated domain|ip-like string as attribute of the given event. The added attribute is of the following category/type: 28 | - Network activity/domain|ip 29 | * floodemail: add a generated email-like string as attribute of the given event. The added attributes can be of the following category/type: 30 | - Payload delivery/email-src 31 | - Payload delivery/email-dst 32 | * floodattachmentent: add a dummy file as attribute of the given event. The added attribute is of the following category/type: 33 | - Payload delivery/attachment 34 | 35 | * create\_dummy\_event: create a dummy event named "dummy event" with these caracteristics: 36 | - Distribution: Your organisation only 37 | - Analysis: Initial 38 | - Threat Level: Undefined 39 | - Number of Attributes: 2 40 | - Attribute: 41 | - category/type: Network activity/domain|ip 42 | - value: Randomly generated 43 | - Attribute: 44 | -category/type: Payload delivery/attachment 45 | - value: 'dummy' file 46 | * create\_massive\_dummy\_events: create a dummy event named "massive dummy event" with these caracteristics: 47 | - Distribution: Your organisation only 48 | - Analysis: Initial 49 | - Threat Level: Undefined 50 | - Number of Attributes: Given as argument 51 | - Attribute: 52 | - category/type: Randomly chosen 53 | - value: Randomly generated or dummy file 54 | -------------------------------------------------------------------------------- /tests/57c4445b-c548-4654-af0b-4be3950d210f.json: -------------------------------------------------------------------------------- 1 | {"Event": {"info": "Ransomware - Xorist", "publish_timestamp": "1472548231", "timestamp": "1472541011", "analysis": "2", "Attribute": [{"category": "External analysis", "comment": "Imported via the Freetext Import Tool - Xchecked via VT: b3c4ae251f8094fa15b510051835c657eaef2a6cea46075d3aec964b14a99f68", "uuid": "57c5300c-0560-4146-bfaa-40e802de0b81", "timestamp": "1472540684", "to_ids": false, "value": "https://www.virustotal.com/file/b3c4ae251f8094fa15b510051835c657eaef2a6cea46075d3aec964b14a99f68/analysis/1469554268/", "type": "link"}, {"category": "External analysis", "comment": "", "uuid": "57c5310b-dc34-43cb-8b8e-4846950d210f", "timestamp": "1472541011", "to_ids": false, "value": "http://www.xylibox.com/2011/06/have-fun-with-trojan-ransomwin32xorist.html", "type": "link"}, {"category": "Other", "comment": "", "uuid": "57c444c0-8004-48fa-9c33-8aca950d210f", "timestamp": "1472480448", "to_ids": false, "value": "UPX packed", "type": "comment"}, {"category": "Other", "comment": "", "uuid": "57c44648-96f4-45d4-a8eb-453e950d210f", "timestamp": "1472480840", "to_ids": false, "value": "Key: 85350044dF4AC3518D185678A9414A7F,\r\nEncryption rounds:8,\r\nStart offset: 64,\r\nAlgorithm: TEA", "type": "text"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448a-fb04-457d-87e7-4127950d210f", "timestamp": "1472480394", "to_ids": true, "value": "3Z4wnG9603it23y.exe", "type": "filename"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448b-454c-4d17-90d1-4d2f950d210f", "timestamp": "1472480395", "to_ids": true, "value": "0749bae92ca336a02c83d126e04ec628", "type": "md5"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448a-bef0-4ba7-a071-444e950d210f", "timestamp": "1472480394", "to_ids": true, "value": "77b0c41b7d340b8a3d903f21347bbf06aa766b5b", "type": "sha1"}, {"category": "Payload delivery", "comment": "Imported via the Freetext Import Tool", "uuid": "57c4448b-3fa4-4d65-9ccc-4afa950d210f", "timestamp": "1472480395", "to_ids": true, "value": "b3c4ae251f8094fa15b510051835c657eaef2a6cea46075d3aec964b14a99f68", "type": "sha256"}, {"category": "Persistence mechanism", "comment": "", "uuid": "57c54b0f-27a4-458b-8e63-4455950d210f", "timestamp": "1472547599", "to_ids": true, "value": "Software\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Run|%TEMP%\\3Z4wnG9603it23y.exe", "type": "regkey|value"}], "Tag": [{"colour": "#ffffff", "exportable": true, "name": "tlp:white"}, {"colour": "#3d7a00", "exportable": true, "name": "circl:incident-classification=\"malware\""}, {"colour": "#420053", "exportable": true, "name": "ms-caro-malware:malware-type=\"Ransom\""}, {"colour": "#2c4f00", "exportable": true, "name": "malware_classification:malware-category=\"Ransomware\""}], "published": true, "date": "2016-08-29", "Orgc": {"name": "CIRCL", "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f"}, "threat_level_id": "3", "uuid": "57c4445b-c548-4654-af0b-4be3950d210f"}} -------------------------------------------------------------------------------- /pymisp/tools/emailobject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from ..exceptions import InvalidMISPObject 5 | from .abstractgenerator import AbstractMISPObjectGenerator 6 | from io import BytesIO 7 | import logging 8 | from email import message_from_bytes, policy 9 | 10 | logger = logging.getLogger('pymisp') 11 | 12 | 13 | class EMailObject(AbstractMISPObjectGenerator): 14 | 15 | def __init__(self, filepath=None, pseudofile=None, attach_original_email=True, standalone=True, **kwargs): 16 | if filepath: 17 | with open(filepath, 'rb') as f: 18 | self.__pseudofile = BytesIO(f.read()) 19 | elif pseudofile and isinstance(pseudofile, BytesIO): 20 | self.__pseudofile = pseudofile 21 | else: 22 | raise InvalidMISPObject('File buffer (BytesIO) or a path is required.') 23 | # PY3 way: 24 | # super().__init__('file') 25 | super(EMailObject, self).__init__('email', standalone=standalone, **kwargs) 26 | self.__email = message_from_bytes(self.__pseudofile.getvalue(), policy=policy.default) 27 | if attach_original_email: 28 | self.add_attribute('eml', value='Full email.eml', data=self.__pseudofile) 29 | self.generate_attributes() 30 | 31 | @property 32 | def email(self): 33 | return self.__email 34 | 35 | @property 36 | def attachments(self): 37 | to_return = [] 38 | for attachment in self.__email.iter_attachments(): 39 | to_return.append((attachment.get_filename(), BytesIO(attachment.get_content()))) 40 | return to_return 41 | 42 | def generate_attributes(self): 43 | if self.__email.get_body(preferencelist=('html', 'plain')): 44 | self.add_attribute('email-body', value=self.__email.get_body(preferencelist=('html', 'plain')).get_payload(decode=True).decode('utf8', 'surrogateescape')) 45 | if 'Reply-To' in self.__email: 46 | self.add_attribute('reply-to', value=self.__email['Reply-To']) 47 | if 'Message-ID' in self.__email: 48 | self.add_attribute('message-id', value=self.__email['Message-ID']) 49 | if 'To' in self.__email: 50 | for to in self.__email['To'].split(','): 51 | self.add_attribute('to', value=to.strip()) 52 | if 'Cc' in self.__email: 53 | for cc in self.__email['Cc'].split(','): 54 | self.add_attribute('cc', value=cc.strip()) 55 | if 'Subject' in self.__email: 56 | self.add_attribute('subject', value=self.__email['Subject']) 57 | if 'From' in self.__email: 58 | for e_from in self.__email['From'].split(','): 59 | self.add_attribute('from', value=e_from.strip()) 60 | if 'Return-Path' in self.__email: 61 | self.add_attribute('return-path', value=self.__email['Return-Path']) 62 | if 'User-Agent' in self.__email: 63 | self.add_attribute('user-agent', value=self.__email['User-Agent']) 64 | -------------------------------------------------------------------------------- /examples/events/tools.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import random 5 | from random import randint 6 | import string 7 | 8 | 9 | def randomStringGenerator(size, chars=string.ascii_lowercase + string.digits): 10 | return ''.join(random.choice(chars) for _ in range(size)) 11 | 12 | 13 | def randomIpGenerator(): 14 | return str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255)) 15 | 16 | 17 | def floodtxt(misp, event, maxlength=255): 18 | text = randomStringGenerator(randint(1, maxlength)) 19 | textfunctions = [misp.add_internal_comment, misp.add_internal_text, misp.add_internal_other, misp.add_email_subject, misp.add_mutex, misp.add_filename] 20 | textfunctions[randint(0, 5)](event, text) 21 | 22 | 23 | def floodip(misp, event): 24 | ip = randomIpGenerator() 25 | ipfunctions = [misp.add_ipsrc, misp.add_ipdst] 26 | ipfunctions[randint(0, 1)](event, ip) 27 | 28 | 29 | def flooddomain(misp, event, maxlength=25): 30 | a = randomStringGenerator(randint(1, maxlength)) 31 | b = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase) 32 | domain = a + '.' + b 33 | domainfunctions = [misp.add_hostname, misp.add_domain] 34 | domainfunctions[randint(0, 1)](event, domain) 35 | 36 | 37 | def flooddomainip(misp, event, maxlength=25): 38 | a = randomStringGenerator(randint(1, maxlength)) 39 | b = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase) 40 | domain = a + '.' + b 41 | ip = randomIpGenerator() 42 | misp.add_domain_ip(event, domain, ip) 43 | 44 | 45 | def floodemail(misp, event, maxlength=25): 46 | a = randomStringGenerator(randint(1, maxlength)) 47 | b = randomStringGenerator(randint(1, maxlength)) 48 | c = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase) 49 | email = a + '@' + b + '.' + c 50 | emailfunctions = [misp.add_email_src, misp.add_email_dst] 51 | emailfunctions[randint(0, 1)](event, email) 52 | 53 | 54 | def floodattachment(misp, eventid, distribution, to_ids, category, comment, info, analysis, threat_level_id): 55 | filename = randomStringGenerator(randint(1, 128)) 56 | misp.upload_sample(filename, 'dummy', eventid, distribution, to_ids, category, comment, info, analysis, threat_level_id) 57 | 58 | 59 | def create_dummy_event(misp): 60 | event = misp.new_event(0, 4, 0, 'dummy event') 61 | flooddomainip(misp, event) 62 | floodattachment(misp, event['Event']['id'], event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) 63 | 64 | 65 | def create_massive_dummy_events(misp, nbattribute): 66 | event = misp.new_event(0, 4, 0, 'massive dummy event') 67 | eventid = event['Event']['id'] 68 | functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment] 69 | for i in range(nbattribute): 70 | choice = randint(0, 5) 71 | if choice == 5: 72 | floodattachment(misp, eventid, event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id']) 73 | else: 74 | functions[choice](misp, event) 75 | -------------------------------------------------------------------------------- /examples/situational-awareness/tag_search.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | from datetime import datetime 7 | import argparse 8 | import tools 9 | import date_tools 10 | 11 | 12 | def init(url, key): 13 | return PyMISP(url, key, misp_verifycert, 'json') 14 | 15 | # ######### fetch data ########## 16 | 17 | 18 | if __name__ == '__main__': 19 | parser = argparse.ArgumentParser(description='Take a sample of events (based on last.py) and give the number of occurrence of the given tag in this sample.') 20 | parser.add_argument("-t", "--tag", required=True, help="tag to search (search for multiple tags is possible by using |. example : \"osint|OSINT\")") 21 | parser.add_argument("-d", "--days", type=int, help="number of days before today to search. If not define, default value is 7") 22 | parser.add_argument("-b", "--begindate", help="The research will look for tags attached to events posted at or after the given startdate (format: yyyy-mm-dd): If no date is given, default time is epoch time (1970-1-1)") 23 | parser.add_argument("-e", "--enddate", help="The research will look for tags attached to events posted at or before the given enddate (format: yyyy-mm-dd): If no date is given, default time is now()") 24 | 25 | args = parser.parse_args() 26 | 27 | misp = init(misp_url, misp_key) 28 | 29 | if args.days is None: 30 | args.days = 7 31 | result = misp.search(last='{}d'.format(args.days), metadata=True) 32 | 33 | date_tools.checkDateConsistancy(args.begindate, args.enddate, date_tools.getLastdate(args.days)) 34 | 35 | if args.begindate is None: 36 | args.begindate = date_tools.getLastdate(args.days) 37 | else: 38 | args.begindate = date_tools.setBegindate(date_tools.toDatetime(args.begindate), tools.getLastdate(args.days)) 39 | 40 | if args.enddate is None: 41 | args.enddate = datetime.now() 42 | else: 43 | args.enddate = date_tools.setEnddate(date_tools.toDatetime(args.enddate)) 44 | 45 | if 'response' in result: 46 | events = tools.selectInRange(tools.eventsListBuildFromArray(result), begin=args.begindate, end=args.enddate) 47 | totalPeriodEvents = tools.getNbitems(events) 48 | tags = tools.tagsListBuild(events) 49 | result = tools.isTagIn(tags, args.tag) 50 | totalPeriodTags = len(result) 51 | 52 | text = 'Studied pediod: from ' 53 | if args.begindate is None: 54 | text = text + '1970-01-01' 55 | else: 56 | text = text + str(args.begindate.date()) 57 | text = text + ' to ' 58 | if args.enddate is None: 59 | text = text + str(datetime.now().date()) 60 | else: 61 | text = text + str(args.enddate.date()) 62 | 63 | print('\n========================================================') 64 | print(text) 65 | print('During the studied pediod, ' + str(totalPeriodTags) + ' events out of ' + str(totalPeriodEvents) + ' contains at least one tag with ' + args.tag + '.') 66 | if totalPeriodEvents != 0: 67 | print('It represents {}% of the events in this period.'.format(round(100 * totalPeriodTags / totalPeriodEvents, 3))) 68 | else: 69 | print ('There is no event answering the research criteria') 70 | 71 | -------------------------------------------------------------------------------- /examples/ioc-2-misp/keys.py.sample: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | mispUrl = '' 5 | mispKey = '' 6 | 7 | ############################### 8 | # file use for internal tag 9 | # some sample can be find here : 10 | # https://github.com/eset/malware-ioc 11 | # https://github.com/fireeye/iocs 12 | csvTaxonomyFile = "taxonomy.csv" 13 | 14 | # csv delimiter : ";" with quotechar : " 15 | 16 | ############################### 17 | # link sample 18 | #~ 19 | #~ APT 20 | #~ APT12 21 | #~ Backdoor 22 | #~ Apache 2.0 23 | #~ 24 | 25 | # @link from csv 26 | # = rel attribut from 27 | # @value from csv 28 | # = value 29 | # @keep 30 | # 0 : don't create tag 31 | # 1 : tag created 32 | # @taxonomy 33 | # define tag for misp 34 | # @comment 35 | # litte description but not use 36 | 37 | 38 | ######################################### 39 | # https://www.circl.lu/doc/misp/categories-and-types/index.html 40 | # /\ 41 | # || 42 | # || 43 | # \/ 44 | # http://schemas.mandiant.com/ 45 | 46 | # @index = Context/search form ioc 47 | # @(1, 2, 3) 48 | # 1. categorie mapping 49 | # 2. type mapping 50 | # 3. optionnal comment 51 | 52 | 53 | iocMispMapping = { 54 | 55 | ('DriverItem/DriverName') : (u'Artifacts dropped',u'other', u'DriverName. '), 56 | 57 | ('DnsEntryItem/Host') : (u'Network activity',u'domain'), 58 | 59 | ('Email/To') : (u'Targeting data',u'target-email'), 60 | ('Email/Date') : (u'Other',u'comment',u'EmailDate. '), 61 | ('Email/Body') : (u'Payload delivery',u'email-subject'), 62 | ('Email/From') : (u'Payload delivery',u'email-dst'), 63 | ('Email/Subject') : (u'Payload delivery',u'email-subject'), 64 | ('Email/Attachment/Name') : (u'Payload delivery',u'email-attachment'), 65 | 66 | ('FileItem/Md5sum') : (u'External analysis',u'md5'), 67 | ('FileItem/Sha1sum') : (u'External analysis',u'sha1'), 68 | ('FileItem/FileName') : (u'External analysis',u'filename'), 69 | ('FileItem/FullPath') : (u'External analysis',u'filename'), 70 | ('FileItem/FilePath') : (u'External analysis',u'filename'), 71 | ('FileItem/Sha256sum') : (u'External analysis',u'sha256'), 72 | 73 | ('Network/URI') : (u'Network activity',u'uri'), 74 | ('Network/DNS') : (u'Network activity',u'domain'), 75 | ('Network/String') : (u'Network activity',u'ip-dst'), 76 | ('Network/UserAgent') : (u'Network activity',u'user-agent'), 77 | 78 | ('PortItem/localIP') : (u'Network activity',u'ip-dst'), 79 | 80 | ('ProcessItem/name') : (u'External analysis',u'pattern-in-memory', u'ProcessName. '), 81 | ('ProcessItem/path') : (u'External analysis',u'pattern-in-memory', u'ProcessPath. '), 82 | ('ProcessItem/Mutex') : (u'Artifacts dropped',u'mutex', u'mutex'), 83 | ('ProcessItem/Pipe/Name') : (u'Artifacts dropped',u'named pipe'), 84 | ('ProcessItem/Mutex/Name') : (u'Artifacts dropped',u'mutex', u'MutexName. '), 85 | 86 | ('RegistryItem/Text') : (u'Artifacts dropped',u'regkey', u'RegistryText. '), 87 | ('RegistryItem/Path') : (u'Artifacts dropped',u'regkey', u'RegistryPath. '), 88 | 89 | ('ServiceItem/name') : (u'Artifacts dropped',u'windows-service-name'), 90 | ('ServiceItem/type') : (u'Artifacts dropped',u'pattern-in-memory', u'ServiceType. '), 91 | 92 | ('Snort/Snort') : (u'Network activity',u'snort'), 93 | 94 | } 95 | -------------------------------------------------------------------------------- /pymisp/tools/vtreportobject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import re 5 | 6 | import requests 7 | try: 8 | import validators 9 | has_validators = True 10 | except ImportError: 11 | has_validators = False 12 | 13 | 14 | from .abstractgenerator import AbstractMISPObjectGenerator 15 | from .. import InvalidMISPObject 16 | 17 | 18 | class VTReportObject(AbstractMISPObjectGenerator): 19 | ''' 20 | VirusTotal Report 21 | 22 | :apikey: VirusTotal API key (private works, but only public features are supported right now) 23 | 24 | :indicator: IOC to search VirusTotal for 25 | ''' 26 | def __init__(self, apikey, indicator, vt_proxies=None, standalone=True, **kwargs): 27 | # PY3 way: 28 | # super().__init__("virustotal-report") 29 | super(VTReportObject, self).__init__("virustotal-report", standalone=standalone, **kwargs) 30 | indicator = indicator.strip() 31 | self._resource_type = self.__validate_resource(indicator) 32 | if self._resource_type: 33 | self._proxies = vt_proxies 34 | self._report = self.__query_virustotal(apikey, indicator) 35 | self.generate_attributes() 36 | else: 37 | error_msg = "A valid indicator is required. (One of type url, md5, sha1, sha256). Received '{}' instead".format(indicator) 38 | raise InvalidMISPObject(error_msg) 39 | 40 | def get_report(self): 41 | return self._report 42 | 43 | def generate_attributes(self): 44 | ''' Parse the VirusTotal report for relevant attributes ''' 45 | self.add_attribute("last-submission", value=self._report["scan_date"]) 46 | self.add_attribute("permalink", value=self._report["permalink"]) 47 | ratio = "{}/{}".format(self._report["positives"], self._report["total"]) 48 | self.add_attribute("detection-ratio", value=ratio) 49 | 50 | def __validate_resource(self, ioc): 51 | ''' 52 | Validate the data type of an indicator. 53 | Domains and IP addresses aren't supported because 54 | they don't return the same type of data as the URLs/files do 55 | 56 | :ioc: Indicator to search VirusTotal for 57 | ''' 58 | if not has_validators: 59 | raise Exception('You need to install validators: pip install validators') 60 | if validators.url(ioc): 61 | return "url" 62 | elif re.match(r"\b([a-fA-F0-9]{32}|[a-fA-F0-9]{40}|[a-fA-F0-9]{64})\b", ioc): 63 | return "file" 64 | return False 65 | 66 | def __query_virustotal(self, apikey, resource): 67 | ''' 68 | Query VirusTotal for information about an indicator 69 | 70 | :apikey: VirusTotal API key 71 | 72 | :resource: Indicator to search in VirusTotal 73 | ''' 74 | url = "https://www.virustotal.com/vtapi/v2/{}/report".format(self._resource_type) 75 | params = {"apikey": apikey, "resource": resource} 76 | # for now assume we're using a public API key - we'll figure out private keys later 77 | if self._proxies: 78 | report = requests.get(url, params=params, proxies=self._proxies) 79 | else: 80 | report = requests.get(url, params=params) 81 | report = report.json() 82 | if report["response_code"] == 1: 83 | return report 84 | else: 85 | error_msg = "{}: {}".format(resource, report["verbose_msg"]) 86 | raise InvalidMISPObject(error_msg) 87 | -------------------------------------------------------------------------------- /pymisp/tools/fileobject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from ..exceptions import InvalidMISPObject 5 | from .abstractgenerator import AbstractMISPObjectGenerator 6 | import os 7 | from io import BytesIO 8 | from hashlib import md5, sha1, sha256, sha512 9 | import math 10 | from collections import Counter 11 | import logging 12 | 13 | logger = logging.getLogger('pymisp') 14 | 15 | 16 | try: 17 | import pydeep 18 | HAS_PYDEEP = True 19 | except ImportError: 20 | HAS_PYDEEP = False 21 | 22 | try: 23 | import magic 24 | HAS_MAGIC = True 25 | except ImportError: 26 | HAS_MAGIC = False 27 | 28 | 29 | class FileObject(AbstractMISPObjectGenerator): 30 | 31 | def __init__(self, filepath=None, pseudofile=None, filename=None, standalone=True, **kwargs): 32 | if not HAS_PYDEEP: 33 | logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") 34 | if not HAS_MAGIC: 35 | logger.warning("Please install python-magic: pip install python-magic.") 36 | if filename: 37 | # Useful in case the file is copied with a pre-defined name by a script but we want to keep the original name 38 | self.__filename = filename 39 | elif filepath: 40 | self.__filename = os.path.basename(filepath) 41 | else: 42 | raise InvalidMISPObject('A file name is required (either in the path, or as a parameter).') 43 | 44 | if filepath: 45 | with open(filepath, 'rb') as f: 46 | self.__pseudofile = BytesIO(f.read()) 47 | elif pseudofile and isinstance(pseudofile, BytesIO): 48 | # WARNING: lief.parse requires a path 49 | self.__pseudofile = pseudofile 50 | else: 51 | raise InvalidMISPObject('File buffer (BytesIO) or a path is required.') 52 | # PY3 way: 53 | # super().__init__('file') 54 | super(FileObject, self).__init__('file', standalone=standalone, **kwargs) 55 | self.__data = self.__pseudofile.getvalue() 56 | self.generate_attributes() 57 | 58 | def generate_attributes(self): 59 | self.add_attribute('filename', value=self.__filename) 60 | size = self.add_attribute('size-in-bytes', value=len(self.__data)) 61 | if int(size.value) > 0: 62 | self.add_attribute('entropy', value=self.__entropy_H(self.__data)) 63 | self.add_attribute('md5', value=md5(self.__data).hexdigest()) 64 | self.add_attribute('sha1', value=sha1(self.__data).hexdigest()) 65 | self.add_attribute('sha256', value=sha256(self.__data).hexdigest()) 66 | self.add_attribute('sha512', value=sha512(self.__data).hexdigest()) 67 | self.add_attribute('malware-sample', value=self.__filename, data=self.__pseudofile) 68 | if HAS_MAGIC: 69 | self.add_attribute('mimetype', value=magic.from_buffer(self.__data)) 70 | if HAS_PYDEEP: 71 | self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode()) 72 | 73 | def __entropy_H(self, data): 74 | """Calculate the entropy of a chunk of data.""" 75 | # NOTE: copy of the entropy function from pefile 76 | 77 | if len(data) == 0: 78 | return 0.0 79 | 80 | occurences = Counter(bytearray(data)) 81 | 82 | entropy = 0 83 | for x in occurences.values(): 84 | p_x = float(x) / len(data) 85 | entropy -= p_x * math.log(p_x, 2) 86 | 87 | return entropy 88 | -------------------------------------------------------------------------------- /examples/feed-generator-from-redis/MISPItemToRedis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import redis 3 | import json 4 | 5 | 6 | class MISPItemToRedis: 7 | """This class provides a simple normalization to add MISP item to 8 | redis, so that they can easily be processed and added to MISP later on.""" 9 | SUFFIX_SIGH = '_sighting' 10 | SUFFIX_ATTR = '_attribute' 11 | SUFFIX_OBJ = '_object' 12 | SUFFIX_LIST = [SUFFIX_SIGH, SUFFIX_ATTR, SUFFIX_OBJ] 13 | 14 | def __init__(self, keyname, host='localhost', port=6379, db=0): 15 | self.host = host 16 | self.port = port 17 | self.db = db 18 | self.keyname = keyname 19 | self.serv = redis.StrictRedis(self.host, self.port, self.db) 20 | 21 | def push_json(self, jdata, keyname, action): 22 | all_action = [s.lstrip('_') for s in self.SUFFIX_LIST] 23 | if action not in all_action: 24 | raise('Error: Invalid action. (Allowed: {})'.format(all_action)) 25 | key = keyname + '_' + action 26 | self.serv.lpush(key, jdata) 27 | 28 | def push_attribute(self, type_value, value, category=None, to_ids=False, 29 | comment=None, distribution=None, proposal=False, **kwargs): 30 | to_push = {} 31 | to_push['type'] = type_value 32 | to_push['value'] = value 33 | if category is not None: 34 | to_push['category'] = category 35 | if to_ids is not None: 36 | to_push['to_ids'] = to_ids 37 | if comment is not None: 38 | to_push['comment'] = comment 39 | if distribution is not None: 40 | to_push['distribution'] = distribution 41 | if proposal is not None: 42 | to_push['proposal'] = proposal 43 | for k, v in kwargs.items(): 44 | to_push[k] = v 45 | key = self.keyname + self.SUFFIX_ATTR 46 | self.serv.lpush(key, json.dumps(to_push)) 47 | 48 | def push_attribute_obj(self, MISP_Attribute, keyname): 49 | key = keyname + self.SUFFIX_ATTR 50 | jdata = MISP_Attribute.to_json() 51 | self.serv.lpush(key, jdata) 52 | 53 | def push_object(self, dict_values): 54 | # check that 'name' field is present 55 | if 'name' not in dict_values: 56 | print("Error: JSON must contain the field 'name'") 57 | key = self.keyname + self.SUFFIX_OBJ 58 | self.serv.lpush(key, json.dumps(dict_values)) 59 | 60 | def push_object_obj(self, MISP_Object, keyname): 61 | key = keyname + self.SUFFIX_OBJ 62 | jdata = MISP_Object.to_json() 63 | self.serv.lpush(key, jdata) 64 | 65 | def push_sighting(self, value=None, uuid=None, id=None, source=None, 66 | type=0, timestamp=None, **kargs): 67 | to_push = {} 68 | if value is not None: 69 | to_push['value'] = value 70 | if uuid is not None: 71 | to_push['uuid'] = uuid 72 | if id is not None: 73 | to_push['id'] = id 74 | if source is not None: 75 | to_push['source'] = source 76 | if type is not None: 77 | to_push['type'] = type 78 | if timestamp is not None: 79 | to_push['timestamp'] = timestamp 80 | 81 | for k, v in kargs.items(): 82 | if v is not None: 83 | to_push[k] = v 84 | key = self.keyname + self.SUFFIX_SIGH 85 | self.serv.lpush(key, json.dumps(to_push)) 86 | 87 | def push_sighting_obj(self, MISP_Sighting, keyname): 88 | key = keyname + self.SUFFIX_SIGH 89 | jdata = MISP_Sighting.to_json() 90 | self.serv.lpush(key, jdata) 91 | -------------------------------------------------------------------------------- /examples/add_fail2ban_object.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP, MISPEvent 5 | from pymisp.tools import Fail2BanObject 6 | import argparse 7 | from base64 import b64decode 8 | from io import BytesIO 9 | import os 10 | from datetime import date, datetime 11 | from dateutil.parser import parse 12 | 13 | 14 | try: 15 | from keys import misp_url, misp_key, misp_verifycert 16 | except Exception: 17 | misp_url = 'URL' 18 | misp_key = 'AUTH_KEY' 19 | misp_verifycert = True 20 | 21 | 22 | def create_new_event(): 23 | me = MISPEvent() 24 | me.info = "Fail2Ban blocking" 25 | me.add_tag(args.tag) 26 | start = datetime.now() 27 | me.add_attribute('datetime', start.isoformat(), comment='Start Time') 28 | return me 29 | 30 | 31 | if __name__ == '__main__': 32 | parser = argparse.ArgumentParser(description='Add Fail2ban object.') 33 | parser.add_argument("-b", "--banned_ip", required=True, help="Banned IP address.") 34 | parser.add_argument("-a", "--attack_type", required=True, help="Type of attack.") 35 | parser.add_argument("-t", "--tag", required=True, help="Tag to search on MISP.") 36 | parser.add_argument("-p", "--processing_timestamp", help="Processing timestamp.") 37 | parser.add_argument("-f", "--failures", help="Amount of failures that lead to the ban.") 38 | parser.add_argument("-s", "--sensor", help="Sensor identifier.") 39 | parser.add_argument("-v", "--victim", help="Victim identifier.") 40 | parser.add_argument("-l", "--logline", help="Logline (base64 encoded).") 41 | parser.add_argument("-F", "--logfile", help="Path to a logfile to attach.") 42 | parser.add_argument("-n", "--force_new", action='store_true', default=False, help="Force new MISP event.") 43 | parser.add_argument("-d", "--disable_new", action='store_true', default=False, help="Do not create a new Event.") 44 | args = parser.parse_args() 45 | 46 | pymisp = PyMISP(misp_url, misp_key, misp_verifycert, debug=True) 47 | event_id = -1 48 | me = None 49 | if args.force_new: 50 | me = create_new_event() 51 | else: 52 | response = pymisp.search_index(tag=args.tag, timestamp='1h') 53 | if response['response']: 54 | if args.disable_new: 55 | event_id = response['response'][0]['id'] 56 | else: 57 | last_event_date = parse(response['response'][0]['date']).date() 58 | nb_attr = response['response'][0]['attribute_count'] 59 | if last_event_date < date.today() or int(nb_attr) > 1000: 60 | me = create_new_event() 61 | else: 62 | event_id = response['response'][0]['id'] 63 | else: 64 | me = create_new_event() 65 | 66 | parameters = {'banned-ip': args.banned_ip, 'attack-type': args.attack_type} 67 | if args.processing_timestamp: 68 | parameters['processing-timestamp'] = args.processing_timestamp 69 | if args.failures: 70 | parameters['failures'] = args.failures 71 | if args.sensor: 72 | parameters['sensor'] = args.sensor 73 | if args.victim: 74 | parameters['victim'] = args.victim 75 | if args.logline: 76 | parameters['logline'] = b64decode(args.logline).decode() 77 | if args.logfile: 78 | with open(args.logfile, 'rb') as f: 79 | parameters['logfile'] = {'value': os.path.basename(args.logfile), 80 | 'data': BytesIO(f.read())} 81 | f2b = Fail2BanObject(parameters=parameters, standalone=False) 82 | if me: 83 | me.add_object(f2b) 84 | pymisp.add_event(me) 85 | elif event_id: 86 | template_id = pymisp.get_object_template_id(f2b.template_uuid) 87 | a = pymisp.add_object(event_id, template_id, f2b) 88 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | README 2 | ====== 3 | 4 | [![Documentation Status](https://readthedocs.org/projects/pymisp/badge/?version=latest)](http://pymisp.readthedocs.io/?badge=latest) 5 | [![Build Status](https://travis-ci.org/MISP/PyMISP.svg?branch=master)](https://travis-ci.org/MISP/PyMISP) 6 | [![Coverage Status](https://coveralls.io/repos/github/MISP/PyMISP/badge.svg?branch=master)](https://coveralls.io/github/MISP/PyMISP?branch=master) 7 | 8 | # PyMISP - Python Library to access MISP 9 | 10 | PyMISP is a Python library to access [MISP](https://github.com/MISP/MISP) platforms via their REST API. 11 | 12 | PyMISP allows you to fetch events, add or update events/attributes, add or update samples or search for attributes. 13 | 14 | ## Requirements 15 | 16 | * [requests](http://docs.python-requests.org) 17 | 18 | ## Install from pip 19 | 20 | ``` 21 | pip3 install pymisp 22 | ``` 23 | 24 | ## Install the latest version from repo 25 | 26 | ``` 27 | git clone https://github.com/MISP/PyMISP.git && cd PyMISP 28 | git submodule update --init 29 | pip3 install -I . 30 | ``` 31 | 32 | ## Samples and how to use PyMISP 33 | 34 | Various examples and samples scripts are in the [examples/](examples/) directory. 35 | 36 | In the examples directory, you will need to change the keys.py.sample to enter your MISP url and API key. 37 | 38 | ``` 39 | cd examples 40 | cp keys.py.sample keys.py 41 | vim keys.py 42 | ``` 43 | 44 | The API key of MISP is available in the Automation section of the MISP web interface. 45 | 46 | To test if your URL and API keys are correct, you can test with examples/last.py to 47 | fetch the last 10 events published. 48 | 49 | ``` 50 | cd examples 51 | python3 last.py -l 10 52 | ``` 53 | 54 | ## Debugging 55 | 56 | You have two options there: 57 | 58 | 1. Pass `debug=True` to `PyMISP` and it will enable logging.DEBUG to stderr on the whole module 59 | 60 | 2. Use the python logging module directly: 61 | 62 | ```python 63 | 64 | import logging 65 | logger = logging.getLogger('pymisp') 66 | 67 | # Configure it as you whish, for example, enable DEBUG mode: 68 | logger.setLevel(logging.DEBUG) 69 | ``` 70 | 71 | Or if you want to write the debug output to a file instead of stderr: 72 | 73 | ```python 74 | import pymisp 75 | import logging 76 | 77 | logger = logging.getLogger('pymisp') 78 | logging.basicConfig(level=logging.DEBUG, filename="debug.log", filemode='w', format=pymisp.FORMAT) 79 | ``` 80 | 81 | ## Documentation 82 | 83 | [PyMISP API documentation is available](https://media.readthedocs.org/pdf/pymisp/latest/pymisp.pdf). 84 | 85 | Documentation can be generated with epydoc: 86 | 87 | ``` 88 | epydoc --url https://github.com/MISP/PyMISP --graph all --name PyMISP --pdf pymisp -o doc 89 | ``` 90 | 91 | ## Everything is a Mutable Mapping 92 | 93 | ... or at least everything that can be imported/exported from/to a json blob 94 | 95 | `AbstractMISP` is the master class, and inherit `collections.MutableMapping` which means 96 | the class can be represented as a python dictionary. 97 | 98 | The abstraction assumes every property that should not be seen in the dictionary is prepended with a `_`, 99 | or its name is added to the private list `__not_jsonable` (accessible through `update_not_jsonable` and `set_not_jsonable`. 100 | 101 | This master class has helpers that will make it easy to load, and export, to, and from, a json string. 102 | 103 | `MISPEvent`, `MISPAttribute`, `MISPObjectReference`, `MISPObjectAttribute`, and `MISPObject` 104 | are subclasses of AbstractMISP, which mean that they can be handled as python dictionaries. 105 | 106 | ## MISP Objects 107 | 108 | Creating a new MISP object generator should be done using a pre-defined template and inherit `AbstractMISPObjectGenerator`. 109 | 110 | Your new MISPObject generator need to generate attributes, and add them as class properties using `add_attribute`. 111 | 112 | When the object is sent to MISP, all the class properties will be exported to the JSON export. 113 | -------------------------------------------------------------------------------- /pymisp/tools/machoobject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from ..exceptions import InvalidMISPObject 5 | from .abstractgenerator import AbstractMISPObjectGenerator 6 | from io import BytesIO 7 | from hashlib import md5, sha1, sha256, sha512 8 | import logging 9 | 10 | logger = logging.getLogger('pymisp') 11 | 12 | 13 | try: 14 | import lief 15 | HAS_LIEF = True 16 | except ImportError: 17 | HAS_LIEF = False 18 | 19 | try: 20 | import pydeep 21 | HAS_PYDEEP = True 22 | except ImportError: 23 | HAS_PYDEEP = False 24 | 25 | 26 | class MachOObject(AbstractMISPObjectGenerator): 27 | 28 | def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): 29 | if not HAS_PYDEEP: 30 | logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") 31 | if not HAS_LIEF: 32 | raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') 33 | if pseudofile: 34 | if isinstance(pseudofile, BytesIO): 35 | self.__macho = lief.MachO.parse(raw=pseudofile.getvalue()) 36 | elif isinstance(pseudofile, bytes): 37 | self.__macho = lief.MachO.parse(raw=pseudofile) 38 | else: 39 | raise InvalidMISPObject('Pseudo file can be BytesIO or bytes got {}'.format(type(pseudofile))) 40 | elif filepath: 41 | self.__macho = lief.MachO.parse(filepath) 42 | elif parsed: 43 | # Got an already parsed blob 44 | if isinstance(parsed, lief.MachO.Binary): 45 | self.__macho = parsed 46 | else: 47 | raise InvalidMISPObject('Not a lief.MachO.Binary: {}'.format(type(parsed))) 48 | # Python3 way 49 | # super().__init__('elf') 50 | super(MachOObject, self).__init__('macho', standalone=standalone, **kwargs) 51 | self.generate_attributes() 52 | 53 | def generate_attributes(self): 54 | self.add_attribute('type', value=str(self.__macho.header.file_type).split('.')[1]) 55 | self.add_attribute('name', value=self.__macho.name) 56 | # General information 57 | if self.__macho.has_entrypoint: 58 | self.add_attribute('entrypoint-address', value=self.__macho.entrypoint) 59 | # Sections 60 | self.sections = [] 61 | if self.__macho.sections: 62 | pos = 0 63 | for section in self.__macho.sections: 64 | s = MachOSectionObject(section, self._standalone, default_attributes_parameters=self._default_attributes_parameters) 65 | self.add_reference(s.uuid, 'included-in', 'Section {} of MachO'.format(pos)) 66 | pos += 1 67 | self.sections.append(s) 68 | self.add_attribute('number-sections', value=len(self.sections)) 69 | 70 | 71 | class MachOSectionObject(AbstractMISPObjectGenerator): 72 | 73 | def __init__(self, section, standalone=True, **kwargs): 74 | # Python3 way 75 | # super().__init__('pe-section') 76 | super(MachOSectionObject, self).__init__('macho-section', standalone=standalone, **kwargs) 77 | self.__section = section 78 | self.__data = bytes(self.__section.content) 79 | self.generate_attributes() 80 | 81 | def generate_attributes(self): 82 | self.add_attribute('name', value=self.__section.name) 83 | size = self.add_attribute('size-in-bytes', value=self.__section.size) 84 | if int(size.value) > 0: 85 | self.add_attribute('entropy', value=self.__section.entropy) 86 | self.add_attribute('md5', value=md5(self.__data).hexdigest()) 87 | self.add_attribute('sha1', value=sha1(self.__data).hexdigest()) 88 | self.add_attribute('sha256', value=sha256(self.__data).hexdigest()) 89 | self.add_attribute('sha512', value=sha512(self.__data).hexdigest()) 90 | if HAS_PYDEEP: 91 | self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode()) 92 | -------------------------------------------------------------------------------- /examples/yara_dump.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | ''' 4 | YARA dumper for MISP 5 | by Christophe Vandeplas 6 | ''' 7 | 8 | import keys 9 | from pymisp import PyMISP 10 | import yara 11 | import re 12 | 13 | 14 | def dirty_cleanup(value): 15 | changed = False 16 | substitutions = (('”', '"'), 17 | ('“', '"'), 18 | ('″', '"'), 19 | ('`', "'"), 20 | ('\r', ''), 21 | ('Rule ', 'rule ') # some people write this with the wrong case 22 | # ('$ ', '$'), # this breaks rules 23 | # ('\t\t', '\n'), # this breaks rules 24 | ) 25 | for substitution in substitutions: 26 | if substitution[0] in value: 27 | changed = True 28 | value = value.replace(substitution[0], substitution[1]) 29 | return value, changed 30 | 31 | 32 | misp = PyMISP(keys.misp_url, keys.misp_key, keys.misp_verify, 'json') 33 | result = misp.search(controller='attributes', type_attribute='yara') 34 | 35 | attr_cnt = 0 36 | attr_cnt_invalid = 0 37 | attr_cnt_duplicate = 0 38 | attr_cnt_changed = 0 39 | yara_rules = [] 40 | yara_rule_names = [] 41 | if 'response' in result and 'Attribute' in result['response']: 42 | for attribute in result['response']['Attribute']: 43 | value = attribute['value'] 44 | event_id = attribute['event_id'] 45 | attribute_id = attribute['id'] 46 | 47 | value = re.sub('^[ \t]*rule ', 'rule misp_e{}_'.format(event_id), value, flags=re.MULTILINE) 48 | value, changed = dirty_cleanup(value) 49 | if changed: 50 | attr_cnt_changed += 1 51 | if 'global rule' in value: # refuse any global rules as they might disable everything 52 | continue 53 | if 'private rule' in value: # private rules need some more rewriting 54 | priv_rules = re.findall('private rule (\w+)', value, flags=re.MULTILINE) 55 | for priv_rule in priv_rules: 56 | value = re.sub(priv_rule, 'misp_e{}_{}'.format(event_id, priv_rule), value, flags=re.MULTILINE) 57 | 58 | # compile the yara rule to confirm it's validity 59 | # if valid, ignore duplicate rules 60 | try: 61 | attr_cnt += 1 62 | yara.compile(source=value) 63 | yara_rules.append(value) 64 | # print("Rule e{} a{} OK".format(event_id, attribute_id)) 65 | except yara.SyntaxError as e: 66 | attr_cnt_invalid += 1 67 | # print("Rule e{} a{} NOK - {}".format(event_id, attribute_id, e)) 68 | except yara.Error as e: 69 | attr_cnt_invalid += 1 70 | print(e) 71 | import traceback 72 | print(traceback.format_exc()) 73 | 74 | # remove duplicates - process the full yara rule list and process errors to eliminate duplicate rule names 75 | all_yara_rules = '\n'.join(yara_rules) 76 | while True: 77 | try: 78 | yara.compile(source=all_yara_rules) 79 | except yara.SyntaxError as e: 80 | if 'duplicated identifier' in e.args[0]: 81 | duplicate_rule_names = re.findall('duplicated identifier "(.*)"', e.args[0]) 82 | for item in duplicate_rule_names: 83 | all_yara_rules = all_yara_rules.replace('rule {}'.format(item), 'rule duplicate_{}'.format(item), 1) 84 | attr_cnt_duplicate += 1 85 | continue 86 | else: 87 | # This should never happen as all rules were processed before separately. So logically we should only have duplicates. 88 | exit("ERROR SyntaxError in rules: {}".format(e.args)) 89 | break 90 | 91 | # save to a file 92 | fname = 'misp.yara' 93 | with open(fname, 'w') as f_out: 94 | f_out.write(all_yara_rules) 95 | 96 | print("") 97 | print("MISP attributes with YARA rules: total={} valid={} invalid={} duplicate={} changed={}.".format(attr_cnt, attr_cnt - attr_cnt_invalid, attr_cnt_invalid, attr_cnt_duplicate, attr_cnt_changed)) 98 | print("Valid YARA rule file save to file '{}'. Invalid rules/attributes were ignored.".format(fname)) 99 | -------------------------------------------------------------------------------- /examples/situational-awareness/tags_to_graphs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from pymisp import PyMISP 5 | from keys import misp_url, misp_key, misp_verifycert 6 | import argparse 7 | import tools 8 | import date_tools 9 | import bokeh_tools 10 | 11 | 12 | def formattingDataframe(dataframe, dates, NanValue): 13 | dataframe.reverse() 14 | dates.reverse() 15 | dataframe = tools.concat(dataframe) 16 | dataframe = tools.renameColumns(dataframe, dates) 17 | dataframe = tools.replaceNaN(dataframe, 0) 18 | return dataframe 19 | 20 | if __name__ == '__main__': 21 | parser = argparse.ArgumentParser(description='Show the evolution of trend of tags.') 22 | parser.add_argument("-p", "--period", help='Define the studied period. Can be the past year (y), month (m) or week (w). Week is the default value if no valid value is given.') 23 | parser.add_argument("-a", "--accuracy", help='Define the accuracy of the splits on the studied period. Can be per month (m) -for year only-, week (w) -month only- or day (d). The default value is always the biggest available.') 24 | parser.add_argument("-o", "--order", type=int, help='Define the accuracy of the curve fitting. Default value is 3') 25 | 26 | args = parser.parse_args() 27 | 28 | misp = PyMISP(misp_url, misp_key, misp_verifycert) 29 | 30 | if args.period == "y": 31 | if args.accuracy == "d": 32 | split = 360 33 | size = 1 34 | else: 35 | split = 12 36 | size = 30 37 | last = '360d' 38 | title = 'Tags repartition over the last 360 days' 39 | elif args.period == "m": 40 | if args.accuracy == "d": 41 | split = 28 42 | size = 1 43 | else: 44 | split = 4 45 | size = 7 46 | last = '28d' 47 | title = 'Tags repartition over the last 28 days' 48 | else: 49 | split = 7 50 | size = 1 51 | last = '7d' 52 | title = 'Tags repartition over the last 7 days' 53 | 54 | result = misp.search(last=last, metadata=True) 55 | if 'response' in result: 56 | events = tools.eventsListBuildFromArray(result) 57 | result = [] 58 | dates = [] 59 | enddate = date_tools.getToday() 60 | colourDict = {} 61 | faketag = False 62 | 63 | for i in range(split): 64 | begindate = date_tools.getNDaysBefore(enddate, size) 65 | dates.append(str(enddate.date())) 66 | eventstemp = tools.selectInRange(events, begin=begindate, end=enddate) 67 | if eventstemp is not None: 68 | tags = tools.tagsListBuild(eventstemp) 69 | if tags is not None: 70 | tools.createDictTagsColour(colourDict, tags) 71 | result.append(tools.getNbOccurenceTags(tags)) 72 | else: 73 | result.append(tools.createFakeEmptyTagsSeries()) 74 | faketag = True 75 | else: 76 | result.append(tools.createFakeEmptyTagsSeries()) 77 | faketag = True 78 | enddate = begindate 79 | 80 | result = formattingDataframe(result, dates, 0) 81 | if faketag: 82 | result = tools.removeFaketagRow(result) 83 | 84 | taxonomies, emptyOther = tools.getTaxonomies(tools.getCopyDataframe(result)) 85 | 86 | tools.tagsToLineChart(tools.getCopyDataframe(result), title, dates, colourDict) 87 | tools.tagstrendToLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict) 88 | tools.tagsToTaxoLineChart(tools.getCopyDataframe(result), title, dates, colourDict, taxonomies, emptyOther) 89 | tools.tagstrendToTaxoLineChart(tools.getCopyDataframe(result), title, dates, split, colourDict, taxonomies, emptyOther) 90 | if args.order is None: 91 | args.order = 3 92 | tools.tagsToPolyChart(tools.getCopyDataframe(result), split, colourDict, taxonomies, emptyOther, args.order) 93 | tools.createVisualisation(taxonomies) 94 | 95 | else: 96 | print('There is no event during the studied period') 97 | -------------------------------------------------------------------------------- /pymisp/tools/elfobject.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | from .abstractgenerator import AbstractMISPObjectGenerator 5 | from ..exceptions import InvalidMISPObject 6 | from io import BytesIO 7 | from hashlib import md5, sha1, sha256, sha512 8 | import logging 9 | 10 | logger = logging.getLogger('pymisp') 11 | 12 | try: 13 | import lief 14 | HAS_LIEF = True 15 | except ImportError: 16 | HAS_LIEF = False 17 | 18 | try: 19 | import pydeep 20 | HAS_PYDEEP = True 21 | except ImportError: 22 | HAS_PYDEEP = False 23 | 24 | 25 | class ELFObject(AbstractMISPObjectGenerator): 26 | 27 | def __init__(self, parsed=None, filepath=None, pseudofile=None, standalone=True, **kwargs): 28 | if not HAS_PYDEEP: 29 | logger.warning("Please install pydeep: pip install git+https://github.com/kbandla/pydeep.git") 30 | if not HAS_LIEF: 31 | raise ImportError('Please install lief, documentation here: https://github.com/lief-project/LIEF') 32 | if pseudofile: 33 | if isinstance(pseudofile, BytesIO): 34 | self.__elf = lief.ELF.parse(raw=pseudofile.getvalue()) 35 | elif isinstance(pseudofile, bytes): 36 | self.__elf = lief.ELF.parse(raw=pseudofile) 37 | else: 38 | raise InvalidMISPObject('Pseudo file can be BytesIO or bytes got {}'.format(type(pseudofile))) 39 | elif filepath: 40 | self.__elf = lief.ELF.parse(filepath) 41 | elif parsed: 42 | # Got an already parsed blob 43 | if isinstance(parsed, lief.ELF.Binary): 44 | self.__elf = parsed 45 | else: 46 | raise InvalidMISPObject('Not a lief.ELF.Binary: {}'.format(type(parsed))) 47 | super(ELFObject, self).__init__('elf', standalone=standalone, **kwargs) 48 | self.generate_attributes() 49 | 50 | def generate_attributes(self): 51 | # General information 52 | self.add_attribute('type', value=str(self.__elf.header.file_type).split('.')[1]) 53 | self.add_attribute('entrypoint-address', value=self.__elf.entrypoint) 54 | self.add_attribute('arch', value=str(self.__elf.header.machine_type).split('.')[1]) 55 | self.add_attribute('os_abi', value=str(self.__elf.header.identity_os_abi).split('.')[1]) 56 | # Sections 57 | self.sections = [] 58 | if self.__elf.sections: 59 | pos = 0 60 | for section in self.__elf.sections: 61 | s = ELFSectionObject(section, self._standalone, default_attributes_parameters=self._default_attributes_parameters) 62 | self.add_reference(s.uuid, 'included-in', 'Section {} of ELF'.format(pos)) 63 | pos += 1 64 | self.sections.append(s) 65 | self.add_attribute('number-sections', value=len(self.sections)) 66 | 67 | 68 | class ELFSectionObject(AbstractMISPObjectGenerator): 69 | 70 | def __init__(self, section, standalone=True, **kwargs): 71 | # Python3 way 72 | # super().__init__('pe-section') 73 | super(ELFSectionObject, self).__init__('elf-section', standalone=standalone, **kwargs) 74 | self.__section = section 75 | self.__data = bytes(self.__section.content) 76 | self.generate_attributes() 77 | 78 | def generate_attributes(self): 79 | self.add_attribute('name', value=self.__section.name) 80 | self.add_attribute('type', value=str(self.__section.type).split('.')[1]) 81 | for flag in self.__section.flags_list: 82 | self.add_attribute('flag', value=str(flag).split('.')[1]) 83 | size = self.add_attribute('size-in-bytes', value=self.__section.size) 84 | if int(size.value) > 0: 85 | self.add_attribute('entropy', value=self.__section.entropy) 86 | self.add_attribute('md5', value=md5(self.__data).hexdigest()) 87 | self.add_attribute('sha1', value=sha1(self.__data).hexdigest()) 88 | self.add_attribute('sha256', value=sha256(self.__data).hexdigest()) 89 | self.add_attribute('sha512', value=sha512(self.__data).hexdigest()) 90 | if HAS_PYDEEP: 91 | self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode()) 92 | -------------------------------------------------------------------------------- /pymisp/tools/create_misp_object.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import six 5 | 6 | from . import FileObject, PEObject, ELFObject, MachOObject 7 | from ..exceptions import MISPObjectException 8 | import logging 9 | 10 | logger = logging.getLogger('pymisp') 11 | 12 | try: 13 | import lief 14 | from lief import Logger 15 | Logger.disable() 16 | HAS_LIEF = True 17 | except ImportError: 18 | HAS_LIEF = False 19 | 20 | 21 | class FileTypeNotImplemented(MISPObjectException): 22 | pass 23 | 24 | 25 | def make_pe_objects(lief_parsed, misp_file, standalone=True, default_attributes_parameters={}): 26 | pe_object = PEObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters) 27 | misp_file.add_reference(pe_object.uuid, 'included-in', 'PE indicators') 28 | pe_sections = [] 29 | for s in pe_object.sections: 30 | pe_sections.append(s) 31 | return misp_file, pe_object, pe_sections 32 | 33 | 34 | def make_elf_objects(lief_parsed, misp_file, standalone=True, default_attributes_parameters={}): 35 | elf_object = ELFObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters) 36 | misp_file.add_reference(elf_object.uuid, 'included-in', 'ELF indicators') 37 | elf_sections = [] 38 | for s in elf_object.sections: 39 | elf_sections.append(s) 40 | return misp_file, elf_object, elf_sections 41 | 42 | 43 | def make_macho_objects(lief_parsed, misp_file, standalone=True, default_attributes_parameters={}): 44 | macho_object = MachOObject(parsed=lief_parsed, standalone=standalone, default_attributes_parameters=default_attributes_parameters) 45 | misp_file.add_reference(macho_object.uuid, 'included-in', 'MachO indicators') 46 | macho_sections = [] 47 | for s in macho_object.sections: 48 | macho_sections.append(s) 49 | return misp_file, macho_object, macho_sections 50 | 51 | 52 | def make_binary_objects(filepath=None, pseudofile=None, filename=None, standalone=True, default_attributes_parameters={}): 53 | misp_file = FileObject(filepath=filepath, pseudofile=pseudofile, filename=filename, 54 | standalone=standalone, default_attributes_parameters=default_attributes_parameters) 55 | if HAS_LIEF and filepath or (pseudofile and filename): 56 | try: 57 | if filepath: 58 | lief_parsed = lief.parse(filepath=filepath) 59 | else: 60 | if six.PY2: 61 | logger.critical('Pseudofile is not supported in python2. Just update.') 62 | lief_parsed = None 63 | else: 64 | lief_parsed = lief.parse(raw=pseudofile.getvalue(), name=filename) 65 | if isinstance(lief_parsed, lief.PE.Binary): 66 | return make_pe_objects(lief_parsed, misp_file, standalone, default_attributes_parameters) 67 | elif isinstance(lief_parsed, lief.ELF.Binary): 68 | return make_elf_objects(lief_parsed, misp_file, standalone, default_attributes_parameters) 69 | elif isinstance(lief_parsed, lief.MachO.Binary): 70 | return make_macho_objects(lief_parsed, misp_file, standalone, default_attributes_parameters) 71 | except lief.bad_format as e: 72 | logger.warning('Bad format: {}'.format(e)) 73 | except lief.bad_file as e: 74 | logger.warning('Bad file: {}'.format(e)) 75 | except lief.conversion_error as e: 76 | logger.warning('Conversion file: {}'.format(e)) 77 | except lief.builder_error as e: 78 | logger.warning('Builder file: {}'.format(e)) 79 | except lief.parser_error as e: 80 | logger.warning('Parser error: {}'.format(e)) 81 | except lief.integrity_error as e: 82 | logger.warning('Integrity error: {}'.format(e)) 83 | except lief.pe_error as e: 84 | logger.warning('PE error: {}'.format(e)) 85 | except lief.type_error as e: 86 | logger.warning('Type error: {}'.format(e)) 87 | except lief.exception as e: 88 | logger.warning('Lief exception: {}'.format(e)) 89 | except FileTypeNotImplemented as e: 90 | logger.warning(e) 91 | if not HAS_LIEF: 92 | logger.warning('Please install lief, documentation here: https://github.com/lief-project/LIEF') 93 | return misp_file, None, None 94 | -------------------------------------------------------------------------------- /examples/et2misp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copy Emerging Threats Block IPs list to several MISP events 5 | # Because of the large size of the list the first run will take a minute 6 | # Running it again will update the MISP events if changes are detected 7 | # 8 | # This script requires PyMISP 2.4.50 or later 9 | 10 | import sys, json, time, requests 11 | from pymisp import PyMISP 12 | from keys import misp_url, misp_key 13 | 14 | et_url = 'https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt' 15 | et_str = 'Emerging Threats ' 16 | 17 | def init_misp(): 18 | global mymisp 19 | mymisp = PyMISP(misp_url, misp_key) 20 | 21 | def load_misp_event(eid): 22 | global et_attr 23 | global et_drev 24 | global et_event 25 | et_attr = {} 26 | et_drev = {} 27 | 28 | et_event = mymisp.get(eid) 29 | echeck(et_event) 30 | for a in et_event['Event']['Attribute']: 31 | if a['category'] == 'Network activity': 32 | et_attr[a['value']] = a['id'] 33 | continue 34 | if a['category'] == 'Internal reference': 35 | et_drev = a; 36 | 37 | def init_et(): 38 | global et_data 39 | global et_rev 40 | requests.packages.urllib3.disable_warnings() 41 | s = requests.Session() 42 | r = s.get(et_url) 43 | if r.status_code != 200: 44 | raise Exception('Error getting ET data: {}'.format(r.text)) 45 | name = '' 46 | et_data = {} 47 | et_rev = 0 48 | for line in r.text.splitlines(): 49 | if line.startswith('# Rev '): 50 | et_rev = int(line[6:]) 51 | continue 52 | if line.startswith('#'): 53 | name = line[1:].strip() 54 | if et_rev and not et_data.get(name): 55 | et_data[name] = {} 56 | continue 57 | l = line.rstrip() 58 | if l: 59 | et_data[name][l] = name 60 | 61 | def update_et_event(name): 62 | if et_drev and et_rev and int(et_drev['value']) < et_rev: 63 | # Copy MISP attributes to new dict 64 | et_ips = dict.fromkeys(et_attr.keys()) 65 | 66 | # Weed out attributes still in ET data 67 | for k,v in et_data[name].items(): 68 | et_attr.pop(k, None) 69 | 70 | # Delete the leftover attributes from MISP 71 | for k,v in et_attr.items(): 72 | r = mymisp.delete_attribute(v) 73 | if r.get('errors'): 74 | print "Error deleting attribute {} ({}): {}\n".format(v,k,r['errors']) 75 | 76 | # Weed out ips already in the MISP event 77 | for k,v in et_ips.items(): 78 | et_data[name].pop(k, None) 79 | 80 | # Add new attributes to MISP event 81 | ipdst = [] 82 | for i,k in enumerate(et_data[name].items(), 1-len(et_data[name])): 83 | ipdst.append(k[0]) 84 | if i % 100 == 0: 85 | r = mymisp.add_ipdst(et_event, ipdst) 86 | echeck(r, et_event['Event']['id']) 87 | ipdst = [] 88 | 89 | # Update revision number 90 | et_drev['value'] = et_rev 91 | et_drev.pop('timestamp', None) 92 | attr = [] 93 | attr.append(et_drev) 94 | 95 | # Publish updated MISP event 96 | et_event['Event']['Attribute'] = attr 97 | et_event['Event']['published'] = False 98 | et_event['Event']['date'] = time.strftime('%Y-%m-%d') 99 | r = mymisp.publish(et_event) 100 | echeck(r, et_event['Event']['id']) 101 | 102 | def echeck(r, eid=None): 103 | if r.get('errors'): 104 | if eid: 105 | print "Processing event {} failed: {}".format(eid, r['errors']) 106 | else: 107 | print r['errors'] 108 | sys.exit(1) 109 | 110 | if __name__ == '__main__': 111 | init_misp() 112 | init_et() 113 | 114 | for et_type in set(et_data.keys()): 115 | info = et_str + et_type 116 | r = mymisp.search_index(eventinfo=info) 117 | if r['response']: 118 | eid=r['response'][0]['id'] 119 | else: # event not found, create it 120 | new_event = mymisp.new_event(info=info, distribution=3, threat_level_id=4, analysis=1) 121 | echeck(new_event) 122 | eid=new_event['Event']['id'] 123 | r = mymisp.add_internal_text(new_event, 1, comment='Emerging Threats revision number') 124 | echeck(r, eid) 125 | load_misp_event(eid) 126 | update_et_event(et_type) 127 | -------------------------------------------------------------------------------- /examples/asciidoc_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import argparse 5 | from datetime import date 6 | import importlib 7 | 8 | from pymisp import MISPEvent 9 | from defang import defang 10 | from pytaxonomies import Taxonomies 11 | 12 | 13 | class ReportGenerator(): 14 | def __init__(self, profile="daily_report"): 15 | self.taxonomies = Taxonomies() 16 | self.report = '' 17 | profile_name = "profiles.{}".format(profile) 18 | self.template = importlib.import_module(name=profile_name) 19 | 20 | def from_remote(self, event_id): 21 | from pymisp import PyMISP 22 | from keys import misp_url, misp_key, misp_verifycert 23 | misp = PyMISP(misp_url, misp_key, misp_verifycert) 24 | result = misp.get(event_id) 25 | self.misp_event = MISPEvent() 26 | self.misp_event.load(result) 27 | 28 | def from_file(self, path): 29 | self.misp_event = MISPEvent() 30 | self.misp_event.load_file(path) 31 | 32 | def attributes(self): 33 | if not self.misp_event.attributes: 34 | return '' 35 | list_attributes = [] 36 | for attribute in self.misp_event.attributes: 37 | if attribute.type in self.template.types_to_attach: 38 | list_attributes.append("* {}".format(defang(attribute.value))) 39 | for obj in self.misp_event.Object: 40 | if obj.name in self.template.objects_to_attach: 41 | for attribute in obj.Attribute: 42 | if attribute.type in self.template.types_to_attach: 43 | list_attributes.append("* {}".format(defang(attribute.value))) 44 | return self.template.attributes.format(list_attributes="\n".join(list_attributes)) 45 | 46 | def _get_tag_info(self, machinetag): 47 | return self.taxonomies.revert_machinetag(machinetag) 48 | 49 | def report_headers(self): 50 | content = {'org_name': 'name', 51 | 'date': date.today().isoformat()} 52 | self.report += self.template.headers.format(**content) 53 | 54 | def event_level_tags(self): 55 | if not self.misp_event.Tag: 56 | return '' 57 | for tag in self.misp_event.Tag: 58 | # Only look for TLP for now 59 | if tag['name'].startswith('tlp'): 60 | tax, predicate = self._get_tag_info(tag['name']) 61 | return self.template.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded) 62 | 63 | def title(self): 64 | internal_id = '' 65 | summary = '' 66 | # Get internal refs for report 67 | for obj in self.misp_event.Object: 68 | if obj.name != 'report': 69 | continue 70 | for a in obj.Attribute: 71 | if a.object_relation == 'case-number': 72 | internal_id = a.value 73 | if a.object_relation == 'summary': 74 | summary = a.value 75 | 76 | return self.template.title.format(internal_id=internal_id, title=self.misp_event.info, 77 | summary=summary) 78 | 79 | def asciidoc(self, lang='en'): 80 | self.report += self.title() 81 | self.report += self.event_level_tags() 82 | self.report += self.attributes() 83 | 84 | 85 | if __name__ == '__main__': 86 | try: 87 | parser = argparse.ArgumentParser(description='Create a human-readable report out of a MISP event') 88 | parser.add_argument("--profile", default="daily_report", help="Profile template to use") 89 | parser.add_argument("-o", "--output", help="Output file to write to (generally ends in .adoc)") 90 | group = parser.add_mutually_exclusive_group(required=True) 91 | group.add_argument("-e", "--event", default=[], nargs='+', help="Event ID to get.") 92 | group.add_argument("-p", "--path", default=[], nargs='+', help="Path to the JSON dump.") 93 | 94 | args = parser.parse_args() 95 | 96 | report = ReportGenerator(args.profile) 97 | report.report_headers() 98 | 99 | if args.event: 100 | for eid in args.event: 101 | report.from_remote(eid) 102 | report.asciidoc() 103 | else: 104 | for f in args.path: 105 | report.from_file(f) 106 | report.asciidoc() 107 | 108 | if args.output: 109 | with open(args.output, "w") as ofile: 110 | ofile.write(report.report) 111 | else: 112 | print(report.report) 113 | except ModuleNotFoundError as err: 114 | print(err) 115 | -------------------------------------------------------------------------------- /examples/feed-generator-from-redis/fromredis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | import json 6 | import argparse 7 | import datetime 8 | import time 9 | import redis 10 | 11 | import settings 12 | 13 | from generator import FeedGenerator 14 | 15 | 16 | def beautyful_sleep(sleep, additional): 17 | length = 20 18 | sleeptime = float(sleep) / float(length) 19 | for i in range(length): 20 | temp_string = '|'*i + ' '*(length-i-1) 21 | print('sleeping [{}]\t{}'.format(temp_string, additional), end='\r', sep='') 22 | sys.stdout.flush() 23 | time.sleep(sleeptime) 24 | 25 | 26 | class RedisToMISPFeed: 27 | SUFFIX_SIGH = '_sighting' 28 | SUFFIX_ATTR = '_attribute' 29 | SUFFIX_OBJ = '_object' 30 | SUFFIX_LIST = [SUFFIX_SIGH, SUFFIX_ATTR, SUFFIX_OBJ] 31 | 32 | def __init__(self): 33 | self.host = settings.host 34 | self.port = settings.port 35 | self.db = settings.db 36 | self.serv = redis.StrictRedis(self.host, self.port, self.db, decode_responses=True) 37 | 38 | self.generator = FeedGenerator() 39 | 40 | self.keynames = [] 41 | for k in settings.keyname_pop: 42 | for s in self.SUFFIX_LIST: 43 | self.keynames.append(k+s) 44 | 45 | self.keynameError = settings.keyname_error 46 | 47 | self.update_last_action("Init system") 48 | 49 | def consume(self): 50 | self.update_last_action("Started consuming redis") 51 | while True: 52 | for key in self.keynames: 53 | while True: 54 | data = self.pop(key) 55 | if data is None: 56 | break 57 | try: 58 | self.perform_action(key, data) 59 | except Exception as error: 60 | self.save_error_to_redis(error, data) 61 | 62 | beautyful_sleep(5, self.format_last_action()) 63 | 64 | def pop(self, key): 65 | popped = self.serv.rpop(key) 66 | if popped is None: 67 | return None 68 | try: 69 | popped = json.loads(popped) 70 | except ValueError as error: 71 | self.save_error_to_redis(error, popped) 72 | except ValueError as error: 73 | self.save_error_to_redis(error, popped) 74 | return popped 75 | 76 | def perform_action(self, key, data): 77 | # sighting 78 | if key.endswith(self.SUFFIX_SIGH): 79 | if self.generator.add_sighting_on_attribute(): 80 | self.update_last_action("Added sighting") 81 | else: 82 | self.update_last_action("Error while adding sighting") 83 | 84 | # attribute 85 | elif key.endswith(self.SUFFIX_ATTR): 86 | attr_type = data.pop('type') 87 | attr_value = data.pop('value') 88 | if self.generator.add_attribute_to_event(attr_type, attr_value, **data): 89 | self.update_last_action("Added attribute") 90 | else: 91 | self.update_last_action("Error while adding attribute") 92 | 93 | # object 94 | elif key.endswith(self.SUFFIX_OBJ): 95 | # create the MISP object 96 | obj_name = data.pop('name') 97 | if self.generator.add_object_to_event(obj_name, **data): 98 | self.update_last_action("Added object") 99 | else: 100 | self.update_last_action("Error while adding object") 101 | 102 | else: 103 | # Suffix not valid 104 | self.update_last_action("Redis key suffix not supported") 105 | 106 | # OTHERS 107 | def update_last_action(self, action): 108 | self.last_action = action 109 | self.last_action_time = datetime.datetime.now() 110 | 111 | def format_last_action(self): 112 | return "Last action: [{}] @ {}".format( 113 | self.last_action, 114 | self.last_action_time.isoformat().replace('T', ' '), 115 | ) 116 | 117 | 118 | def save_error_to_redis(self, error, item): 119 | to_push = {'error': str(error), 'item': str(item)} 120 | print('Error:', str(error), '\nOn adding:', item) 121 | self.serv.lpush(self.keynameError, to_push) 122 | 123 | 124 | if __name__ == '__main__': 125 | parser = argparse.ArgumentParser(description="Pop item fom redis and add " 126 | + "it to the MISP feed. By default, each action are pushed into a " 127 | + "daily named event. Configuration taken from the file settings.py.") 128 | args = parser.parse_args() 129 | 130 | redisToMISP = RedisToMISPFeed() 131 | redisToMISP.consume() 132 | -------------------------------------------------------------------------------- /tests/mispevent_testfiles/shadow.json: -------------------------------------------------------------------------------- 1 | { 2 | "Event": { 3 | "Attribute": [ 4 | { 5 | "ShadowAttribute": [ 6 | { 7 | "Org": { 8 | "id": "1", 9 | "name": "CIRCL", 10 | "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" 11 | }, 12 | "category": "Artifacts dropped", 13 | "comment": "", 14 | "disable_correlation": false, 15 | "event_id": "6676", 16 | "event_uuid": "5a4cb19a-f550-437f-bd29-48ed950d210f", 17 | "id": "3770", 18 | "old_id": "811578", 19 | "org_id": "1", 20 | "proposal_to_delete": false, 21 | "timestamp": "1514975846", 22 | "to_ids": true, 23 | "type": "filename", 24 | "uuid": "5a4cb1c7-fa84-45fa-8d27-4822950d210f", 25 | "value": "blah.exe.jpg" 26 | } 27 | ], 28 | "category": "Artifacts dropped", 29 | "comment": "", 30 | "deleted": false, 31 | "disable_correlation": false, 32 | "distribution": "5", 33 | "event_id": "6676", 34 | "id": "811578", 35 | "object_id": "0", 36 | "sharing_group_id": "0", 37 | "timestamp": "1514975687", 38 | "to_ids": false, 39 | "type": "filename", 40 | "uuid": "5a4cb1c7-fa84-45fa-8d27-4822950d210f", 41 | "value": "blah.exe" 42 | } 43 | ], 44 | "Object": [ 45 | { 46 | "Attribute": [ 47 | { 48 | "ShadowAttribute": [ 49 | { 50 | "Org": { 51 | "id": "1", 52 | "name": "CIRCL", 53 | "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" 54 | }, 55 | "category": "Payload delivery", 56 | "comment": "", 57 | "disable_correlation": false, 58 | "event_id": "6676", 59 | "event_uuid": "5a4cb19a-f550-437f-bd29-48ed950d210f", 60 | "id": "3771", 61 | "old_id": "811579", 62 | "org_id": "1", 63 | "proposal_to_delete": false, 64 | "timestamp": "1514976196", 65 | "to_ids": true, 66 | "type": "filename", 67 | "uuid": "5a4cb2b8-4748-4c72-96e6-4588950d210f", 68 | "value": "baz.png.exe" 69 | } 70 | ], 71 | "category": "Payload delivery", 72 | "comment": "", 73 | "deleted": false, 74 | "disable_correlation": false, 75 | "distribution": "5", 76 | "event_id": "6676", 77 | "id": "811579", 78 | "object_id": "2278", 79 | "object_relation": "filename", 80 | "sharing_group_id": "0", 81 | "timestamp": "1514975928", 82 | "to_ids": true, 83 | "type": "filename", 84 | "uuid": "5a4cb2b8-4748-4c72-96e6-4588950d210f", 85 | "value": "baz.png" 86 | }, 87 | { 88 | "category": "Other", 89 | "comment": "", 90 | "deleted": false, 91 | "disable_correlation": true, 92 | "distribution": "5", 93 | "event_id": "6676", 94 | "id": "811580", 95 | "object_id": "2278", 96 | "object_relation": "state", 97 | "sharing_group_id": "0", 98 | "timestamp": "1514975928", 99 | "to_ids": false, 100 | "type": "text", 101 | "uuid": "5a4cb2b9-92b4-4d3a-82df-4e86950d210f", 102 | "value": "Malicious" 103 | } 104 | ], 105 | "comment": "", 106 | "deleted": false, 107 | "description": "File object describing a file with meta-information", 108 | "distribution": "5", 109 | "event_id": "6676", 110 | "id": "2278", 111 | "meta-category": "file", 112 | "name": "file", 113 | "sharing_group_id": "0", 114 | "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", 115 | "template_version": "7", 116 | "timestamp": "1514975928", 117 | "uuid": "5a4cb2b8-7958-4323-852c-4d2a950d210f" 118 | } 119 | ], 120 | "Org": { 121 | "id": "1", 122 | "name": "CIRCL", 123 | "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" 124 | }, 125 | "Orgc": { 126 | "id": "1", 127 | "name": "CIRCL", 128 | "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" 129 | }, 130 | "analysis": "2", 131 | "attribute_count": "3", 132 | "date": "2018-01-03", 133 | "disable_correlation": false, 134 | "distribution": "0", 135 | "event_creator_email": "raphael.vinot@circl.lu", 136 | "id": "6676", 137 | "info": "Test proposals / ShadowAttributes", 138 | "locked": false, 139 | "org_id": "1", 140 | "orgc_id": "1", 141 | "proposal_email_lock": true, 142 | "publish_timestamp": "0", 143 | "published": false, 144 | "sharing_group_id": "0", 145 | "threat_level_id": "1", 146 | "timestamp": "1514975929", 147 | "uuid": "5a4cb19a-f550-437f-bd29-48ed950d210f" 148 | } 149 | } 150 | --------------------------------------------------------------------------------