├── .gitignore ├── AUTHORS ├── LICENSE ├── README.md ├── minemeld.json ├── mmmisp ├── __init__.py ├── node.py ├── prototypes │ └── misp.yml └── webui │ ├── extension.js │ ├── misp.miner.info.html │ ├── misp.miner.sak.modal.html │ ├── misp.miner.surl.modal.html │ └── misp.miner.uploadcert.modal.html ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | *.whl 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # IPython Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # dotenv 80 | .env 81 | 82 | # virtualenv 83 | venv/ 84 | ENV/ 85 | 86 | # Spyder project settings 87 | .spyderproject 88 | 89 | # Rope project settings 90 | .ropeproject 91 | .vscode 92 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Luigi Mori 2 | Simon Coggins 3 | Akihiro Hosono 4 | 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # minemeld-misp 2 | MineMeld nodes for [MISP](http://www.misp-project.org/) 3 | 4 | ## Requirements 5 | 6 | MineMeld >= 0.9.37b1 7 | 8 | ## Installation 9 | 10 | - in SYSTEM > EXTENSIONS install the extension using *git* button (https://github.com/PaloAltoNetworks/minemeld-misp.git) 11 | - (temporary workaround for a bug in MineMeld) open a shell on MineMeld and restart the API daemon: 12 | 13 | ``$ sudo -u minemeld mm-supervisorctl restart minemeld-web`` 14 | 15 | - refresh your browser 16 | 17 | ## Miner 18 | 19 | To use the Miner you should create a new prototype based on *misp.anyEvent* prototype and add the *url* parameter. 20 | 21 | After COMMIT you will be able to specify the authentication key directly from the WebUI. 22 | 23 | ### Prototype parameters 24 | 25 | ```yaml 26 | # source name, to identify the origin of the indicators inside MineMeld 27 | source_name: misp.test 28 | # URL of MISP 29 | url: https://misp.example.com 30 | # filters for MISP query 31 | # default: none 32 | # this one check for published events with tag tlp:white 33 | # you can specify a time window of the last N days using datefrom: d 34 | # check the search_index API in PyMISP for available filter parameters 35 | filters: 36 | published: 1 37 | tag: 'tlp:white' 38 | # datefrom: 180d 39 | # select specific inidicator types, default: null (any) 40 | # indicator_types: ['URL', 'IPv4', 'IPv6'] 41 | indicator_types: null 42 | # honour IDS flag, if true only events with IDS set will be exported 43 | honour_ids_flag: true 44 | # a dictionary of event attributes to be extracted, the value 45 | # of each in key in the dictionary is a JMESPATH expression 46 | # default: 47 | # event_attributes: 48 | # info: info 49 | # org: Org 50 | # orgc: Orgc 51 | # threat_level_id: threat_level_id 52 | # uuid: uuid 53 | # tags: Tag[*].name 54 | # a dictionary of attribute attributes to be extracted, the value 55 | # of each in key in the dictionary is a JMESPATH expression 56 | # default: 57 | # attribute_attributes: 58 | # uuid: info 59 | # category: Org 60 | # comment: Orgc 61 | # prefix to be applied to indicator attributes, default: misp 62 | # prefix: misp 63 | # verify remote certificate, default true 64 | verify_cert: true 65 | # require a client certificate, default false 66 | client_cert_required: false 67 | # age out of indicators 68 | age_out: 69 | sudden_death: true 70 | default: null 71 | # flag indicators with share level white, if not specified 72 | # by tag 73 | attributes: 74 | confidence: 70 75 | # if not specified in the event, default is white for 76 | # this prototype 77 | share_leve: white 78 | ``` 79 | -------------------------------------------------------------------------------- /minemeld.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.4.96b4", 3 | "name": "minemeld-misp", 4 | "author": "Palo Alto Networks", 5 | "author_email": "techbizdev@paloaltonetworks.com", 6 | "description": "MineMeld extension for MISP", 7 | "entry_points": { 8 | "minemeld_webui": { 9 | "mmmispWebui": "mmmisp:webui_blueprint" 10 | }, 11 | "minemeld_nodes": { 12 | "mmmisp.Miner": "mmmisp.node:Miner" 13 | }, 14 | "minemeld_prototypes": { 15 | "mmmisp.prototypes": "mmmisp:prototypes" 16 | } 17 | 18 | }, 19 | "manifest_version": 0 20 | } 21 | -------------------------------------------------------------------------------- /mmmisp/__init__.py: -------------------------------------------------------------------------------- 1 | def webui_blueprint(): 2 | from minemeld.flask import aaa 3 | 4 | return aaa.MMBlueprint('mmmispWebui', __name__, static_folder='webui', static_url_path='') 5 | 6 | 7 | def prototypes(): 8 | import os 9 | 10 | return os.path.join(os.path.dirname(__file__), 'prototypes') 11 | -------------------------------------------------------------------------------- /mmmisp/node.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import re 4 | import copy 5 | from functools import partial 6 | from itertools import imap 7 | from datetime import datetime 8 | 9 | import yaml 10 | import jmespath 11 | from netaddr import IPNetwork, AddrFormatError 12 | from pymisp import PyMISP 13 | from minemeld.ft.basepoller import BasePollerFT 14 | 15 | LOG = logging.getLogger(__name__) 16 | 17 | 18 | _MISP_TO_MINEMELD = { 19 | 'url': 'URL', 20 | 'domain': 'domain', 21 | 'hostname': 'domain', 22 | 'md5': 'md5', 23 | 'sha256': 'sha256', 24 | 'sha1': 'sha1', 25 | 'sha512': 'sha512', 26 | 'ssdeep': 'ssdeep', 27 | 'mutex': 'mutex', 28 | 'filename': 'file.name' 29 | } 30 | 31 | 32 | class Miner(BasePollerFT): 33 | def __init__(self, name, chassis, config): 34 | self.automation_key = None 35 | self.url = None 36 | self.verify_cert = True 37 | 38 | self.datefrom_re = re.compile('^([0-9]+)d$') 39 | 40 | super(Miner, self).__init__(name, chassis, config) 41 | 42 | def configure(self): 43 | super(Miner, self).configure() 44 | 45 | self.prefix = self.config.get('prefix', 'misp') 46 | self.indicator_types = self.config.get('indicator_types', None) 47 | 48 | self.url = self.config.get('url', None) 49 | self.filters = self.config.get('filters', None) 50 | 51 | # option for enabling client cert, default disabled 52 | self.client_cert_required = self.config.get('client_cert_required', False) 53 | if self.client_cert_required: 54 | self.key_file = os.path.join( 55 | os.environ['MM_CONFIG_DIR'], 56 | '%s.pem' % self.name 57 | ) 58 | self.cert_file = os.path.join( 59 | os.environ['MM_CONFIG_DIR'], 60 | '%s.crt' % self.name 61 | ) 62 | 63 | self.honour_ids_flag = self.config.get('honour_ids_flag', True) 64 | 65 | teventattrs = self.config.get( 66 | 'event_attributes', 67 | dict( 68 | info='info', 69 | org='Org.name', 70 | orgc='Orgc.name', 71 | threat_level_id='threat_level_id', 72 | tags='Tag[*].name', 73 | uuid='uuid' 74 | ) 75 | ) 76 | self.event_attributes = {} 77 | for aname, aexpr in teventattrs.iteritems(): 78 | self.event_attributes[aname] = jmespath.compile(aexpr) 79 | 80 | tattrattributes = self.config.get( 81 | 'attribute_attributes', 82 | dict( 83 | uuid='uuid', 84 | category='category', 85 | comment='comment', 86 | tags='Tag[*].name' 87 | ) 88 | ) 89 | self.attribute_attributes = {} 90 | for aname, aexpr in tattrattributes.iteritems(): 91 | self.attribute_attributes[aname] = jmespath.compile(aexpr) 92 | 93 | self.side_config_path = self.config.get('side_config', None) 94 | if self.side_config_path is None: 95 | self.side_config_path = os.path.join( 96 | os.environ['MM_CONFIG_DIR'], 97 | '%s_side_config.yml' % self.name 98 | ) 99 | 100 | self._load_side_config() 101 | 102 | def _load_side_config(self): 103 | try: 104 | with open(self.side_config_path, 'r') as f: 105 | sconfig = yaml.safe_load(f) 106 | 107 | except Exception as e: 108 | LOG.error('%s - Error loading side config: %s', self.name, str(e)) 109 | return 110 | 111 | self.automation_key = sconfig.get('automation_key', None) 112 | self.verify_cert = sconfig.get('verify_cert', True) 113 | 114 | turl = sconfig.get('url', None) 115 | if turl is not None: 116 | self.url = turl 117 | LOG.info('{} - url set'.format(self.name)) 118 | 119 | def _load_event(self, misp, event): 120 | euuid = event.get('uuid', None) 121 | if euuid is None: 122 | LOG.error('{} - event with no uuid: {!r}'.format(self.name, event)) 123 | return None 124 | 125 | return misp.get(event['uuid']) 126 | 127 | def _build_iterator(self, now): 128 | if self.automation_key is None: 129 | raise RuntimeError('{} - MISP Automation Key not set'.format(self.name)) 130 | 131 | if self.url is None: 132 | raise RuntimeError('{} - MISP URL not set'.format(self.name)) 133 | 134 | kwargs = {'ssl': self.verify_cert} 135 | if self.verify_cert and 'REQUESTS_CA_BUNDLE' in os.environ: 136 | kwargs['ssl'] = os.environ['REQUESTS_CA_BUNDLE'] 137 | 138 | if self.client_cert_required: 139 | kwargs['cert'] = (self.cert_file, self.key_file) 140 | 141 | misp = PyMISP(self.url, self.automation_key, **kwargs) 142 | 143 | filters = None 144 | if self.filters is not None: 145 | filters = self.filters.copy() 146 | if 'datefrom' in filters: 147 | df = filters.pop('datefrom') 148 | 149 | mo = self.datefrom_re.match(df) 150 | if mo is not None: 151 | deltad = int(mo.group(1)) 152 | df = datetime.utcfromtimestamp(now/1000 - 86400 * deltad).strftime('%Y-%m-%d') 153 | 154 | filters['datefrom'] = df 155 | 156 | du = filters.pop('dateuntil', None) 157 | if du is not None: 158 | filters['dateuntil'] = du 159 | LOG.info('{} - query filters: {!r}'.format(self.name, filters)) 160 | 161 | r = misp.get_index(filters) 162 | 163 | events = r['response'] 164 | 165 | return imap(partial(self._load_event, misp), events) 166 | 167 | def _detect_ip_version(self, ip_addr): 168 | try: 169 | parsed = IPNetwork(ip_addr) 170 | except (AddrFormatError, ValueError): 171 | LOG.error('{} - Unknown IP version: {}'.format(self.name, ip_addr)) 172 | return None 173 | 174 | if parsed.version == 4: 175 | return 'IPv4' 176 | 177 | if parsed.version == 6: 178 | return 'IPv6' 179 | 180 | return None 181 | 182 | def _process_item(self, event): 183 | event = event.get('Event', None) 184 | if event is None: 185 | return [] 186 | 187 | result = [] 188 | 189 | base_value = {} 190 | for aname, aexpr in self.event_attributes.iteritems(): 191 | try: 192 | eresult = aexpr.search(event) 193 | except: 194 | continue 195 | 196 | if eresult is None: 197 | continue 198 | 199 | base_value['{}_event_{}'.format(self.prefix, aname)] = eresult 200 | 201 | # check tlp tag 202 | tags = event.get('Tag', []) 203 | for t in tags: 204 | tname = t.get('name', None) 205 | if tname is None: 206 | continue 207 | 208 | if tname.startswith('tlp:'): 209 | base_value['share_level'] = tname[4:] 210 | 211 | attributes = event.get('Attribute', []) 212 | for a in attributes: 213 | if self.honour_ids_flag: 214 | to_ids = a.get('to_ids', False) 215 | if not to_ids: 216 | continue 217 | 218 | indicator = a.get('value', None) 219 | if indicator is None: 220 | LOG.error('{} - attribute with no value: {!r}'.format(self.name, a)) 221 | continue 222 | 223 | iv = {} 224 | 225 | # Populate iv with the attributes from the event. 226 | for aname, aexpr in self.attribute_attributes.iteritems(): 227 | try: 228 | eresult = aexpr.search(a) 229 | except: 230 | continue 231 | 232 | if eresult is None: 233 | continue 234 | 235 | iv['{}_attribute_{}'.format(self.prefix, aname)] = eresult 236 | 237 | iv.update(base_value) 238 | 239 | itype = a.get('type', None) 240 | if itype == 'ip-src': 241 | iv['type'] = self._detect_ip_version(indicator) 242 | iv['direction'] = 'inbound' 243 | elif itype == 'ip-src|port': 244 | indicator, _ = indicator.split('|', 1) 245 | iv['type'] = self._detect_ip_version(indicator) 246 | iv['direction'] = 'inbound' 247 | elif itype == 'ip-dst': 248 | iv['type'] = self._detect_ip_version(indicator) 249 | iv['direction'] = 'outbound' 250 | elif itype == 'ip-dst|port': 251 | indicator, _ = indicator.split('|', 1) 252 | iv['type'] = self._detect_ip_version(indicator) 253 | iv['direction'] = 'outbound' 254 | elif itype[:9] == 'filename|': 255 | indicator, indicator2 = indicator.split('|', 1) 256 | iv['type'] = 'file.name' 257 | 258 | # If we know the 2nd indicator type, clone the iv as it's the same event, and append it it to results 259 | itype2 = _MISP_TO_MINEMELD.get(itype[9:], None) 260 | if itype2 is not None: 261 | iv2 = copy.deepcopy(iv) # Copy IV since it's the same event, just different type 262 | iv2['type'] = itype2 263 | result.append([indicator2, iv2]) # Append our second indicator 264 | 265 | else: 266 | iv['type'] = _MISP_TO_MINEMELD.get(itype, None) 267 | 268 | if iv['type'] is None: 269 | LOG.error('{} - Unhandled indicator type: {!r}'.format(self.name, a)) 270 | continue 271 | 272 | result.append([indicator, iv]) 273 | 274 | if self.indicator_types is not None: 275 | result = [[ti, tiv] for ti, tiv in result if tiv['type'] in self.indicator_types] 276 | 277 | return result 278 | 279 | def hup(self, source=None): 280 | LOG.info('%s - hup received, reload side config', self.name) 281 | self._load_side_config() 282 | super(Miner, self).hup(source) 283 | 284 | @staticmethod 285 | def gc(name, config=None): 286 | BasePollerFT.gc(name, config=config) 287 | 288 | side_config_path = None 289 | if config is not None: 290 | side_config_path = config.get('side_config', None) 291 | if side_config_path is None: 292 | side_config_path = os.path.join( 293 | os.environ['MM_CONFIG_DIR'], 294 | '{}_side_config.yml'.format(name) 295 | ) 296 | 297 | try: 298 | os.remove(side_config_path) 299 | except: 300 | pass 301 | 302 | client_cert_required = False 303 | if config is not None: 304 | client_cert_required = config.get('client_cert_required', False) 305 | 306 | if client_cert_required: 307 | cert_path = os.path.join( 308 | os.environ['MM_CONFIG_DIR'], 309 | '{}.crt'.format(name) 310 | ) 311 | 312 | try: 313 | os.remove(cert_path) 314 | except: 315 | pass 316 | 317 | key_path = os.path.join( 318 | os.environ['MM_CONFIG_DIR'], 319 | '{}.pem'.format(name) 320 | ) 321 | 322 | try: 323 | os.remove(key_path) 324 | except: 325 | pass 326 | -------------------------------------------------------------------------------- /mmmisp/prototypes/misp.yml: -------------------------------------------------------------------------------- 1 | url: https://www.misp-project.org/ 2 | description: > 3 | MineMeld nodes for MISP Threat Intelligence Platform. 4 | 5 | prototypes: 6 | tlpWhiteEvent: 7 | author: Palo Alto Networks TBD 8 | class: mmmisp.Miner 9 | development_status: EXPERIMENTAL 10 | node_type: miner 11 | indicator_types: 12 | - any 13 | tags: 14 | - extension 15 | - misp 16 | description: > 17 | Miner for MISP. This will extract all published events with tlp:white tag. 18 | config: 19 | # source name used in the indicators 20 | source_name: misp.tlpWhiteEvent 21 | # filters for MISP query 22 | # default: none 23 | # this one check for published events with tag tlp:white 24 | filters: 25 | published: 1 26 | tag: 'tlp:white' 27 | datefrom: 180d 28 | # datefrom: YYYY-MM-DD 29 | # select specific inidicator types, default: null (any) 30 | indicator_types: null 31 | # verify remote certificate, default true 32 | verify_cert: true 33 | # age out of indicators 34 | # disabled, removed when they disappear from the channel 35 | age_out: 36 | sudden_death: true 37 | default: null 38 | # flag indicators with share level white, if not specified 39 | # by tag 40 | attributes: 41 | confidence: 70 42 | # if not specified in the event, default is white for 43 | # this prototype 44 | share_leve: white 45 | anyEvent: 46 | author: Palo Alto Networks TBD 47 | class: mmmisp.Miner 48 | development_status: EXPERIMENTAL 49 | node_type: miner 50 | indicator_types: 51 | - any 52 | tags: 53 | - extension 54 | - misp 55 | description: > 56 | Miner for MISP. This will extract any event and mark them 57 | as share level red 58 | config: 59 | # source name used in the indicators 60 | source_name: misp.anyEvent 61 | filters: null 62 | indicator_types: null 63 | verify_cert: true 64 | age_out: 65 | sudden_death: true 66 | default: null 67 | attributes: 68 | confidence: 70 69 | share_leve: red 70 | -------------------------------------------------------------------------------- /mmmisp/webui/extension.js: -------------------------------------------------------------------------------- 1 | console.log('Loading mmmisp WebUI'); 2 | 3 | (function() { 4 | 5 | function MISPSideConfigController($scope, MinemeldConfigService, MineMeldRunningConfigStatusService, 6 | toastr, $modal, ConfirmService, $timeout) { 7 | var vm = this; 8 | 9 | // side config settings 10 | vm.verify_cert = undefined; 11 | vm.automation_key = undefined; 12 | vm.url = undefined; 13 | 14 | vm.clientCertSet = undefined; 15 | vm.clientCertEnabled = undefined; 16 | 17 | vm.loadSideConfig = function() { 18 | var nodename = $scope.$parent.vm.nodename; 19 | 20 | MinemeldConfigService.getDataFile(nodename + '_side_config') 21 | .then((result) => { 22 | if (!result) { 23 | return; 24 | } 25 | 26 | if (result.automation_key) { 27 | vm.automation_key = result.automation_key; 28 | } else { 29 | vm.automation_key = undefined; 30 | } 31 | 32 | if (typeof result.verify_cert !== 'undefined') { 33 | vm.verify_cert = result.verify_cert; 34 | } else { 35 | vm.verify_cert = undefined; 36 | } 37 | 38 | if (result.url) { 39 | vm.url = result.url; 40 | } else { 41 | vm.url = undefined; 42 | } 43 | }, (error) => { 44 | toastr.error('ERROR RETRIEVING NODE SIDE CONFIG: ' + error.status); 45 | vm.automation_key = undefined; 46 | vm.verify_cert = undefined; 47 | }) 48 | .finally(vm.checkClientCertificate); 49 | }; 50 | 51 | vm.saveSideConfig = function() { 52 | var side_config = {}; 53 | var hup_node = undefined; 54 | var nodename = $scope.$parent.vm.nodename; 55 | 56 | if (vm.automation_key) { 57 | side_config.automation_key = vm.automation_key; 58 | } 59 | 60 | if (vm.url) { 61 | side_config.url = vm.url; 62 | } 63 | 64 | if (typeof vm.verify_cert !== 'undefined') { 65 | side_config.verify_cert = vm.verify_cert; 66 | } 67 | 68 | return MinemeldConfigService.saveDataFile( 69 | nodename + '_side_config', 70 | side_config 71 | ); 72 | }; 73 | 74 | vm.checkClientCertificate = function() { 75 | var client_cert_required = false; 76 | var nodename = $scope.$parent.vm.nodename; 77 | 78 | MineMeldRunningConfigStatusService.getStatus().then((result) => { 79 | var node = result.nodes[nodename]; 80 | 81 | vm.clientCertSet = false; 82 | vm.clientCertEnabled = false; 83 | 84 | if (node.resolvedPrototype && node.resolvedPrototype.config) { 85 | if (typeof node.resolvedPrototype.config.client_cert_required !== 'undefined') { 86 | client_cert_required = node.resolvedPrototype.config.client_cert_required; 87 | } 88 | } 89 | 90 | if (node.node.config) { 91 | if (typeof node.node.config.client_cert_required !== 'undefined') { 92 | client_cert_required = node.node.config.client_cert_required; 93 | } 94 | } 95 | 96 | if (!client_cert_required) { 97 | return; 98 | } 99 | 100 | vm.clientCertEnabled = true; 101 | 102 | MinemeldConfigService.getDataFile(nodename, 'cert').then((result) => { 103 | if (result == null) { 104 | vm.clientCertSet = false; 105 | return; 106 | } 107 | 108 | MinemeldConfigService.getDataFile(nodename, 'pkey').then((result) => { 109 | if (result == null) { 110 | vm.clientCertSet = false; 111 | return; 112 | } 113 | 114 | vm.clientCertSet = true; 115 | }, (error) => { 116 | vm.clientCertSet = false; 117 | }); 118 | }, (error) => { 119 | vm.clientCertSet = false; 120 | }); 121 | }); 122 | }; 123 | 124 | vm.setAutomationKey = function() { 125 | var mi = $modal.open({ 126 | templateUrl: '/extensions/webui/mmmispWebui/misp.miner.sak.modal.html', 127 | controller: ['$modalInstance', MISPAutomationKeyController], 128 | controllerAs: 'vm', 129 | bindToController: true, 130 | backdrop: 'static', 131 | animation: false 132 | }); 133 | 134 | mi.result.then((result) => { 135 | vm.automation_key = result.automation_key; 136 | 137 | return vm.saveSideConfig(); 138 | }) 139 | .then((result) => { 140 | toastr.success('AUTOMATION KEY SET'); 141 | vm.loadSideConfig(); 142 | }, (error) => { 143 | toastr.error('ERROR SETTING AUTOMATION KEY: ' + error.statusText); 144 | }); 145 | }; 146 | 147 | vm.setURL = function() { 148 | var mi = $modal.open({ 149 | templateUrl: '/extensions/webui/mmmispWebui/misp.miner.surl.modal.html', 150 | controller: ['$modalInstance', MISPURLController], 151 | controllerAs: 'vm', 152 | bindToController: true, 153 | backdrop: 'static', 154 | animation: false 155 | }); 156 | 157 | mi.result.then((result) => { 158 | vm.url = result.url; 159 | 160 | return vm.saveSideConfig(); 161 | }) 162 | .then(() => { 163 | toastr.success('URL SET'); 164 | vm.loadSideConfig(); 165 | }, (error) => { 166 | toastr.error('ERROR SETTING URL: ' + error.statusText); 167 | }); 168 | }; 169 | 170 | vm.toggleCertificateVerification = function() { 171 | var p, new_value; 172 | 173 | if (typeof this.verify_cert === 'undefined' || this.verify_cert) { 174 | new_value = false; 175 | p = ConfirmService.show( 176 | 'MISP CERT VERIFICATION', 177 | 'Are you sure you want to disable certificate verification ?' 178 | ); 179 | } else { 180 | new_value = true; 181 | p = ConfirmService.show( 182 | 'MISP CERT VERIFICATION', 183 | 'Are you sure you want to enable certificate verification ?' 184 | ); 185 | } 186 | 187 | p.then((result) => { 188 | vm.verify_cert = new_value; 189 | 190 | return vm.saveSideConfig().then((result) => { 191 | toastr.success('CERT VERIFICATION TOGGLED'); 192 | vm.loadSideConfig(); 193 | }, (error) => { 194 | toastr.error('ERROR TOGGLING CERT VERIFICATION: ' + error.statusText); 195 | }); 196 | }); 197 | }; 198 | 199 | vm.uploadClientCertificate = function() { 200 | var mi; 201 | 202 | mi = $modal.open({ 203 | templateUrl: '/extensions/webui/mmmispWebui/misp.miner.uploadcert.modal.html', 204 | controller: ['$modalInstance', 'FileUploader', 'toastr', 'nodeName', MISPUploadClientCertController], 205 | controllerAs: 'vm', 206 | bindToController: true, 207 | backdrop: 'static', 208 | animation: false, 209 | resolve: { 210 | nodeName: () => { return $scope.$parent.vm.nodename; } 211 | } 212 | }); 213 | 214 | mi.result.then((result) => { 215 | vm.loadSideConfig(); 216 | }); 217 | } 218 | 219 | vm.loadSideConfig(); 220 | } 221 | 222 | function MISPAutomationKeyController($modalInstance) { 223 | var vm = this; 224 | 225 | vm.automation_key = undefined; 226 | vm.automation_key2 = undefined; 227 | 228 | vm.valid = function() { 229 | if (vm.automation_key !== vm.automation_key2) { 230 | angular.element('#fgPassword1').addClass('has-error'); 231 | angular.element('#fgPassword2').addClass('has-error'); 232 | 233 | return false; 234 | } 235 | angular.element('#fgPassword1').removeClass('has-error'); 236 | angular.element('#fgPassword2').removeClass('has-error'); 237 | 238 | if (!vm.automation_key) { 239 | return false; 240 | } 241 | 242 | return true; 243 | }; 244 | 245 | vm.save = function() { 246 | var result = {}; 247 | 248 | result.automation_key = vm.automation_key; 249 | 250 | $modalInstance.close(result); 251 | } 252 | 253 | vm.cancel = function() { 254 | $modalInstance.dismiss(); 255 | } 256 | } 257 | 258 | function MISPURLController($modalInstance, url) { 259 | var vm = this; 260 | 261 | vm.url = url; 262 | 263 | vm.valid = function() { 264 | angular.element('#url').removeClass('has-error'); 265 | 266 | if (!vm.url) { 267 | return false; 268 | } 269 | 270 | return true; 271 | }; 272 | 273 | vm.save = function() { 274 | var result = {}; 275 | 276 | result.url = vm.url; 277 | 278 | $modalInstance.close(result); 279 | } 280 | 281 | vm.cancel = function() { 282 | $modalInstance.dismiss(); 283 | } 284 | } 285 | 286 | function MISPUploadClientCertController($modalInstance, FileUploader, toastr, nodeName) { 287 | var vm = this; 288 | 289 | vm.uploading = false; 290 | vm.certUploader = undefined; 291 | vm.keyUploader = undefined; 292 | 293 | vm.uploadAll = function() { 294 | vm.uploading = true; 295 | vm.certUploader.uploadAll(); 296 | } 297 | 298 | vm.cancel = function() { 299 | $modalInstance.dismiss('cancel'); 300 | } 301 | 302 | vm.onErrorItem = function(item, response, status) { 303 | vm.uploading = false; 304 | 305 | if (status === 400) { 306 | toastr.error('ERROR UPLOADING: ' + response.error.message); 307 | return; 308 | } 309 | 310 | toastr.error('ERROR UPLOADING: ' + status); 311 | }; 312 | 313 | vm.certUploader = new FileUploader({ 314 | url: '/config/data/' + nodeName + '?t=cert', 315 | method: 'PUT', 316 | queueLimit: 1, 317 | removeAfterUpload: true 318 | }); 319 | vm.keyUploader = new FileUploader({ 320 | url: '/config/data/' + nodeName + '?t=pkey', 321 | method: 'PUT', 322 | queueLimit: 1, 323 | removeAfterUpload: true 324 | }); 325 | vm.certUploader.onErrorItem = this.onErrorItem; 326 | vm.keyUploader.onErrorItem = this.onErrorItem; 327 | vm.certUploader.onSuccessItem = (item) => { 328 | vm.keyUploader.uploadAll(); 329 | }; 330 | vm.keyUploader.onSuccessItem = (item) => { 331 | vm.uploading = false; 332 | toastr.success('CLIENT CERT SET'); 333 | $modalInstance.close('ok'); 334 | }; 335 | } 336 | 337 | angular.module('mmmispWebui', []) 338 | .controller('MISPSideConfigController', [ 339 | '$scope', 'MinemeldConfigService', 'MineMeldRunningConfigStatusService', 340 | 'toastr', '$modal', 'ConfirmService', '$timeout', 341 | MISPSideConfigController 342 | ]) 343 | .config(['$stateProvider', function($stateProvider) { 344 | $stateProvider.state('nodedetail.mispinfo', { 345 | templateUrl: '/extensions/webui/mmmispWebui/misp.miner.info.html', 346 | controller: 'NodeDetailInfoController', 347 | controllerAs: 'vm' 348 | }); 349 | }]) 350 | .run(['NodeDetailResolver', '$state', function(NodeDetailResolver, $state) { 351 | NodeDetailResolver.registerClass('mmmisp.node.Miner', { 352 | tabs: [{ 353 | icon: 'fa fa-circle-o', 354 | tooltip: 'INFO', 355 | state: 'nodedetail.mispinfo', 356 | active: false 357 | }, 358 | { 359 | icon: 'fa fa-area-chart', 360 | tooltip: 'STATS', 361 | state: 'nodedetail.stats', 362 | active: false 363 | }, 364 | { 365 | icon: 'fa fa-asterisk', 366 | tooltip: 'GRAPH', 367 | state: 'nodedetail.graph', 368 | active: false 369 | }] 370 | }); 371 | 372 | // if a nodedetail is already shown, reload the current state to apply changes 373 | // we should definitely find a better way to handle this... 374 | if ($state.$current.toString().startsWith('nodedetail.')) { 375 | $state.reload(); 376 | } 377 | }]); 378 | })(); 379 | -------------------------------------------------------------------------------- /mmmisp/webui/misp.miner.info.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
STATUS
4 |
5 |
6 |
7 |
8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 24 | 25 | 26 | 27 | 31 | 32 | 33 | 34 | 49 | 50 | 51 | 52 | 53 | 54 | 55 |
CLASS{{ vm.nodeState.class }}
PROTOTYPE{{ vm.nodeConfig.prototype }}
STATE 28 | {{ vm.nodeState.stateAsString }} 29 | {{ vm.nodeState.stateAsString }} 30 |
LAST RUN 35 | 36 | 39 | 43 | 48 |
# INDICATORS{{ vm.nodeState.indicators }}
56 |
57 |
58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 70 | 71 | 72 | 73 | 79 | 80 | 81 |
OUTPUT 67 | ENABLED 68 | DISABLED 69 |
INPUTS 74 | 77 | none 78 |
82 |
83 |
84 |
85 |
86 |
SETTINGS
87 |
88 |
89 |
90 |
91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 103 | 104 | 105 | 106 | 110 | 111 | 112 | 113 | 118 | 119 | 120 | 121 | 125 | 126 | 127 |
KEY 100 | Not set 101 | 102 |
URL 107 | Not set 108 | {{ sideConfig.url }} 109 |
VERIFY CERT 114 | Not set (default enabled) 115 | 116 | 117 |
CLIENT CERTIFICATE 122 | Not set 123 | 124 |
128 |
129 |
130 |
131 |
132 |
CONFIG
133 | 134 |
135 |
136 | -------------------------------------------------------------------------------- /mmmisp/webui/misp.miner.sak.modal.html: -------------------------------------------------------------------------------- 1 |
2 |

MISP KEY

3 |
4 |
5 |
6 |
7 |
8 | 9 |
10 | 11 |
12 |
13 |
14 | 15 |
16 | 17 |
18 |
19 |
20 |
21 |
22 |
23 | 24 | 25 |
26 | -------------------------------------------------------------------------------- /mmmisp/webui/misp.miner.surl.modal.html: -------------------------------------------------------------------------------- 1 |
2 |

MISP URL

3 |
4 |
5 |
6 |
7 |
8 | 9 |
10 | 11 |
12 |
13 |
14 |
15 |
16 |
17 | 18 | 19 |
-------------------------------------------------------------------------------- /mmmisp/webui/misp.miner.uploadcert.modal.html: -------------------------------------------------------------------------------- 1 |
2 |

MISP CLIENT CERTIFICATE

3 |
4 |
5 |
6 |
7 |
8 | 9 |
10 | 11 |
12 |
13 |
14 | 15 |
16 | 17 |
18 |
19 |
20 |
21 |
22 |
23 | 26 | 27 |
-------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | minemeld-core>=0.9.37b1 2 | pymisp==2.4.96 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import Extension, setup, find_packages 2 | 3 | import sys 4 | import json 5 | import os.path 6 | sys.path.insert(0, os.path.abspath('.')) 7 | 8 | with open('requirements.txt') as f: 9 | _requirements = f.read().splitlines() 10 | 11 | with open('minemeld.json') as f: 12 | _metadata = json.load(f) 13 | 14 | _entry_points={} 15 | if 'entry_points' in _metadata: 16 | for epgroup, epoints in _metadata['entry_points'].iteritems(): 17 | _entry_points[epgroup] = ['{} = {}'.format(k, v) for k, v in epoints.iteritems()] 18 | 19 | setup( 20 | name=_metadata['name'], 21 | version=_metadata['version'], 22 | author=_metadata['author'], 23 | author_email=_metadata['author_email'], 24 | description=_metadata['description'], 25 | classifiers=[ 26 | 'Framework :: MineMeld', 27 | 'Development Status :: 3 - Alpha', 28 | 'License :: OSI Approved :: Apache Software License', 29 | 'Programming Language :: Python :: 2.7', 30 | 'Topic :: Security', 31 | 'Topic :: Internet' 32 | ], 33 | packages=find_packages(), 34 | provides=find_packages(), 35 | install_requires=_requirements, 36 | package_data = { 37 | '': ['prototypes/*.yml'] 38 | }, 39 | entry_points=_entry_points 40 | ) 41 | --------------------------------------------------------------------------------