├── .gitignore ├── LICENSE ├── README.md ├── requirements.txt ├── setup.py └── src └── sublime ├── __init__.py ├── __version__.py ├── api.py ├── cli ├── __init__.py ├── decorator.py ├── formatter.py ├── outlookmsgfile_helper.py ├── subcommand.py └── templates │ ├── analyze.txt.j2 │ ├── analyze_multi.txt.j2 │ ├── feedback_result.txt.j2 │ ├── macros.txt.j2 │ ├── me_result.txt.j2 │ └── message_data_model.txt.j2 ├── error.py └── util.py /.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | *.swp 3 | __pycache__ 4 | build/ 5 | dist/ 6 | sublime_cli.egg-info/ 7 | *.mdm 8 | *.eml 9 | *.txt 10 | *.yml 11 | *.pql 12 | .DS_Store 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 Sublime Security 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sublime CLI 2 | 3 | ![MIT license](https://img.shields.io/badge/License-MIT-blue.svg) ![Python version](https://img.shields.io/badge/python-3.7+-blue.svg) ![PyPI version](https://badge.fury.io/py/sublime-cli.svg) [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fsublime-security%2Fsublime-cli.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fsublime-security%2Fsublime-cli?ref=badge_shield) 4 | 5 | The Sublime CLI lets you interact with the Sublime Analysis API right from the terminal. The CLI can also be used as a Python module for programmatic implementations. 6 | 7 | **With the CLI/Python module, you can:** 8 | 9 | - Analyze and query raw messages (EMLs and MSGs), MBOX files, and [Message Data Models (MDMs)](https://docs.sublimesecurity.com/docs/mdm) 10 | - Write your own detection rules and queries 11 | - Detect many different types of phishing attacks like executive impersonation and lookalike domains 12 | - Triage reported phish 13 | 14 | ## Installation 15 | 16 | Sublime CLI is available for Windows, macOS, and Linux. Refer to the [Quickstart](https://docs.sublimesecurity.com/reference/analysis-api-quickstart) documentation for installation instructions. 17 | 18 | ## Documentation 19 | 20 | For a full reference, see the [CLI reference docs](https://docs.sublimesecurity.com/reference/analysis-api-cli). 21 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ansimarkup==1.5.0 2 | cachetools==4.2.1 3 | certifi==2020.12.5 4 | chardet==4.0.0 5 | click==7.1.2 6 | click-default-group==1.2.2 7 | click-repl==0.1.6 8 | colorama==0.4.4 9 | compoundfiles==0.3 10 | compressed-rtf==1.0.6 11 | gron==1.1.4 12 | halo==0.0.31 13 | idna==2.10 14 | Jinja2==2.11.3 15 | log-symbols==0.0.14 16 | MarkupSafe==1.1.1 17 | more-itertools==8.6.0 18 | msg-parser==1.2.0 19 | olefile==0.46 20 | prompt-toolkit==3.0.14 21 | PyYAML==5.4.1 22 | requests==2.25.1 23 | six==1.15.0 24 | spinners==0.0.24 25 | structlog==20.2.0 26 | -e git+git@github.com:sublime-security/sublime-cli.git@236675f8775caaf5af2f802cbb974f7754df60dd#egg=sublime 27 | termcolor==1.1.0 28 | urllib3==1.26.3 29 | wcwidth==0.2.5 30 | websockets==8.1 31 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Sublime API client package.""" 3 | import os 4 | 5 | from setuptools import find_packages, setup 6 | 7 | 8 | def read(fname): 9 | """Read file and return its contents.""" 10 | with open(os.path.join(os.path.dirname(__file__), fname)) as input_file: 11 | return input_file.read() 12 | 13 | 14 | INSTALL_REQUIRES = [ 15 | "Click>=7.0", 16 | "ansimarkup", 17 | "cachetools", 18 | "click-default-group", 19 | "click-repl", 20 | "compoundfiles", 21 | "compressed-rtf", 22 | "gron", 23 | "halo", 24 | "jinja2", 25 | "more-itertools", 26 | "msg_parser", 27 | "olefile", 28 | "pyyaml", 29 | "requests", 30 | "six", 31 | "structlog", 32 | "websockets" 33 | ] 34 | 35 | setup( 36 | name="sublime-cli", 37 | version="0.0.32", 38 | description="Abstraction to interact with the Sublime API.", 39 | url="https://sublimesecurity.com/", 40 | author="Sublime Security", 41 | author_email="hello@sublimesecurity.com", 42 | license="MIT", 43 | package_dir={"": "src"}, 44 | packages=find_packages(where="src"), 45 | package_data={"sublime.cli": ["templates/*.j2", "subcommand_groups/*.py"]}, 46 | install_requires=INSTALL_REQUIRES, 47 | long_description=read("README.md") + "\n\n", 48 | classifiers=[ 49 | "Development Status :: 4 - Beta", 50 | "Intended Audience :: Developers", 51 | "License :: OSI Approved :: MIT License", 52 | "Natural Language :: English", 53 | "Programming Language :: Python", 54 | "Programming Language :: Python :: 3.7", 55 | "Topic :: Software Development :: Libraries", 56 | ], 57 | entry_points={"console_scripts": ["sublime = sublime.cli:main"]}, 58 | zip_safe=False, 59 | keywords=["security", "phishing", "analysts", "soc", 60 | "threat intelligence", "security-automation", "email security"], 61 | download_url="https://github.com/sublime-security/sublime-cli", 62 | ) 63 | -------------------------------------------------------------------------------- /src/sublime/__init__.py: -------------------------------------------------------------------------------- 1 | """Sublime CLI.""" 2 | 3 | from sublime.__version__ import ( # noqa 4 | __author__, 5 | __copyright__, 6 | __credits__, 7 | __email__, 8 | __license__, 9 | __maintainer__, 10 | __status__, 11 | __version__, 12 | ) 13 | from sublime.api import Sublime # noqa 14 | -------------------------------------------------------------------------------- /src/sublime/__version__.py: -------------------------------------------------------------------------------- 1 | __author__ = "Sublime Security" 2 | __copyright__ = "Copyright, Sublime Security, Inc." 3 | __credits__ = ["Sublime Security"] 4 | __license__ = "MIT" 5 | __maintainer__ = "Sublime Security" 6 | __email__ = "hi@sublimesecurity.com" 7 | __status__ = "BETA" 8 | __version__ = "0.0.32" 9 | -------------------------------------------------------------------------------- /src/sublime/api.py: -------------------------------------------------------------------------------- 1 | """Sublime API client.""" 2 | 3 | import os 4 | import json 5 | import time 6 | import datetime 7 | 8 | import requests 9 | import structlog 10 | 11 | from sublime.__version__ import __version__ 12 | from sublime.error import RateLimitError, InvalidRequestError, APIError, AuthenticationError 13 | from sublime.util import load_config 14 | 15 | LOGGER = structlog.get_logger() 16 | 17 | 18 | class Sublime(object): 19 | """ 20 | Sublime API client. 21 | 22 | :param api_key: Key used to access the API. 23 | :type api_key: str 24 | 25 | """ 26 | 27 | _NAME = "Sublime" 28 | _BASE_URL = os.environ.get('BASE_URL') 29 | _BASE_URL = _BASE_URL if _BASE_URL else "https://analyzer.sublime.security" 30 | _API_VERSION = "v1" 31 | _EP_ME = "me" 32 | _EP_FEEDBACK = "feedback" 33 | _EP_MESSAGES_CREATE = "messages/create" 34 | _EP_MESSAGES_ANALYZE = "messages/analyze" 35 | _EP_PRIVACY_ACCEPT = "privacy/accept" 36 | _EP_PRIVACY_DECLINE = "privacy/decline" 37 | _EP_NOT_IMPLEMENTED = "request/{subcommand}" 38 | 39 | # NOTE: since there are two api versions, you must add logic to 40 | # _is_public_endpoint if you add a public v0 path 41 | _API_VERSION_PUBLIC = "v0" 42 | _EP_PUBLIC_BINEXPLODE_SCAN = "binexplode/scan" 43 | _EP_PUBLIC_BINEXPLODE_SCAN_RESULT = "binexplode/scan/{id}" 44 | _EP_PUBLIC_TASK_STATUS = "tasks/{id}" 45 | 46 | def __init__(self, api_key=None): 47 | if api_key is None: 48 | config = load_config() 49 | api_key = config.get("api_key") 50 | self._api_key = api_key 51 | self.session = requests.Session() 52 | 53 | def _is_public_endpoint(self, endpoint): 54 | if endpoint in [self._EP_PUBLIC_BINEXPLODE_SCAN, self._EP_MESSAGES_ANALYZE, self._EP_MESSAGES_CREATE]: 55 | return True 56 | if endpoint.startswith("binexplode") or endpoint.startswith("tasks/"): 57 | return True 58 | 59 | return False 60 | 61 | def _request(self, endpoint, request_type='GET', params=None, json=None): 62 | """Handle the requesting of information from the API. 63 | 64 | :param endpoint: Endpoint to send the request to. 65 | :type endpoint: str 66 | :param params: Request parameters. 67 | :type param: dict 68 | :param json: Request's JSON payload. 69 | :type json: dict 70 | :returns: Response's JSON payload 71 | :rtype: dict 72 | :raises InvalidRequestError: when HTTP status code is 400 or 404 73 | :raises RateLimitError: when HTTP status code is 429 74 | :raises APIError: for all other 4xx or 5xx status codes 75 | 76 | """ 77 | if params is None: 78 | params = {} 79 | headers = { 80 | "User-Agent": "sublime-cli/{}".format(__version__) 81 | } 82 | if self._api_key: 83 | headers["Key"] = self._api_key 84 | 85 | is_public = self._is_public_endpoint(endpoint) 86 | api_version = self._API_VERSION_PUBLIC if is_public else self._API_VERSION 87 | 88 | url = "/".join([self._BASE_URL, api_version, endpoint]) 89 | 90 | # LOGGER.debug("Sending API request...", url=url, params=params, json=json) 91 | 92 | if request_type == 'GET': 93 | response = self.session.get( 94 | url, headers=headers, params=params, json=json 95 | ) 96 | elif request_type == 'POST': 97 | response = self.session.post( 98 | url, headers=headers, json=json 99 | ) 100 | elif request_type == 'PATCH': 101 | response = self.session.patch( 102 | url, headers=headers, json=json 103 | ) 104 | elif request_type == 'DELETE': 105 | response = self.session.delete( 106 | url, headers=headers, params=params 107 | ) 108 | else: 109 | raise NotImplementedError("Method {} is not implemented", request_type) 110 | 111 | if "application/json" in response.headers.get("Content-Type", ""): 112 | # 204 has no content and will trigger an exception 113 | if response.status_code != 204: 114 | body = response.json() 115 | else: 116 | body = None 117 | else: 118 | body = response.text 119 | 120 | if response.status_code >= 400: 121 | self._handle_error_response(response, body) 122 | 123 | return body, response.headers 124 | 125 | def _handle_error_response(self, resp, resp_body): 126 | try: 127 | error_data = resp_body["error"] 128 | message = error_data["message"] 129 | except: 130 | raise APIError( 131 | "Invalid response from API: %r (HTTP response code " 132 | "was %d)" % (resp_body, resp.status_code), 133 | status_code=resp.status_code, 134 | headers=resp.headers) 135 | 136 | if resp.status_code in [400, 404]: 137 | err = InvalidRequestError( 138 | message=message, 139 | status_code=resp.status_code, 140 | headers=resp.headers) 141 | elif resp.status_code == 401: 142 | err = AuthenticationError( 143 | message=message, 144 | status_code=resp.status_code, 145 | headers=resp.headers) 146 | elif resp.status_code == 429: 147 | err = RateLimitError( 148 | message=message, 149 | status_code=resp.status_code, 150 | headers=resp.headers) 151 | else: 152 | err = APIError( 153 | message=message, 154 | status_code=resp.status_code, 155 | headers=resp.headers) 156 | 157 | raise err 158 | 159 | def me(self): 160 | """Get information about the currently authenticated Sublime user.""" 161 | 162 | endpoint = self._EP_ME 163 | response, _ = self._request(endpoint, request_type='GET') 164 | return response 165 | 166 | def create_message(self, raw_message, mailbox_email_address=None, message_type=None): 167 | """Create a Message Data Model from a raw message. 168 | 169 | :param raw_message: Base64 encoded raw message 170 | :type raw_message: str 171 | :param mailbox_email_address: Email address of the mailbox 172 | :type mailbox_email_address: str 173 | :param message_type: The type of message from the perspective of your organization (inbound, internal, outbound) 174 | :type message_type: str 175 | :rtype: dict 176 | 177 | """ 178 | 179 | # LOGGER.debug("Creating a message data model...") 180 | 181 | body = {} 182 | body["raw_message"] = raw_message 183 | 184 | if mailbox_email_address: 185 | body["mailbox_email_address"] = mailbox_email_address 186 | if message_type: 187 | if message_type == "inbound": 188 | body["message_type"] = {"inbound": True} 189 | elif message_type == "internal": 190 | body["message_type"] = {"internal": True} 191 | elif message_type == "outbound": 192 | body["message_type"] = {"outbound": True} 193 | else: 194 | raise Exception("Unsupported message_type") 195 | 196 | endpoint = self._EP_MESSAGES_CREATE 197 | response, _ = self._request(endpoint, request_type='POST', json=body) 198 | return response 199 | 200 | def analyze_message(self, raw_message, rules, queries, run_all_detection_rules=False, run_active_detection_rules=False, run_all_insights=False): 201 | """Analyze a Message Data Model against a list of rules or queries. 202 | 203 | :param raw_message: Base64 encoded raw message 204 | :type raw_message: str 205 | :param rules: Rules to run 206 | :type rules: list 207 | :param queries: Queries to run 208 | :type queries: list 209 | :rtype: dict 210 | :param run_all_detection_rules: whether to run all detection rules against the given message 211 | :type run_all_detection_rules: bool 212 | :param run_active_detection_rules: whether to run active detection rules against the given message 213 | :type run_active_detection_rules: bool 214 | :param run_all_insights: whether to run all insight queries against the given message 215 | :type run_all_insights: bool 216 | 217 | """ 218 | 219 | # LOGGER.debug("Analyzing message data model...") 220 | 221 | body = { 222 | "raw_message": raw_message, 223 | "rules": rules, 224 | "queries": queries, 225 | "run_all_detection_rules": run_all_detection_rules, 226 | "run_active_detection_rules": run_active_detection_rules, 227 | "run_all_insights": run_all_insights, 228 | } 229 | 230 | endpoint = self._EP_MESSAGES_ANALYZE 231 | response, _ = self._request(endpoint, request_type='POST', json=body) 232 | return response 233 | 234 | def poll_task_status(self, task_id): 235 | while True: 236 | endpoint = self._EP_PUBLIC_TASK_STATUS.format(id=task_id) 237 | response, _ = self._request(endpoint, request_type='GET') 238 | if response.get("state"): 239 | if response["state"] in ("pending", "started", "retrying"): 240 | time.sleep(1) 241 | continue 242 | else: 243 | # state in ("succeeded", "failed") 244 | break 245 | 246 | return response 247 | 248 | def binexplode_scan(self, file_contents, file_name): 249 | """Scan a binary using binexplode. 250 | 251 | :param file_contents: Base64 encoded file contents 252 | :type file_contents: str 253 | :param file_name: File name 254 | :type file_name: str 255 | :rtype: dict 256 | 257 | """ 258 | 259 | # LOGGER.debug("Scanning binary using binexplode...") 260 | 261 | body = {"file_contents": file_contents, "file_name": file_name} 262 | 263 | endpoint = self._EP_PUBLIC_BINEXPLODE_SCAN 264 | response, _ = self._request(endpoint, request_type='POST', json=body) 265 | task_id = response.get('task_id') 266 | if task_id: 267 | response = self.poll_task_status(task_id) 268 | if response.get("state") == "succeeded": 269 | endpoint = self._EP_PUBLIC_BINEXPLODE_SCAN_RESULT.format(id=task_id) 270 | response, _ = self._request(endpoint, request_type='GET') 271 | 272 | return response 273 | 274 | def feedback(self, feedback): 275 | """Send feedback directly to the Sublime team. 276 | 277 | :param feedback: Feedback 278 | :type feedback: str 279 | :rtype: dict 280 | 281 | """ 282 | 283 | # LOGGER.debug("Sending feedback...") 284 | 285 | body = {"feedback": feedback} 286 | 287 | endpoint = self._EP_FEEDBACK 288 | response, _ = self._request(endpoint, request_type='POST', json=body) 289 | return response 290 | 291 | def privacy_ack(self, accept): 292 | """Sends privacy acknowledgement to the Sublime server.""" 293 | if accept: 294 | endpoint = self._EP_PRIVACY_ACCEPT 295 | else: 296 | endpoint = self._EP_PRIVACY_DECLINE 297 | 298 | response, _ = self._request(endpoint, request_type='POST') 299 | return response 300 | 301 | def _not_implemented(self, subcommand_name): 302 | """Send request for a not implemented CLI subcommand. 303 | 304 | :param subcommand_name: Name of the CLI subcommand 305 | :type subcommand_name: str 306 | 307 | """ 308 | endpoint = self._EP_NOT_IMPLEMENTED.format(subcommand=subcommand_name) 309 | response, _ = self._request(endpoint) 310 | return response 311 | 312 | 313 | class JSONEncoder(json.JSONEncoder): 314 | def default(self, obj): 315 | if isinstance(obj, datetime.datetime): 316 | return obj.isoformat() 317 | return json.JSONEncoder.default(self, obj) 318 | -------------------------------------------------------------------------------- /src/sublime/cli/__init__.py: -------------------------------------------------------------------------------- 1 | """Sublime command line Interface.""" 2 | 3 | import logging 4 | import sys 5 | 6 | import click 7 | import structlog 8 | from click_default_group import DefaultGroup 9 | from click_repl import register_repl 10 | 11 | from sublime.cli import subcommand 12 | 13 | 14 | def configure_logging(): 15 | """Configure logging.""" 16 | logging.basicConfig(stream=sys.stderr, format="%(message)s", level=logging.CRITICAL) 17 | logging.getLogger("sublime").setLevel(logging.WARNING) 18 | structlog.configure( 19 | processors=[ 20 | structlog.stdlib.add_logger_name, 21 | structlog.stdlib.add_log_level, 22 | structlog.stdlib.PositionalArgumentsFormatter(), 23 | structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M.%S"), 24 | structlog.processors.StackInfoRenderer(), 25 | structlog.processors.format_exc_info, 26 | structlog.dev.ConsoleRenderer(), 27 | ], 28 | context_class=dict, 29 | logger_factory=structlog.stdlib.LoggerFactory(), 30 | wrapper_class=structlog.stdlib.BoundLogger, 31 | cache_logger_on_first_use=True, 32 | ) 33 | 34 | 35 | @click.group( 36 | cls=DefaultGroup, 37 | # default="query", 38 | default_if_no_args=False, 39 | context_settings={"help_option_names": ("-h", "--help")}, 40 | ) 41 | def main(): 42 | """Sublime CLI.""" 43 | configure_logging() 44 | 45 | 46 | SUBCOMMAND_FUNCTIONS = [ 47 | subcommand_function 48 | for subcommand_function in vars(subcommand).values() 49 | if isinstance(subcommand_function, click.Command) 50 | ] 51 | 52 | for subcommand_function in SUBCOMMAND_FUNCTIONS: 53 | main.add_command(subcommand_function) 54 | 55 | SUBCOMMAND_GROUPS = [] 56 | for sub in (): 57 | SUBCOMMAND_GROUPS.extend( 58 | subcommand_group 59 | for subcommand_group in vars(sub).values() 60 | if isinstance(subcommand_group, click.Group) 61 | ) 62 | 63 | for subcommand_group in SUBCOMMAND_GROUPS: 64 | main.add_command(subcommand_group) 65 | 66 | register_repl(main) 67 | main() 68 | -------------------------------------------------------------------------------- /src/sublime/cli/decorator.py: -------------------------------------------------------------------------------- 1 | """CLI subcommand decorators. 2 | 3 | Decorators used to add common functionality to subcommands. 4 | 5 | """ 6 | import os 7 | import functools 8 | import base64 9 | 10 | import click 11 | import structlog 12 | from requests.exceptions import RequestException 13 | 14 | from sublime.api import Sublime 15 | from sublime.cli.formatter import FORMATTERS, ANSI_MARKUP 16 | from sublime.error import * 17 | from sublime.util import load_config 18 | 19 | LOGGER = structlog.get_logger() 20 | 21 | 22 | def echo_result(function): 23 | """Decorator that prints subcommand results correctly formatted. 24 | 25 | :param function: Subcommand that returns a result from the API. 26 | :type function: callable 27 | :returns: Wrapped function that prints subcommand results 28 | :rtype: callable 29 | 30 | """ 31 | 32 | @functools.wraps(function) 33 | def wrapper(*args, **kwargs): 34 | result = function(*args, **kwargs) 35 | context = click.get_current_context() 36 | params = context.params 37 | if params.get("output_format"): 38 | output_format = params["output_format"] 39 | else: 40 | output_format = "txt" 41 | formatter = FORMATTERS[output_format] 42 | config = load_config() 43 | if isinstance(formatter, dict): 44 | # For the text formatter, there's a separate formatter for each 45 | if isinstance(context.parent.command, click.Group) and \ 46 | context.parent.command.name != 'main': 47 | # sub-sub command 48 | parent_name = context.parent.command.name 49 | cur_name = context.command.name 50 | name = f"{parent_name}_{cur_name}" 51 | formatter = formatter[name] 52 | else: 53 | # regular subcommand 54 | formatter = formatter[context.command.name] 55 | 56 | if context.command.name in ("create", "binexplode"): 57 | # default behavior is to always save the MDM and binexplode output 58 | # even if no output file is specified 59 | if not params.get("output_file"): 60 | input_file_relative_name = params.get('input_file').name 61 | input_file_relative_no_ext, _ = os.path.splitext( 62 | input_file_relative_name) 63 | input_file_name_no_ext = os.path.basename( 64 | input_file_relative_no_ext) 65 | output_file_name = f'{input_file_name_no_ext}' 66 | 67 | if output_format == "json": 68 | if context.command.name == "create": 69 | output_file_name += ".mdm" 70 | else: 71 | output_file_name += ".json" 72 | elif output_format == "txt": 73 | output_file_name += ".txt" 74 | 75 | # if the user has a default save directory configured, 76 | # store the file there 77 | if config["save_dir"]: 78 | output_file_name = os.path.join(config["save_dir"], output_file_name) 79 | 80 | params["output_file"] = click.open_file(output_file_name, mode="w") 81 | 82 | if context.command.name == "create": 83 | # strip the extra info and just save the unenriched MDM 84 | result = result["data_model"] 85 | 86 | output = formatter(result, 87 | params.get("verbose", False)).strip("\n") 88 | 89 | click.echo( 90 | output, 91 | file=params.get("output_file", click.open_file("-", mode="w")) 92 | ) 93 | 94 | file_name = params.get("output_file") 95 | if file_name: 96 | click.echo(ANSI_MARKUP(f"Output saved to {file_name.name}")) 97 | 98 | return wrapper 99 | 100 | 101 | def handle_exceptions(function): 102 | """Print error and exit on API client errors. 103 | 104 | :param function: Subcommand that returns a result from the API. 105 | :type function: callable 106 | :returns: Wrapped function that prints subcommand results 107 | :rtype: callable 108 | 109 | """ 110 | 111 | @functools.wraps(function) 112 | def wrapper(*args, **kwargs): 113 | try: 114 | return function(*args, **kwargs) 115 | except RateLimitError as error: 116 | error_message = "API error: {}".format(error.message) 117 | LOGGER.error(error_message) 118 | click.get_current_context().exit(-1) 119 | except InvalidRequestError as error: 120 | error_message = "API error: {}".format(error.message) 121 | LOGGER.error(error_message) 122 | click.get_current_context().exit(-1) 123 | except APIError as error: 124 | error_message = "API error: {}".format(error.message) 125 | LOGGER.error(error_message) 126 | click.get_current_context().exit(-1) 127 | except LoadRuleError as error: 128 | error_message = "Load rule error: {}".format(error.message) 129 | LOGGER.error(error_message) 130 | click.get_current_context().exit(-1) 131 | except LoadEMLError as error: 132 | error_message = "Load EML error: {}".format(error.message) 133 | LOGGER.error(error_message) 134 | click.get_current_context().exit(-1) 135 | except LoadMSGError as error: 136 | error_message = "Load MSG error: {}".format(error.message) 137 | LOGGER.error(error_message) 138 | click.get_current_context().exit(-1) 139 | except LoadMessageDataModelError as error: 140 | error_message = "Load MDM error: {}".format(error.message) 141 | LOGGER.error(error_message) 142 | click.get_current_context().exit(-1) 143 | except RequestException as error: 144 | error_message = "Request error: {}".format(error) 145 | LOGGER.error(error_message) 146 | click.get_current_context().exit(-1) 147 | except AuthenticationError as error: 148 | error_message = "API error: {}".format(error) 149 | LOGGER.error(error_message) 150 | 151 | # check to see if an API key is present, if not 152 | # print a helpful message 153 | context = click.get_current_context() 154 | api_key = context.params.get("api_key") 155 | config = load_config() 156 | 157 | if api_key is None: 158 | if not config["api_key"]: 159 | prog_name = context.parent.info_name 160 | click.echo( 161 | "\nError: API key not found.\n\n" 162 | "To fix this problem, please use any of the following methods " 163 | "(in order of precedence):\n" 164 | "- Pass it using the -k/--api-key option.\n" 165 | "- Set it in the SUBLIME_API_KEY environment variable.\n" 166 | "- Run 'setup -k' to save it to the configuration file.\n" 167 | ) 168 | click.get_current_context().exit(-1) 169 | 170 | return wrapper 171 | 172 | 173 | def pass_api_client(function): 174 | """Create API client form API key and pass it to subcommand. 175 | 176 | :param function: Subcommand that returns a result from the API. 177 | :type function: callable 178 | :returns: Wrapped function that prints subcommand results 179 | :rtype: callable 180 | 181 | """ 182 | 183 | @functools.wraps(function) 184 | def wrapper(*args, **kwargs): 185 | context = click.get_current_context() 186 | api_key = context.params.get("api_key") 187 | config = load_config() 188 | 189 | if api_key is None: 190 | if not config["api_key"]: 191 | pass 192 | ''' 193 | prog_name = context.parent.info_name 194 | click.echo( 195 | "\nError: API key not found.\n\n" 196 | "To fix this problem, please use any of the following methods " 197 | "(in order of precedence):\n" 198 | "- Pass it using the -k/--api-key option.\n" 199 | "- Set it in the SUBLIME_API_KEY environment variable.\n" 200 | "- Run 'setup -k' to save it to the configuration file.\n" 201 | ) 202 | context.exit(-1) 203 | ''' 204 | else: 205 | api_key = config["api_key"] 206 | 207 | api_client = Sublime(api_key=api_key) 208 | return function(api_client, *args, **kwargs) 209 | 210 | return wrapper 211 | 212 | 213 | def create_command(function): 214 | """Decorator that groups decorators common to create subcommand.""" 215 | 216 | @click.command() 217 | @click.option("-k", "--api-key", help="Key to include in API requests [optional]") 218 | @click.option( 219 | "-i", "--input", "input_file", type=click.File(), 220 | help="Input EML file", required=True 221 | ) 222 | @click.option("-t", "--type", "message_type", 223 | type=click.Choice(['inbound', 'internal', 'outbound'], case_sensitive=False), 224 | default="inbound", 225 | show_default=True, 226 | help="Set the message type [optional]" 227 | ) 228 | @click.option( 229 | "-o", "--output", "output_file", type=click.File(mode="w"), 230 | help=( 231 | "Output file. Defaults to the input_file name in the current " 232 | "directory with a .mdm extension if none is specified" 233 | ) 234 | ) 235 | @click.option( 236 | "-f", 237 | "--format", 238 | "output_format", 239 | type=click.Choice(["json", "txt"]), 240 | default="json", 241 | show_default=True, 242 | help="Output format", 243 | ) 244 | @click.option("-m", "--mailbox", "mailbox_email_address", 245 | help="Mailbox email address that received the message [optional]" 246 | ) 247 | @pass_api_client 248 | @click.pass_context 249 | @echo_result 250 | @handle_exceptions 251 | @functools.wraps(function) 252 | def wrapper(*args, **kwargs): 253 | return function(*args, **kwargs) 254 | 255 | return wrapper 256 | 257 | 258 | def analyze_command(function): 259 | """Decorator that groups decorators common to analyze subcommand.""" 260 | 261 | @click.command() 262 | 263 | @click.option("-k", "--api-key", "api_key", 264 | help="Key to include in API requests [optional]") 265 | 266 | @click.option("-i", "--input", "input_path", 267 | type=click.Path(exists=True), 268 | help="Input file or directory (.eml, .msg, .mdm and .mbox supported)", 269 | required=True) 270 | 271 | @click.option("-r", "--run", "run_path", 272 | type=click.Path(exists=True), 273 | help=( 274 | "Rule/query file or directory (.yml and .yaml supported). " 275 | "Queries outputs that return false, null, [], {} are not displayed by default" 276 | ) 277 | ) 278 | 279 | @click.option("-q", "--query", "query", 280 | type=str, 281 | help=("Raw MQL. Instead of using a rules file, " 282 | "provide raw MQL, surrounded by single quotes")) 283 | 284 | @click.option("-t", "--type", "message_type", 285 | type=click.Choice(['inbound', 'internal', 'outbound'], case_sensitive=False), 286 | default="inbound", 287 | show_default=True, 288 | help="Set the message type (EML and MSG files only) [optional]") 289 | 290 | @click.option("-m", "--mailbox", "mailbox_email_address", 291 | help=("Mailbox email address that received the " 292 | "message (EML and MSG files only) [optional]")) 293 | 294 | @click.option("-o", "--output", "output_file", 295 | type=click.File(mode="w"), 296 | help="Output file") 297 | 298 | @click.option("-f", "--format", "output_format", 299 | type=click.Choice(["json", "txt"]), 300 | default="txt", 301 | help="Output format") 302 | 303 | @pass_api_client 304 | @click.pass_context 305 | @echo_result 306 | @handle_exceptions 307 | @functools.wraps(function) 308 | def wrapper(*args, **kwargs): 309 | return function(*args, **kwargs) 310 | 311 | return wrapper 312 | 313 | 314 | def binexplode_command(function): 315 | """Decorator that groups decorators common to binexplode subcommand.""" 316 | 317 | @click.command() 318 | @click.option("-k", "--api-key", help="Key to include in API requests") 319 | @click.option("-i", "--input", "input_file", type=click.File(mode="rb"), required=True, 320 | help="Input file to scan using binexplode") 321 | @click.option( 322 | "-o", "--output", "output_file", type=click.File(mode="w"), 323 | help="Output file" 324 | ) 325 | @click.option( 326 | "-f", 327 | "--format", 328 | "output_format", 329 | type=click.Choice(["json"]), 330 | default="json", 331 | help="Output format", 332 | ) 333 | @pass_api_client 334 | @click.pass_context 335 | @echo_result 336 | @handle_exceptions 337 | @functools.wraps(function) 338 | def wrapper(*args, **kwargs): 339 | return function(*args, **kwargs) 340 | 341 | return wrapper 342 | 343 | 344 | def me_command(function): 345 | """Decorator that groups decorators common to me subcommand.""" 346 | 347 | @click.command() 348 | @click.option("-k", "--api-key", help="Key to include in API requests") 349 | @click.option( 350 | "-o", "--output", "output_file", type=click.File(mode="w"), 351 | help="Output file" 352 | ) 353 | @click.option( 354 | "-f", 355 | "--format", 356 | "output_format", 357 | type=click.Choice(["json", "txt"]), 358 | default="txt", 359 | help="Output format", 360 | ) 361 | @pass_api_client 362 | @click.pass_context 363 | @echo_result 364 | @handle_exceptions 365 | @functools.wraps(function) 366 | def wrapper(*args, **kwargs): 367 | return function(*args, **kwargs) 368 | 369 | return wrapper 370 | 371 | 372 | def feedback_command(function): 373 | """Decorator that groups decorators common to me subcommand.""" 374 | 375 | @click.command() 376 | @click.argument("feedback", type=str) 377 | @pass_api_client 378 | @click.pass_context 379 | @echo_result 380 | @handle_exceptions 381 | @functools.wraps(function) 382 | def wrapper(*args, **kwargs): 383 | return function(*args, **kwargs) 384 | 385 | return wrapper 386 | 387 | 388 | class MissingRuleInput(click.ClickException): 389 | """Exception used for analyze commands missing a YAML file or MQL 390 | """ 391 | 392 | def __init__(self): 393 | message = ( 394 | "You must specify either a YAML file/directory (-r) " 395 | "or raw MQL (-q)" 396 | ) 397 | super(MissingRuleInput, self).__init__(message) 398 | 399 | 400 | class SubcommandNotImplemented(click.ClickException): 401 | """Exception used temporarily for subcommands that have not been implemented. 402 | 403 | :param subcommand_name: Name of the subcommand to display in the error message. 404 | :type subcommand_function: str 405 | 406 | """ 407 | 408 | def __init__(self, subcommand_name): 409 | message = "{!r} subcommand is not implemented yet.".format(subcommand_name) 410 | super(SubcommandNotImplemented, self).__init__(message) 411 | 412 | 413 | def not_implemented_command(function): 414 | """Decorator that sends requests for not implemented commands.""" 415 | 416 | @click.command() 417 | @pass_api_client 418 | @functools.wraps(function) 419 | def wrapper(api_client, *args, **kwargs): 420 | command_name = function.__name__ 421 | try: 422 | api_client._not_implemented(command_name) 423 | except Exception: 424 | raise SubcommandNotImplemented(command_name) 425 | 426 | return wrapper 427 | -------------------------------------------------------------------------------- /src/sublime/cli/formatter.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """Output formatters.""" 3 | 4 | from __future__ import print_function 5 | 6 | import re 7 | import functools 8 | import json 9 | from xml.dom.minidom import parseString 10 | 11 | import gron 12 | import ansimarkup 13 | import click 14 | import colorama 15 | from jinja2 import Environment, PackageLoader 16 | 17 | JINJA2_ENV = Environment(loader=PackageLoader("sublime.cli"), 18 | extensions=['jinja2.ext.loopcontrols']) 19 | 20 | colorama.init() 21 | ANSI_MARKUP = ansimarkup.AnsiMarkup( 22 | tags={ 23 | "header": ansimarkup.parse(""), 24 | "key": ansimarkup.parse(""), 25 | "value": ansimarkup.parse(""), 26 | "not-detected": ansimarkup.parse(""), 27 | "fail": ansimarkup.parse(""), 28 | "success": ansimarkup.parse(""), 29 | "unknown": ansimarkup.parse(""), 30 | "detected": ansimarkup.parse(""), 31 | "enrichment": ansimarkup.parse(""), 32 | "warning": ansimarkup.parse(""), 33 | "query": ansimarkup.parse(""), 34 | } 35 | ) 36 | 37 | 38 | def colored_output(function): 39 | """Decorator that converts ansi markup into ansi escape sequences. 40 | 41 | :param function: Function that will return text using ansi markup. 42 | :type function: callable 43 | :returns: Wrapped function that converts markup into escape sequences. 44 | :rtype: callable 45 | 46 | """ 47 | 48 | @functools.wraps(function) 49 | def wrapper(*args, **kwargs): 50 | output = function(*args, **kwargs) 51 | return ANSI_MARKUP(output) 52 | 53 | return wrapper 54 | 55 | 56 | def json_formatter(result, verbose=False, indent=4, offset=0): 57 | """Format result as json.""" 58 | string = json.dumps(result, indent=indent) 59 | string = string.replace("\n", "\n" + " "*offset) 60 | return string 61 | 62 | 63 | def filter_none_recursive(item): 64 | """Recursive Filter Out Values""" 65 | if isinstance(item, list): 66 | return [filter_none_recursive(sub_item) for sub_item in item if sub_item is not None and (not isinstance(sub_item, list) or any(sub_sub_item is not None for sub_sub_item in sub_item))] 67 | return item 68 | 69 | 70 | @colored_output 71 | def analyze_formatter(results, verbose): 72 | """Convert Analyze output into human-readable text.""" 73 | mql_offset = 3 74 | json_offset = 2 75 | template_file = "analyze_multi.txt.j2" if len( 76 | results) > 1 else "analyze.txt.j2" 77 | template = JINJA2_ENV.get_template(template_file) 78 | 79 | # calculate total stats 80 | sample_result = next(iter(results.values())) 81 | summary_stats = { 82 | 'total_messages': len(results), 83 | 'total_rules': len(sample_result['rule_results']), 84 | 'total_queries': len(sample_result['query_results']), 85 | } 86 | rules = [rule for rule in sample_result['rule_results']] 87 | queries = [query for query in sample_result['query_results']] 88 | 89 | # separate matched/unmatched messages and distinguish flagged/unflagged rules 90 | flagged_messages = [] 91 | unflagged_messages = [] 92 | all_flagged_rules = set() 93 | for _, result in results.items(): 94 | normal_queries = [] 95 | falsey_queries = [] 96 | failed_queries = [] 97 | for query in result['query_results']: 98 | result_value = query.get('result') 99 | if isinstance(result_value, list): 100 | # Filter out None values from the result array 101 | filtered_result = filter_none_recursive(result_value) 102 | if filtered_result and any(item is not None for item in filtered_result): 103 | query['result'] = filtered_result 104 | normal_queries.append(query) 105 | elif result_value: 106 | # If the result is not a list and exists (is truthy) 107 | normal_queries.append(query) 108 | elif query.get('success'): 109 | falsey_queries.append(query) 110 | else: 111 | failed_queries.append(query) 112 | result['normal_query_results'] = normal_queries 113 | result['falsey_query_results'] = falsey_queries 114 | result['failed_query_results'] = failed_queries 115 | 116 | flagged_rules = [] 117 | unflagged_rules = [] 118 | failed_rules = [] 119 | for rule in result['rule_results']: 120 | if rule.get('matched'): 121 | flagged_rules.append(rule) 122 | # no unique identifier 123 | all_flagged_rules.add( 124 | rule.get("rule").get('name')+rule.get("rule").get('source')) 125 | elif rule.get('success'): 126 | unflagged_rules.append(rule) 127 | else: 128 | failed_rules.append(rule) 129 | result['flagged_rule_results'] = flagged_rules 130 | result['unflagged_rule_results'] = unflagged_rules 131 | result['failed_rule_results'] = failed_rules 132 | 133 | if len(flagged_rules) > 0: 134 | flagged_messages.append(result) 135 | else: 136 | unflagged_messages.append(result) 137 | 138 | # calculate flagged stats 139 | summary_stats['flagged_rules'] = len(all_flagged_rules) 140 | summary_stats['flagged_messages'] = len(flagged_messages) 141 | 142 | # format mql and json outputs 143 | for msg in flagged_messages + unflagged_messages: 144 | for result in msg['rule_results'] + msg['query_results']: 145 | if 'result' in result and (isinstance(result['result'], dict) or isinstance(result['result'], list)): 146 | result['result'] = json_formatter( 147 | result['result'], 148 | offset=json_offset, 149 | indent=2) 150 | 151 | # TO DO: sort each list of messages by extension and file name (or directory?) 152 | 153 | return template.render( 154 | stats=summary_stats, 155 | flagged_messages=flagged_messages, 156 | unflagged_messages=unflagged_messages, 157 | rules=rules, 158 | queries=queries, 159 | verbose=verbose) 160 | 161 | 162 | def mdm_formatter(results, verbose): 163 | """Convert Message Data Model into human-readable text.""" 164 | gron_output = gron.gron(json.dumps(results)) 165 | gron_output = gron_output.replace('json = {}', 'message_data_model = {}') 166 | gron_output = re.sub(r'\njson\.', '\n', gron_output) 167 | 168 | return gron_output 169 | 170 | # template = JINJA2_ENV.get_template("message_data_model.txt.j2") 171 | # return template.render(results=results, verbose=verbose) 172 | 173 | 174 | @colored_output 175 | def me_formatter(result, verbose): 176 | """Convert 'me' output into human-readable text.""" 177 | template = JINJA2_ENV.get_template("me_result.txt.j2") 178 | 179 | return template.render(result=result, verbose=verbose) 180 | 181 | 182 | @colored_output 183 | def feedback_formatter(result, verbose): 184 | """Convert 'feedback' output into human-readable text.""" 185 | template = JINJA2_ENV.get_template("feedback_result.txt.j2") 186 | 187 | return template.render(result=result, verbose=verbose) 188 | 189 | 190 | FORMATTERS = { 191 | "json": json_formatter, 192 | "txt": { 193 | "me": me_formatter, 194 | "feedback": feedback_formatter, 195 | "create": mdm_formatter, 196 | "analyze": analyze_formatter 197 | }, 198 | } 199 | -------------------------------------------------------------------------------- /src/sublime/cli/outlookmsgfile_helper.py: -------------------------------------------------------------------------------- 1 | # Source: 2 | # https://raw.githubusercontent.com/JoshData/convert-outlook-msg-file/primary/outlookmsgfile.py 3 | # MIT License 4 | # 5 | # Copyright (c) 2018 Joshua Tauberer 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | # This module converts a Microsoft Outlook .msg file into 26 | # a MIME message that can be loaded by most email programs 27 | # or inspected in a text editor. 28 | # 29 | # This script relies on the Python package compoundfiles 30 | # for reading the .msg container format. 31 | # 32 | # Referencecs: 33 | # 34 | # https://msdn.microsoft.com/en-us/library/cc463912.aspx 35 | # https://msdn.microsoft.com/en-us/library/cc463900(v=exchg.80).aspx 36 | # https://msdn.microsoft.com/en-us/library/ee157583(v=exchg.80).aspx 37 | # https://blogs.msdn.microsoft.com/openspecification/2009/11/06/msg-file-format-part-1/ 38 | 39 | import re 40 | import os 41 | import sys 42 | 43 | from functools import reduce 44 | 45 | import email.message, email.parser, email.policy 46 | from email.utils import parsedate_to_datetime, formatdate, formataddr 47 | 48 | import compoundfiles 49 | 50 | 51 | # MAIN FUNCTIONS 52 | 53 | 54 | def load(filename_or_stream): 55 | with compoundfiles.CompoundFileReader(filename_or_stream) as doc: 56 | doc.rtf_attachments = 0 57 | return load_message_stream(doc.root, True, doc) 58 | 59 | 60 | def load_message_stream(entry, is_top_level, doc): 61 | # Load stream data. 62 | props = parse_properties(entry['__properties_version1.0'], is_top_level, entry, doc) 63 | 64 | # Construct the MIME message.... 65 | msg = email.message.EmailMessage() 66 | 67 | # Add the raw headers, if known. 68 | if 'TRANSPORT_MESSAGE_HEADERS' in props: 69 | # Get the string holding all of the headers. 70 | headers = props['TRANSPORT_MESSAGE_HEADERS'] 71 | if isinstance(headers, bytes): 72 | headers = headers.decode("utf-8") 73 | 74 | # Remove content-type header because the body we can get this 75 | # way is just the plain-text portion of the email and whatever 76 | # Content-Type header was in the original is not valid for 77 | # reconstructing it this way. 78 | headers = re.sub("Content-Type: .*(\n\s.*)*\n", "", headers, re.I) 79 | 80 | # Parse them. 81 | headers = email.parser.HeaderParser(policy=email.policy.default)\ 82 | .parsestr(headers) 83 | 84 | # Copy them into the message object. 85 | for header, value in headers.items(): 86 | msg[header] = value 87 | 88 | else: 89 | # Construct common headers from metadata. 90 | 91 | if 'MESSAGE_DELIVERY_TIME' in props: 92 | msg['Date'] = formatdate(props['MESSAGE_DELIVERY_TIME'].timestamp()) 93 | del props['MESSAGE_DELIVERY_TIME'] 94 | 95 | if 'SENDER_NAME' in props: 96 | if 'SENT_REPRESENTING_NAME' in props: 97 | if props['SENT_REPRESENTING_NAME']: 98 | if props['SENDER_NAME'] != props['SENT_REPRESENTING_NAME']: 99 | props['SENDER_NAME'] += " (" + props['SENT_REPRESENTING_NAME'] + ")" 100 | del props['SENT_REPRESENTING_NAME'] 101 | if props['SENDER_NAME']: 102 | msg['From'] = formataddr((props['SENDER_NAME'], "")) 103 | del props['SENDER_NAME'] 104 | 105 | if 'DISPLAY_TO' in props: 106 | if props['DISPLAY_TO']: 107 | msg['To'] = props['DISPLAY_TO'] 108 | del props['DISPLAY_TO'] 109 | 110 | if 'DISPLAY_CC' in props: 111 | if props['DISPLAY_CC']: 112 | msg['CC'] = props['DISPLAY_CC'] 113 | del props['DISPLAY_CC'] 114 | 115 | if 'DISPLAY_BCC' in props: 116 | if props['DISPLAY_BCC']: 117 | msg['BCC'] = props['DISPLAY_BCC'] 118 | del props['DISPLAY_BCC'] 119 | 120 | if 'SUBJECT' in props: 121 | if props['SUBJECT']: 122 | msg['Subject'] = props['SUBJECT'] 123 | del props['SUBJECT'] 124 | 125 | # Add the plain-text body from the BODY field. 126 | if 'BODY' in props: 127 | body = props['BODY'] 128 | if isinstance(body, str): 129 | msg.set_content(body, cte='quoted-printable') 130 | else: 131 | msg.set_content(body, maintype="text", subtype="plain", cte='8bit') 132 | 133 | # Plain-text is not availabe. Use the rich text version. 134 | else: 135 | doc.rtf_attachments += 1 136 | fn = "messagebody_{}.rtf".format(doc.rtf_attachments) 137 | 138 | msg.set_content( 139 | "".format(fn), 140 | cte='quoted-printable') 141 | 142 | # Decompress the value to Rich Text Format. 143 | import compressed_rtf 144 | rtf = props['RTF_COMPRESSED'] 145 | rtf = compressed_rtf.decompress(rtf) 146 | 147 | # Add RTF file as an attachment. 148 | msg.add_attachment( 149 | rtf, 150 | maintype="text", subtype="rtf", 151 | filename=fn) 152 | 153 | # # Copy over string values of remaining properties as headers 154 | # # so we don't lose any information. 155 | # for k, v in props.items(): 156 | # if k == 'RTF_COMPRESSED': continue # not interested, save output 157 | # msg[k] = str(v) 158 | 159 | # Add attachments. 160 | for stream in entry: 161 | if stream.name.startswith("__attach_version1.0_#"): 162 | process_attachment(msg, stream, doc) 163 | 164 | return msg 165 | 166 | 167 | def process_attachment(msg, entry, doc): 168 | # Load attachment stream. 169 | props = parse_properties(entry['__properties_version1.0'], False, entry, doc) 170 | 171 | # The attachment content... 172 | blob = props['ATTACH_DATA_BIN'] 173 | 174 | # Get the filename and MIME type of the attachment. 175 | filename = props.get("ATTACH_LONG_FILENAME") or props.get("ATTACH_FILENAME") or props.get("DISPLAY_NAME") 176 | if isinstance(filename, bytes): filename = filename.decode("utf8") 177 | 178 | mime_type = props.get('ATTACH_MIME_TAG', 'application/octet-stream') 179 | if isinstance(mime_type, bytes): mime_type = mime_type.decode("utf8") 180 | 181 | filename = os.path.basename(filename) 182 | 183 | # Python 3.6. 184 | if isinstance(blob, str): 185 | msg.add_attachment( 186 | blob, 187 | filename=filename) 188 | elif isinstance(blob, bytes): 189 | msg.add_attachment( 190 | blob, 191 | maintype=mime_type.split("/", 1)[0], subtype=mime_type.split("/", 1)[-1], 192 | filename=filename) 193 | else: # a Message instance 194 | msg.add_attachment( 195 | blob, 196 | filename=filename) 197 | 198 | def parse_properties(properties, is_top_level, container, doc): 199 | # Read a properties stream and return a Python dictionary 200 | # of the fields and values, using human-readable field names 201 | # in the mapping at the top of this module. 202 | 203 | # Load stream content. 204 | with doc.open(properties) as stream: 205 | stream = stream.read() 206 | 207 | # Skip header. 208 | i = (32 if is_top_level else 24) 209 | 210 | # Read 16-byte entries. 211 | ret = { } 212 | while i < len(stream): 213 | # Read the entry. 214 | property_type = stream[i+0:i+2] 215 | property_tag = stream[i+2:i+4] 216 | flags = stream[i+4:i+8] 217 | value = stream[i+8:i+16] 218 | i += 16 219 | 220 | # Turn the byte strings into numbers and look up the property type. 221 | property_type = property_type[0] + (property_type[1]<<8) 222 | property_tag = property_tag[0] + (property_tag[1]<<8) 223 | if property_tag not in property_tags: continue # should not happen 224 | tag_name, _ = property_tags[property_tag] 225 | tag_type = property_types.get(property_type) 226 | 227 | # Fixed Length Properties. 228 | if isinstance(tag_type, FixedLengthValueLoader): 229 | value = tag_type.load(value) 230 | 231 | # Variable Length Properties. 232 | elif isinstance(tag_type, VariableLengthValueLoader): 233 | value_length = stream[i+8:i+12] # not used 234 | 235 | # Look up the stream in the document that holds the value. 236 | streamname = "__substg1.0_{0:0{1}X}{2:0{3}X}".format(property_tag,4, property_type,4) 237 | try: 238 | with doc.open(container[streamname]) as innerstream: 239 | value = innerstream.read() 240 | except: 241 | # Stream isn't present! 242 | print("stream missing", streamname, file=sys.stderr) 243 | continue 244 | 245 | value = tag_type.load(value) 246 | 247 | elif isinstance(tag_type, EMBEDDED_MESSAGE): 248 | # Look up the stream in the document that holds the attachment. 249 | streamname = "__substg1.0_{0:0{1}X}{2:0{3}X}".format(property_tag,4, property_type,4) 250 | try: 251 | value = container[streamname] 252 | except: 253 | # Stream isn't present! 254 | print("stream missing", streamname, file=sys.stderr) 255 | continue 256 | value = tag_type.load(value, doc) 257 | 258 | else: 259 | # unrecognized type 260 | print("unhandled property type", hex(property_type), file=sys.stderr) 261 | continue 262 | 263 | ret[tag_name] = value 264 | 265 | return ret 266 | 267 | 268 | # PROPERTY VALUE LOADERS 269 | 270 | class FixedLengthValueLoader(object): 271 | pass 272 | 273 | class NULL(FixedLengthValueLoader): 274 | @staticmethod 275 | def load(value): 276 | # value is an eight-byte long bytestring with unused content. 277 | return None 278 | 279 | class BOOLEAN(FixedLengthValueLoader): 280 | @staticmethod 281 | def load(value): 282 | # value is an eight-byte long bytestring holding a two-byte integer. 283 | return value[0] == 1 284 | 285 | class INTEGER16(FixedLengthValueLoader): 286 | @staticmethod 287 | def load(value): 288 | # value is an eight-byte long bytestring holding a two-byte integer. 289 | return reduce(lambda a, b : (a<<8)+b, reversed(value[0:2])) 290 | 291 | class INTEGER32(FixedLengthValueLoader): 292 | @staticmethod 293 | def load(value): 294 | # value is an eight-byte long bytestring holding a four-byte integer. 295 | return reduce(lambda a, b : (a<<8)+b, reversed(value[0:4])) 296 | 297 | class INTEGER64(FixedLengthValueLoader): 298 | @staticmethod 299 | def load(value): 300 | # value is an eight-byte long bytestring holding an eight-byte integer. 301 | return reduce(lambda a, b : (a<<8)+b, reversed(value)) 302 | 303 | class INTTIME(FixedLengthValueLoader): 304 | @staticmethod 305 | def load(value): 306 | # value is an eight-byte long bytestring encoding the integer number of 307 | # 100-nanosecond intervals since January 1, 1601. 308 | from datetime import datetime, timedelta 309 | value = reduce(lambda a, b : (a<<8)+b, reversed(value)) # bytestring to integer 310 | value = datetime(1601, 1, 1) + timedelta(seconds=value/10000000) 311 | return value 312 | 313 | # TODO: The other fixed-length data types: 314 | # "FLOAT", "DOUBLE", "CURRENCY", "APPTIME", "ERROR" 315 | 316 | class VariableLengthValueLoader(object): 317 | pass 318 | 319 | class BINARY(VariableLengthValueLoader): 320 | @staticmethod 321 | def load(value): 322 | # value is a bytestring. Just return it. 323 | return value 324 | 325 | class STRING8(VariableLengthValueLoader): 326 | @staticmethod 327 | def load(value): 328 | # value is a bytestring. I haven't seen specified what character encoding 329 | # is used when the Unicode storage type is not used, so we'll assume it's 330 | # ASCII or Latin-1 like but we'll use UTF-8 to cover the bases. 331 | return value.decode("utf8") 332 | 333 | class UNICODE(VariableLengthValueLoader): 334 | @staticmethod 335 | def load(value): 336 | # value is a bytestring. I haven't seen specified what character encoding 337 | # is used when the Unicode storage type is not used, so we'll assume it's 338 | # ASCII or Latin-1 like but we'll use UTF-8 to cover the bases. 339 | return value.decode("utf16") 340 | 341 | # TODO: The other variable-length tag types are "CLSID", "OBJECT". 342 | 343 | class EMBEDDED_MESSAGE(object): 344 | @staticmethod 345 | def load(entry, doc): 346 | return load_message_stream(entry, False, doc) 347 | 348 | 349 | # CONSTANTS 350 | 351 | # These constants are defined by the Microsoft Outlook file format 352 | # and identify the data types and data fields in the .msg file. 353 | 354 | # from mapidefs.h via https://github.com/inverse-inc/openchange.old/blob/master/libmapi/mapidefs.h 355 | property_types = { 356 | 0x1: NULL(), 357 | 0x2: INTEGER16(), 358 | 0x3: INTEGER32(), 359 | 0x4: "FLOAT", 360 | 0x5: "DOUBLE", 361 | 0x6: "CURRENCY", 362 | 0x7: "APPTIME", 363 | 0xa: "ERROR", 364 | 0xb: BOOLEAN(), 365 | 0xd: EMBEDDED_MESSAGE(), 366 | 0x14: INTEGER64(), 367 | 0x1e: STRING8(), 368 | 0x1f: UNICODE(), 369 | 0x40: INTTIME(), 370 | 0x48: "CLSID", 371 | 0xFB: "SVREID", 372 | 0xFD: "SRESTRICT", 373 | 0xFE: "ACTIONS", 374 | 0x102: BINARY(), 375 | } 376 | 377 | # from mapitags.h via https://github.com/mvz/email-outlook-message-perl/blob/master/mapitags.h 378 | property_tags = { 379 | 0x01: ('ACKNOWLEDGEMENT_MODE', 'I4'), 380 | 0x02: ('ALTERNATE_RECIPIENT_ALLOWED', 'BOOLEAN'), 381 | 0x03: ('AUTHORIZING_USERS', 'BINARY'), 382 | # Comment on an automatically forwarded message 383 | 0x04: ('AUTO_FORWARD_COMMENT', 'STRING'), 384 | # Whether a message has been automatically forwarded 385 | 0x05: ('AUTO_FORWARDED', 'BOOLEAN'), 386 | 0x06: ('CONTENT_CONFIDENTIALITY_ALGORITHM_ID', 'BINARY'), 387 | 0x07: ('CONTENT_CORRELATOR', 'BINARY'), 388 | 0x08: ('CONTENT_IDENTIFIER', 'STRING'), 389 | # MIME content length 390 | 0x09: ('CONTENT_LENGTH', 'I4'), 391 | 0x0A: ('CONTENT_RETURN_REQUESTED', 'BOOLEAN'), 392 | 0x0B: ('CONVERSATION_KEY', 'BINARY'), 393 | 0x0C: ('CONVERSION_EITS', 'BINARY'), 394 | 0x0D: ('CONVERSION_WITH_LOSS_PROHIBITED', 'BOOLEAN'), 395 | 0x0E: ('CONVERTED_EITS', 'BINARY'), 396 | # Time to deliver for delayed delivery messages 397 | 0x0F: ('DEFERRED_DELIVERY_TIME', 'SYSTIME'), 398 | 0x10: ('DELIVER_TIME', 'SYSTIME'), 399 | # Reason a message was discarded 400 | 0x11: ('DISCARD_REASON', 'I4'), 401 | 0x12: ('DISCLOSURE_OF_RECIPIENTS', 'BOOLEAN'), 402 | 0x13: ('DL_EXPANSION_HISTORY', 'BINARY'), 403 | 0x14: ('DL_EXPANSION_PROHIBITED', 'BOOLEAN'), 404 | 0x15: ('EXPIRY_TIME', 'SYSTIME'), 405 | 0x16: ('IMPLICIT_CONVERSION_PROHIBITED', 'BOOLEAN'), 406 | # Message importance 407 | 0x17: ('IMPORTANCE', 'I4'), 408 | 0x18: ('IPM_ID', 'BINARY'), 409 | 0x19: ('LATEST_DELIVERY_TIME', 'SYSTIME'), 410 | 0x1A: ('MESSAGE_CLASS', 'STRING'), 411 | 0x1B: ('MESSAGE_DELIVERY_ID', 'BINARY'), 412 | 0x1E: ('MESSAGE_SECURITY_LABEL', 'BINARY'), 413 | 0x1F: ('OBSOLETED_IPMS', 'BINARY'), 414 | # Person a message was originally for 415 | 0x20: ('ORIGINALLY_INTENDED_RECIPIENT_NAME', 'BINARY'), 416 | 0x21: ('ORIGINAL_EITS', 'BINARY'), 417 | 0x22: ('ORIGINATOR_CERTIFICATE', 'BINARY'), 418 | 0x23: ('ORIGINATOR_DELIVERY_REPORT_REQUESTED', 'BOOLEAN'), 419 | # Address of the message sender 420 | 0x24: ('ORIGINATOR_RETURN_ADDRESS', 'BINARY'), 421 | 0x25: ('PARENT_KEY', 'BINARY'), 422 | 0x26: ('PRIORITY', 'I4'), 423 | 0x27: ('ORIGIN_CHECK', 'BINARY'), 424 | 0x28: ('PROOF_OF_SUBMISSION_REQUESTED', 'BOOLEAN'), 425 | # Whether a read receipt is desired 426 | 0x29: ('READ_RECEIPT_REQUESTED', 'BOOLEAN'), 427 | # Time a message was received 428 | 0x2A: ('RECEIPT_TIME', 'SYSTIME'), 429 | 0x2B: ('RECIPIENT_REASSIGNMENT_PROHIBITED', 'BOOLEAN'), 430 | 0x2C: ('REDIRECTION_HISTORY', 'BINARY'), 431 | 0x2D: ('RELATED_IPMS', 'BINARY'), 432 | # Sensitivity of the original message 433 | 0x2E: ('ORIGINAL_SENSITIVITY', 'I4'), 434 | 0x2F: ('LANGUAGES', 'STRING'), 435 | 0x30: ('REPLY_TIME', 'SYSTIME'), 436 | 0x31: ('REPORT_TAG', 'BINARY'), 437 | 0x32: ('REPORT_TIME', 'SYSTIME'), 438 | 0x33: ('RETURNED_IPM', 'BOOLEAN'), 439 | 0x34: ('SECURITY', 'I4'), 440 | 0x35: ('INCOMPLETE_COPY', 'BOOLEAN'), 441 | 0x36: ('SENSITIVITY', 'I4'), 442 | # The message subject 443 | 0x37: ('SUBJECT', 'STRING'), 444 | 0x38: ('SUBJECT_IPM', 'BINARY'), 445 | 0x39: ('CLIENT_SUBMIT_TIME', 'SYSTIME'), 446 | 0x3A: ('REPORT_NAME', 'STRING'), 447 | 0x3B: ('SENT_REPRESENTING_SEARCH_KEY', 'BINARY'), 448 | 0x3C: ('X400_CONTENT_TYPE', 'BINARY'), 449 | 0x3D: ('SUBJECT_PREFIX', 'STRING'), 450 | 0x3E: ('NON_RECEIPT_REASON', 'I4'), 451 | 0x3F: ('RECEIVED_BY_ENTRYID', 'BINARY'), 452 | # Received by: entry 453 | 0x40: ('RECEIVED_BY_NAME', 'STRING'), 454 | 0x41: ('SENT_REPRESENTING_ENTRYID', 'BINARY'), 455 | 0x42: ('SENT_REPRESENTING_NAME', 'STRING'), 456 | 0x43: ('RCVD_REPRESENTING_ENTRYID', 'BINARY'), 457 | 0x44: ('RCVD_REPRESENTING_NAME', 'STRING'), 458 | 0x45: ('REPORT_ENTRYID', 'BINARY'), 459 | 0x46: ('READ_RECEIPT_ENTRYID', 'BINARY'), 460 | 0x47: ('MESSAGE_SUBMISSION_ID', 'BINARY'), 461 | 0x48: ('PROVIDER_SUBMIT_TIME', 'SYSTIME'), 462 | # Subject of the original message 463 | 0x49: ('ORIGINAL_SUBJECT', 'STRING'), 464 | 0x4A: ('DISC_VAL', 'BOOLEAN'), 465 | 0x4B: ('ORIG_MESSAGE_CLASS', 'STRING'), 466 | 0x4C: ('ORIGINAL_AUTHOR_ENTRYID', 'BINARY'), 467 | # Author of the original message 468 | 0x4D: ('ORIGINAL_AUTHOR_NAME', 'STRING'), 469 | # Time the original message was submitted 470 | 0x4E: ('ORIGINAL_SUBMIT_TIME', 'SYSTIME'), 471 | 0x4F: ('REPLY_RECIPIENT_ENTRIES', 'BINARY'), 472 | 0x50: ('REPLY_RECIPIENT_NAMES', 'STRING'), 473 | 0x51: ('RECEIVED_BY_SEARCH_KEY', 'BINARY'), 474 | 0x52: ('RCVD_REPRESENTING_SEARCH_KEY', 'BINARY'), 475 | 0x53: ('READ_RECEIPT_SEARCH_KEY', 'BINARY'), 476 | 0x54: ('REPORT_SEARCH_KEY', 'BINARY'), 477 | 0x55: ('ORIGINAL_DELIVERY_TIME', 'SYSTIME'), 478 | 0x56: ('ORIGINAL_AUTHOR_SEARCH_KEY', 'BINARY'), 479 | 0x57: ('MESSAGE_TO_ME', 'BOOLEAN'), 480 | 0x58: ('MESSAGE_CC_ME', 'BOOLEAN'), 481 | 0x59: ('MESSAGE_RECIP_ME', 'BOOLEAN'), 482 | # Sender of the original message 483 | 0x5A: ('ORIGINAL_SENDER_NAME', 'STRING'), 484 | 0x5B: ('ORIGINAL_SENDER_ENTRYID', 'BINARY'), 485 | 0x5C: ('ORIGINAL_SENDER_SEARCH_KEY', 'BINARY'), 486 | 0x5D: ('ORIGINAL_SENT_REPRESENTING_NAME', 'STRING'), 487 | 0x5E: ('ORIGINAL_SENT_REPRESENTING_ENTRYID', 'BINARY'), 488 | 0x5F: ('ORIGINAL_SENT_REPRESENTING_SEARCH_KEY', 'BINARY'), 489 | 0x60: ('START_DATE', 'SYSTIME'), 490 | 0x61: ('END_DATE', 'SYSTIME'), 491 | 0x62: ('OWNER_APPT_ID', 'I4'), 492 | # Whether a response to the message is desired 493 | 0x63: ('RESPONSE_REQUESTED', 'BOOLEAN'), 494 | 0x64: ('SENT_REPRESENTING_ADDRTYPE', 'STRING'), 495 | 0x65: ('SENT_REPRESENTING_EMAIL_ADDRESS', 'STRING'), 496 | 0x66: ('ORIGINAL_SENDER_ADDRTYPE', 'STRING'), 497 | # Email of the original message sender 498 | 0x67: ('ORIGINAL_SENDER_EMAIL_ADDRESS', 'STRING'), 499 | 0x68: ('ORIGINAL_SENT_REPRESENTING_ADDRTYPE', 'STRING'), 500 | 0x69: ('ORIGINAL_SENT_REPRESENTING_EMAIL_ADDRESS', 'STRING'), 501 | 0x70: ('CONVERSATION_TOPIC', 'STRING'), 502 | 0x71: ('CONVERSATION_INDEX', 'BINARY'), 503 | 0x72: ('ORIGINAL_DISPLAY_BCC', 'STRING'), 504 | 0x73: ('ORIGINAL_DISPLAY_CC', 'STRING'), 505 | 0x74: ('ORIGINAL_DISPLAY_TO', 'STRING'), 506 | 0x75: ('RECEIVED_BY_ADDRTYPE', 'STRING'), 507 | 0x76: ('RECEIVED_BY_EMAIL_ADDRESS', 'STRING'), 508 | 0x77: ('RCVD_REPRESENTING_ADDRTYPE', 'STRING'), 509 | 0x78: ('RCVD_REPRESENTING_EMAIL_ADDRESS', 'STRING'), 510 | 0x79: ('ORIGINAL_AUTHOR_ADDRTYPE', 'STRING'), 511 | 0x7A: ('ORIGINAL_AUTHOR_EMAIL_ADDRESS', 'STRING'), 512 | 0x7B: ('ORIGINALLY_INTENDED_RECIP_ADDRTYPE', 'STRING'), 513 | 0x7C: ('ORIGINALLY_INTENDED_RECIP_EMAIL_ADDRESS', 'STRING'), 514 | 0x7D: ('TRANSPORT_MESSAGE_HEADERS', 'STRING'), 515 | 0x7E: ('DELEGATION', 'BINARY'), 516 | 0x7F: ('TNEF_CORRELATION_KEY', 'BINARY'), 517 | 0x1000: ('BODY', 'STRING'), 518 | 0x1001: ('REPORT_TEXT', 'STRING'), 519 | 0x1002: ('ORIGINATOR_AND_DL_EXPANSION_HISTORY', 'BINARY'), 520 | 0x1003: ('REPORTING_DL_NAME', 'BINARY'), 521 | 0x1004: ('REPORTING_MTA_CERTIFICATE', 'BINARY'), 522 | 0x1006: ('RTF_SYNC_BODY_CRC', 'I4'), 523 | 0x1007: ('RTF_SYNC_BODY_COUNT', 'I4'), 524 | 0x1008: ('RTF_SYNC_BODY_TAG', 'STRING'), 525 | 0x1009: ('RTF_COMPRESSED', 'BINARY'), 526 | 0x1010: ('RTF_SYNC_PREFIX_COUNT', 'I4'), 527 | 0x1011: ('RTF_SYNC_TRAILING_COUNT', 'I4'), 528 | 0x1012: ('ORIGINALLY_INTENDED_RECIP_ENTRYID', 'BINARY'), 529 | 0x0C00: ('CONTENT_INTEGRITY_CHECK', 'BINARY'), 530 | 0x0C01: ('EXPLICIT_CONVERSION', 'I4'), 531 | 0x0C02: ('IPM_RETURN_REQUESTED', 'BOOLEAN'), 532 | 0x0C03: ('MESSAGE_TOKEN', 'BINARY'), 533 | 0x0C04: ('NDR_REASON_CODE', 'I4'), 534 | 0x0C05: ('NDR_DIAG_CODE', 'I4'), 535 | 0x0C06: ('NON_RECEIPT_NOTIFICATION_REQUESTED', 'BOOLEAN'), 536 | 0x0C07: ('DELIVERY_POINT', 'I4'), 537 | 0x0C08: ('ORIGINATOR_NON_DELIVERY_REPORT_REQUESTED', 'BOOLEAN'), 538 | 0x0C09: ('ORIGINATOR_REQUESTED_ALTERNATE_RECIPIENT', 'BINARY'), 539 | 0x0C0A: ('PHYSICAL_DELIVERY_BUREAU_FAX_DELIVERY', 'BOOLEAN'), 540 | 0x0C0B: ('PHYSICAL_DELIVERY_MODE', 'I4'), 541 | 0x0C0C: ('PHYSICAL_DELIVERY_REPORT_REQUEST', 'I4'), 542 | 0x0C0D: ('PHYSICAL_FORWARDING_ADDRESS', 'BINARY'), 543 | 0x0C0E: ('PHYSICAL_FORWARDING_ADDRESS_REQUESTED', 'BOOLEAN'), 544 | 0x0C0F: ('PHYSICAL_FORWARDING_PROHIBITED', 'BOOLEAN'), 545 | 0x0C10: ('PHYSICAL_RENDITION_ATTRIBUTES', 'BINARY'), 546 | 0x0C11: ('PROOF_OF_DELIVERY', 'BINARY'), 547 | 0x0C12: ('PROOF_OF_DELIVERY_REQUESTED', 'BOOLEAN'), 548 | 0x0C13: ('RECIPIENT_CERTIFICATE', 'BINARY'), 549 | 0x0C14: ('RECIPIENT_NUMBER_FOR_ADVICE', 'STRING'), 550 | 0x0C15: ('RECIPIENT_TYPE', 'I4'), 551 | 0x0C16: ('REGISTERED_MAIL_TYPE', 'I4'), 552 | 0x0C17: ('REPLY_REQUESTED', 'BOOLEAN'), 553 | 0x0C18: ('REQUESTED_DELIVERY_METHOD', 'I4'), 554 | 0x0C19: ('SENDER_ENTRYID', 'BINARY'), 555 | 0x0C1A: ('SENDER_NAME', 'STRING'), 556 | 0x0C1B: ('SUPPLEMENTARY_INFO', 'STRING'), 557 | 0x0C1C: ('TYPE_OF_MTS_USER', 'I4'), 558 | 0x0C1D: ('SENDER_SEARCH_KEY', 'BINARY'), 559 | 0x0C1E: ('SENDER_ADDRTYPE', 'STRING'), 560 | 0x0C1F: ('SENDER_EMAIL_ADDRESS', 'STRING'), 561 | 0x0E00: ('CURRENT_VERSION', 'I8'), 562 | 0x0E01: ('DELETE_AFTER_SUBMIT', 'BOOLEAN'), 563 | 0x0E02: ('DISPLAY_BCC', 'STRING'), 564 | 0x0E03: ('DISPLAY_CC', 'STRING'), 565 | 0x0E04: ('DISPLAY_TO', 'STRING'), 566 | 0x0E05: ('PARENT_DISPLAY', 'STRING'), 567 | 0x0E06: ('MESSAGE_DELIVERY_TIME', 'SYSTIME'), 568 | 0x0E07: ('MESSAGE_FLAGS', 'I4'), 569 | 0x0E08: ('MESSAGE_SIZE', 'I4'), 570 | 0x0E09: ('PARENT_ENTRYID', 'BINARY'), 571 | 0x0E0A: ('SENTMAIL_ENTRYID', 'BINARY'), 572 | 0x0E0C: ('CORRELATE', 'BOOLEAN'), 573 | 0x0E0D: ('CORRELATE_MTSID', 'BINARY'), 574 | 0x0E0E: ('DISCRETE_VALUES', 'BOOLEAN'), 575 | 0x0E0F: ('RESPONSIBILITY', 'BOOLEAN'), 576 | 0x0E10: ('SPOOLER_STATUS', 'I4'), 577 | 0x0E11: ('TRANSPORT_STATUS', 'I4'), 578 | 0x0E12: ('MESSAGE_RECIPIENTS', 'OBJECT'), 579 | 0x0E13: ('MESSAGE_ATTACHMENTS', 'OBJECT'), 580 | 0x0E14: ('SUBMIT_FLAGS', 'I4'), 581 | 0x0E15: ('RECIPIENT_STATUS', 'I4'), 582 | 0x0E16: ('TRANSPORT_KEY', 'I4'), 583 | 0x0E17: ('MSG_STATUS', 'I4'), 584 | 0x0E18: ('MESSAGE_DOWNLOAD_TIME', 'I4'), 585 | 0x0E19: ('CREATION_VERSION', 'I8'), 586 | 0x0E1A: ('MODIFY_VERSION', 'I8'), 587 | 0x0E1B: ('HASATTACH', 'BOOLEAN'), 588 | 0x0E1D: ('NORMALIZED_SUBJECT', 'STRING'), 589 | 0x0E1F: ('RTF_IN_SYNC', 'BOOLEAN'), 590 | 0x0E20: ('ATTACH_SIZE', 'I4'), 591 | 0x0E21: ('ATTACH_NUM', 'I4'), 592 | 0x0E22: ('PREPROCESS', 'BOOLEAN'), 593 | 0x0E25: ('ORIGINATING_MTA_CERTIFICATE', 'BINARY'), 594 | 0x0E26: ('PROOF_OF_SUBMISSION', 'BINARY'), 595 | # A unique identifier for editing the properties of a MAPI object 596 | 0x0FFF: ('ENTRYID', 'BINARY'), 597 | # The type of an object 598 | 0x0FFE: ('OBJECT_TYPE', 'I4'), 599 | 0x0FFD: ('ICON', 'BINARY'), 600 | 0x0FFC: ('MINI_ICON', 'BINARY'), 601 | 0x0FFB: ('STORE_ENTRYID', 'BINARY'), 602 | 0x0FFA: ('STORE_RECORD_KEY', 'BINARY'), 603 | # Binary identifer for an individual object 604 | 0x0FF9: ('RECORD_KEY', 'BINARY'), 605 | 0x0FF8: ('MAPPING_SIGNATURE', 'BINARY'), 606 | 0x0FF7: ('ACCESS_LEVEL', 'I4'), 607 | # The primary key of a column in a table 608 | 0x0FF6: ('INSTANCE_KEY', 'BINARY'), 609 | 0x0FF5: ('ROW_TYPE', 'I4'), 610 | 0x0FF4: ('ACCESS', 'I4'), 611 | 0x3000: ('ROWID', 'I4'), 612 | # The name to display for a given MAPI object 613 | 0x3001: ('DISPLAY_NAME', 'STRING'), 614 | 0x3002: ('ADDRTYPE', 'STRING'), 615 | # An email address 616 | 0x3003: ('EMAIL_ADDRESS', 'STRING'), 617 | # A comment field 618 | 0x3004: ('COMMENT', 'STRING'), 619 | 0x3005: ('DEPTH', 'I4'), 620 | # Provider-defined display name for a service provider 621 | 0x3006: ('PROVIDER_DISPLAY', 'STRING'), 622 | # The time an object was created 623 | 0x3007: ('CREATION_TIME', 'SYSTIME'), 624 | # The time an object was last modified 625 | 0x3008: ('LAST_MODIFICATION_TIME', 'SYSTIME'), 626 | # Flags describing a service provider, message service, or status object 627 | 0x3009: ('RESOURCE_FLAGS', 'I4'), 628 | # The name of a provider dll, minus any "32" suffix and ".dll" 629 | 0x300A: ('PROVIDER_DLL_NAME', 'STRING'), 630 | 0x300B: ('SEARCH_KEY', 'BINARY'), 631 | 0x300C: ('PROVIDER_UID', 'BINARY'), 632 | 0x300D: ('PROVIDER_ORDINAL', 'I4'), 633 | 0x3301: ('FORM_VERSION', 'STRING'), 634 | 0x3302: ('FORM_CLSID', 'CLSID'), 635 | 0x3303: ('FORM_CONTACT_NAME', 'STRING'), 636 | 0x3304: ('FORM_CATEGORY', 'STRING'), 637 | 0x3305: ('FORM_CATEGORY_SUB', 'STRING'), 638 | 0x3306: ('FORM_HOST_MAP', 'MV_LONG'), 639 | 0x3307: ('FORM_HIDDEN', 'BOOLEAN'), 640 | 0x3308: ('FORM_DESIGNER_NAME', 'STRING'), 641 | 0x3309: ('FORM_DESIGNER_GUID', 'CLSID'), 642 | 0x330A: ('FORM_MESSAGE_BEHAVIOR', 'I4'), 643 | # Is this row the default message store? 644 | 0x3400: ('DEFAULT_STORE', 'BOOLEAN'), 645 | 0x340D: ('STORE_SUPPORT_MASK', 'I4'), 646 | 0x340E: ('STORE_STATE', 'I4'), 647 | 0x3410: ('IPM_SUBTREE_SEARCH_KEY', 'BINARY'), 648 | 0x3411: ('IPM_OUTBOX_SEARCH_KEY', 'BINARY'), 649 | 0x3412: ('IPM_WASTEBASKET_SEARCH_KEY', 'BINARY'), 650 | 0x3413: ('IPM_SENTMAIL_SEARCH_KEY', 'BINARY'), 651 | # Provder-defined message store type 652 | 0x3414: ('MDB_PROVIDER', 'BINARY'), 653 | 0x3415: ('RECEIVE_FOLDER_SETTINGS', 'OBJECT'), 654 | 0x35DF: ('VALID_FOLDER_MASK', 'I4'), 655 | 0x35E0: ('IPM_SUBTREE_ENTRYID', 'BINARY'), 656 | 0x35E2: ('IPM_OUTBOX_ENTRYID', 'BINARY'), 657 | 0x35E3: ('IPM_WASTEBASKET_ENTRYID', 'BINARY'), 658 | 0x35E4: ('IPM_SENTMAIL_ENTRYID', 'BINARY'), 659 | 0x35E5: ('VIEWS_ENTRYID', 'BINARY'), 660 | 0x35E6: ('COMMON_VIEWS_ENTRYID', 'BINARY'), 661 | 0x35E7: ('FINDER_ENTRYID', 'BINARY'), 662 | 0x3600: ('CONTAINER_FLAGS', 'I4'), 663 | 0x3601: ('FOLDER_TYPE', 'I4'), 664 | 0x3602: ('CONTENT_COUNT', 'I4'), 665 | 0x3603: ('CONTENT_UNREAD', 'I4'), 666 | 0x3604: ('CREATE_TEMPLATES', 'OBJECT'), 667 | 0x3605: ('DETAILS_TABLE', 'OBJECT'), 668 | 0x3607: ('SEARCH', 'OBJECT'), 669 | 0x3609: ('SELECTABLE', 'BOOLEAN'), 670 | 0x360A: ('SUBFOLDERS', 'BOOLEAN'), 671 | 0x360B: ('STATUS', 'I4'), 672 | 0x360C: ('ANR', 'STRING'), 673 | 0x360D: ('CONTENTS_SORT_ORDER', 'MV_LONG'), 674 | 0x360E: ('CONTAINER_HIERARCHY', 'OBJECT'), 675 | 0x360F: ('CONTAINER_CONTENTS', 'OBJECT'), 676 | 0x3610: ('FOLDER_ASSOCIATED_CONTENTS', 'OBJECT'), 677 | 0x3611: ('DEF_CREATE_DL', 'BINARY'), 678 | 0x3612: ('DEF_CREATE_MAILUSER', 'BINARY'), 679 | 0x3613: ('CONTAINER_CLASS', 'STRING'), 680 | 0x3614: ('CONTAINER_MODIFY_VERSION', 'I8'), 681 | 0x3615: ('AB_PROVIDER_ID', 'BINARY'), 682 | 0x3616: ('DEFAULT_VIEW_ENTRYID', 'BINARY'), 683 | 0x3617: ('ASSOC_CONTENT_COUNT', 'I4'), 684 | 0x3700: ('ATTACHMENT_X400_PARAMETERS', 'BINARY'), 685 | 0x3701: ('ATTACH_DATA_OBJ', 'OBJECT'), 686 | 0x3701: ('ATTACH_DATA_BIN', 'BINARY'), 687 | 0x3702: ('ATTACH_ENCODING', 'BINARY'), 688 | 0x3703: ('ATTACH_EXTENSION', 'STRING'), 689 | 0x3704: ('ATTACH_FILENAME', 'STRING'), 690 | 0x3705: ('ATTACH_METHOD', 'I4'), 691 | 0x3707: ('ATTACH_LONG_FILENAME', 'STRING'), 692 | 0x3708: ('ATTACH_PATHNAME', 'STRING'), 693 | 0x370A: ('ATTACH_TAG', 'BINARY'), 694 | 0x370B: ('RENDERING_POSITION', 'I4'), 695 | 0x370C: ('ATTACH_TRANSPORT_NAME', 'STRING'), 696 | 0x370D: ('ATTACH_LONG_PATHNAME', 'STRING'), 697 | 0x370E: ('ATTACH_MIME_TAG', 'STRING'), 698 | 0x370F: ('ATTACH_ADDITIONAL_INFO', 'BINARY'), 699 | 0x3900: ('DISPLAY_TYPE', 'I4'), 700 | 0x3902: ('TEMPLATEID', 'BINARY'), 701 | 0x3904: ('PRIMARY_CAPABILITY', 'BINARY'), 702 | 0x39FF: ('7BIT_DISPLAY_NAME', 'STRING'), 703 | 0x3A00: ('ACCOUNT', 'STRING'), 704 | 0x3A01: ('ALTERNATE_RECIPIENT', 'BINARY'), 705 | 0x3A02: ('CALLBACK_TELEPHONE_NUMBER', 'STRING'), 706 | 0x3A03: ('CONVERSION_PROHIBITED', 'BOOLEAN'), 707 | 0x3A04: ('DISCLOSE_RECIPIENTS', 'BOOLEAN'), 708 | 0x3A05: ('GENERATION', 'STRING'), 709 | 0x3A06: ('GIVEN_NAME', 'STRING'), 710 | 0x3A07: ('GOVERNMENT_ID_NUMBER', 'STRING'), 711 | 0x3A08: ('BUSINESS_TELEPHONE_NUMBER', 'STRING'), 712 | 0x3A09: ('HOME_TELEPHONE_NUMBER', 'STRING'), 713 | 0x3A0A: ('INITIALS', 'STRING'), 714 | 0x3A0B: ('KEYWORD', 'STRING'), 715 | 0x3A0C: ('LANGUAGE', 'STRING'), 716 | 0x3A0D: ('LOCATION', 'STRING'), 717 | 0x3A0E: ('MAIL_PERMISSION', 'BOOLEAN'), 718 | 0x3A0F: ('MHS_COMMON_NAME', 'STRING'), 719 | 0x3A10: ('ORGANIZATIONAL_ID_NUMBER', 'STRING'), 720 | 0x3A11: ('SURNAME', 'STRING'), 721 | 0x3A12: ('ORIGINAL_ENTRYID', 'BINARY'), 722 | 0x3A13: ('ORIGINAL_DISPLAY_NAME', 'STRING'), 723 | 0x3A14: ('ORIGINAL_SEARCH_KEY', 'BINARY'), 724 | 0x3A15: ('POSTAL_ADDRESS', 'STRING'), 725 | 0x3A16: ('COMPANY_NAME', 'STRING'), 726 | 0x3A17: ('TITLE', 'STRING'), 727 | 0x3A18: ('DEPARTMENT_NAME', 'STRING'), 728 | 0x3A19: ('OFFICE_LOCATION', 'STRING'), 729 | 0x3A1A: ('PRIMARY_TELEPHONE_NUMBER', 'STRING'), 730 | 0x3A1B: ('BUSINESS2_TELEPHONE_NUMBER', 'STRING'), 731 | 0x3A1C: ('MOBILE_TELEPHONE_NUMBER', 'STRING'), 732 | 0x3A1D: ('RADIO_TELEPHONE_NUMBER', 'STRING'), 733 | 0x3A1E: ('CAR_TELEPHONE_NUMBER', 'STRING'), 734 | 0x3A1F: ('OTHER_TELEPHONE_NUMBER', 'STRING'), 735 | 0x3A20: ('TRANSMITABLE_DISPLAY_NAME', 'STRING'), 736 | 0x3A21: ('PAGER_TELEPHONE_NUMBER', 'STRING'), 737 | 0x3A22: ('USER_CERTIFICATE', 'BINARY'), 738 | 0x3A23: ('PRIMARY_FAX_NUMBER', 'STRING'), 739 | 0x3A24: ('BUSINESS_FAX_NUMBER', 'STRING'), 740 | 0x3A25: ('HOME_FAX_NUMBER', 'STRING'), 741 | 0x3A26: ('COUNTRY', 'STRING'), 742 | 0x3A27: ('LOCALITY', 'STRING'), 743 | 0x3A28: ('STATE_OR_PROVINCE', 'STRING'), 744 | 0x3A29: ('STREET_ADDRESS', 'STRING'), 745 | 0x3A2A: ('POSTAL_CODE', 'STRING'), 746 | 0x3A2B: ('POST_OFFICE_BOX', 'STRING'), 747 | 0x3A2C: ('TELEX_NUMBER', 'STRING'), 748 | 0x3A2D: ('ISDN_NUMBER', 'STRING'), 749 | 0x3A2E: ('ASSISTANT_TELEPHONE_NUMBER', 'STRING'), 750 | 0x3A2F: ('HOME2_TELEPHONE_NUMBER', 'STRING'), 751 | 0x3A30: ('ASSISTANT', 'STRING'), 752 | 0x3A40: ('SEND_RICH_INFO', 'BOOLEAN'), 753 | 0x3A41: ('WEDDING_ANNIVERSARY', 'SYSTIME'), 754 | 0x3A42: ('BIRTHDAY', 'SYSTIME'), 755 | 0x3A43: ('HOBBIES', 'STRING'), 756 | 0x3A44: ('MIDDLE_NAME', 'STRING'), 757 | 0x3A45: ('DISPLAY_NAME_PREFIX', 'STRING'), 758 | 0x3A46: ('PROFESSION', 'STRING'), 759 | 0x3A47: ('PREFERRED_BY_NAME', 'STRING'), 760 | 0x3A48: ('SPOUSE_NAME', 'STRING'), 761 | 0x3A49: ('COMPUTER_NETWORK_NAME', 'STRING'), 762 | 0x3A4A: ('CUSTOMER_ID', 'STRING'), 763 | 0x3A4B: ('TTYTDD_PHONE_NUMBER', 'STRING'), 764 | 0x3A4C: ('FTP_SITE', 'STRING'), 765 | 0x3A4D: ('GENDER', 'I2'), 766 | 0x3A4E: ('MANAGER_NAME', 'STRING'), 767 | 0x3A4F: ('NICKNAME', 'STRING'), 768 | 0x3A50: ('PERSONAL_HOME_PAGE', 'STRING'), 769 | 0x3A51: ('BUSINESS_HOME_PAGE', 'STRING'), 770 | 0x3A52: ('CONTACT_VERSION', 'CLSID'), 771 | 0x3A53: ('CONTACT_ENTRYIDS', 'MV_BINARY'), 772 | 0x3A54: ('CONTACT_ADDRTYPES', 'MV_STRING'), 773 | 0x3A55: ('CONTACT_DEFAULT_ADDRESS_INDEX', 'I4'), 774 | 0x3A56: ('CONTACT_EMAIL_ADDRESSES', 'MV_STRING'), 775 | 0x3A57: ('COMPANY_MAIN_PHONE_NUMBER', 'STRING'), 776 | 0x3A58: ('CHILDRENS_NAMES', 'MV_STRING'), 777 | 0x3A59: ('HOME_ADDRESS_CITY', 'STRING'), 778 | 0x3A5A: ('HOME_ADDRESS_COUNTRY', 'STRING'), 779 | 0x3A5B: ('HOME_ADDRESS_POSTAL_CODE', 'STRING'), 780 | 0x3A5C: ('HOME_ADDRESS_STATE_OR_PROVINCE', 'STRING'), 781 | 0x3A5D: ('HOME_ADDRESS_STREET', 'STRING'), 782 | 0x3A5E: ('HOME_ADDRESS_POST_OFFICE_BOX', 'STRING'), 783 | 0x3A5F: ('OTHER_ADDRESS_CITY', 'STRING'), 784 | 0x3A60: ('OTHER_ADDRESS_COUNTRY', 'STRING'), 785 | 0x3A61: ('OTHER_ADDRESS_POSTAL_CODE', 'STRING'), 786 | 0x3A62: ('OTHER_ADDRESS_STATE_OR_PROVINCE', 'STRING'), 787 | 0x3A63: ('OTHER_ADDRESS_STREET', 'STRING'), 788 | 0x3A64: ('OTHER_ADDRESS_POST_OFFICE_BOX', 'STRING'), 789 | 0x3D00: ('STORE_PROVIDERS', 'BINARY'), 790 | 0x3D01: ('AB_PROVIDERS', 'BINARY'), 791 | 0x3D02: ('TRANSPORT_PROVIDERS', 'BINARY'), 792 | 0x3D04: ('DEFAULT_PROFILE', 'BOOLEAN'), 793 | 0x3D05: ('AB_SEARCH_PATH', 'MV_BINARY'), 794 | 0x3D06: ('AB_DEFAULT_DIR', 'BINARY'), 795 | 0x3D07: ('AB_DEFAULT_PAB', 'BINARY'), 796 | 0x3D09: ('SERVICE_NAME', 'STRING'), 797 | 0x3D0A: ('SERVICE_DLL_NAME', 'STRING'), 798 | 0x3D0B: ('SERVICE_ENTRY_NAME', 'STRING'), 799 | 0x3D0C: ('SERVICE_UID', 'BINARY'), 800 | 0x3D0D: ('SERVICE_EXTRA_UIDS', 'BINARY'), 801 | 0x3D0E: ('SERVICES', 'BINARY'), 802 | 0x3D0F: ('SERVICE_SUPPORT_FILES', 'MV_STRING'), 803 | 0x3D10: ('SERVICE_DELETE_FILES', 'MV_STRING'), 804 | 0x3D11: ('AB_SEARCH_PATH_UPDATE', 'BINARY'), 805 | 0x3D12: ('PROFILE_NAME', 'STRING'), 806 | 0x3E00: ('IDENTITY_DISPLAY', 'STRING'), 807 | 0x3E01: ('IDENTITY_ENTRYID', 'BINARY'), 808 | 0x3E02: ('RESOURCE_METHODS', 'I4'), 809 | # Service provider type 810 | 0x3E03: ('RESOURCE_TYPE', 'I4'), 811 | 0x3E04: ('STATUS_CODE', 'I4'), 812 | 0x3E05: ('IDENTITY_SEARCH_KEY', 'BINARY'), 813 | 0x3E06: ('OWN_STORE_ENTRYID', 'BINARY'), 814 | 0x3E07: ('RESOURCE_PATH', 'STRING'), 815 | 0x3E08: ('STATUS_STRING', 'STRING'), 816 | 0x3E09: ('X400_DEFERRED_DELIVERY_CANCEL', 'BOOLEAN'), 817 | 0x3E0A: ('HEADER_FOLDER_ENTRYID', 'BINARY'), 818 | 0x3E0B: ('REMOTE_PROGRESS', 'I4'), 819 | 0x3E0C: ('REMOTE_PROGRESS_TEXT', 'STRING'), 820 | 0x3E0D: ('REMOTE_VALIDATE_OK', 'BOOLEAN'), 821 | 0x3F00: ('CONTROL_FLAGS', 'I4'), 822 | 0x3F01: ('CONTROL_STRUCTURE', 'BINARY'), 823 | 0x3F02: ('CONTROL_TYPE', 'I4'), 824 | 0x3F03: ('DELTAX', 'I4'), 825 | 0x3F04: ('DELTAY', 'I4'), 826 | 0x3F05: ('XPOS', 'I4'), 827 | 0x3F06: ('YPOS', 'I4'), 828 | 0x3F07: ('CONTROL_ID', 'BINARY'), 829 | 0x3F08: ('INITIAL_DETAILS_PANE', 'I4'), 830 | } 831 | 832 | 833 | # COMMAND-LINE ENTRY POINT 834 | 835 | 836 | if __name__ == "__main__": 837 | # If no command-line arguments are given, convert the .msg 838 | # file on STDIN to .eml format on STDOUT. 839 | if len(sys.argv) <= 1: 840 | print(load(sys.stdin), file=sys.stdout) 841 | 842 | # Otherwise, for each file mentioned on the command-line, 843 | # convert it and save it to a file with ".eml" appended 844 | # to the name. 845 | else: 846 | for fn in sys.argv[1:]: 847 | print(fn + "...") 848 | msg = load(fn) 849 | with open(fn + ".eml", "wb") as f: 850 | f.write(msg.as_bytes()) 851 | -------------------------------------------------------------------------------- /src/sublime/cli/subcommand.py: -------------------------------------------------------------------------------- 1 | """CLI subcommands.""" 2 | 3 | import os 4 | import platform 5 | import base64 6 | 7 | import click 8 | import structlog 9 | from halo import Halo 10 | from pathlib import Path 11 | 12 | from sublime.__version__ import __version__ 13 | from sublime.cli.decorator import ( 14 | me_command, 15 | feedback_command, 16 | analyze_command, 17 | create_command, 18 | binexplode_command, 19 | not_implemented_command, 20 | MissingRuleInput 21 | ) 22 | from sublime.util import * 23 | from sublime.error import AuthenticationError 24 | 25 | # for the listen subcommand 26 | import ssl 27 | import asyncio 28 | from functools import wraps 29 | from sublime.cli.formatter import FORMATTERS 30 | 31 | LOGGER = structlog.get_logger() 32 | 33 | 34 | @click.command(name="help") 35 | @click.pass_context 36 | def help_(context): 37 | """Show this message and exit.""" 38 | click.echo(context.parent.get_help()) 39 | 40 | def clear(): 41 | """Clear the console""" 42 | # check and make call for appropriate operating system 43 | os.system('clear' if os.name =='posix' else 'cls') 44 | 45 | 46 | @create_command 47 | @click.option("-v", "--verbose", count=True, help="Verbose output") 48 | def create( 49 | context, 50 | api_client, 51 | api_key, 52 | input_file, 53 | message_type, 54 | output_file, 55 | output_format, 56 | mailbox_email_address, 57 | verbose, 58 | ): 59 | """Create a Message Data Model from an EML or MSG.""" 60 | request_permission("create", api_key) 61 | 62 | if input_file.name.endswith(".msg"): 63 | raw_message = load_msg_file_handle(input_file) 64 | else: 65 | raw_message = load_eml_file_handle(input_file) 66 | 67 | results = api_client.create_message( 68 | raw_message, 69 | mailbox_email_address, 70 | message_type) 71 | 72 | return results 73 | 74 | 75 | @analyze_command 76 | @click.option("-v", "--verbose", count=True, help="Verbose output") 77 | def analyze( 78 | context, 79 | api_client, 80 | api_key, 81 | input_path, 82 | run_path, 83 | query, 84 | message_type, 85 | mailbox_email_address, 86 | output_file, 87 | output_format, 88 | verbose, 89 | ): 90 | """Analyze a file or directory of EMLs, MSGs, MDMs or MBOX files.""" 91 | request_permission("analyze", api_key) 92 | 93 | if not run_path and not query: 94 | raise MissingRuleInput 95 | 96 | # load all rules and queries 97 | rules, queries = [], [] 98 | if run_path: 99 | if os.path.isfile(run_path): 100 | with open(run_path, encoding='utf-8') as f: 101 | try: 102 | rules, queries = load_yml(f) 103 | except LoadRuleError as error: 104 | LOGGER.warning(error.message) 105 | 106 | elif os.path.isdir(run_path): 107 | rules, queries = load_yml_path(run_path) 108 | 109 | elif query: 110 | queries = [{ 111 | "source": query, 112 | "name": None, 113 | }] 114 | 115 | if not rules and not queries: 116 | LOGGER.error("YML file or raw MQL string required") 117 | context.exit(-1) 118 | 119 | # sort rules and queries in advance so we don't have to later 120 | # analyze endpoint should conserve the order in which they're submitted 121 | rules = sorted(rules, key=lambda i: i['name'].lower() if i.get('name') else '') 122 | queries = sorted(queries, key=lambda i: i['name'].lower() if i.get('name') else '') 123 | 124 | # aggregate all files we need to check 125 | file_paths = [] 126 | if os.path.isfile(input_path): 127 | file_paths.append(input_path) 128 | else: 129 | for extension in ['msg', 'eml', 'mbox']: 130 | for file_path in Path(input_path).rglob('*.' + extension): 131 | file_paths.append(str(file_path)) 132 | if not file_paths: 133 | LOGGER.error("Input file(s) must have .eml, .msg, or .mbox extension") 134 | context.exit(-1) 135 | 136 | # analyze each file and aggregate all responses 137 | results = {} 138 | errors = [] 139 | num_files = len(file_paths) 140 | with Halo(text="", spinner='dots') as halo: 141 | for i in range(num_files): 142 | halo.start() 143 | file_path = file_paths[i] 144 | file_dir, _, file_name = file_path.rpartition('/') 145 | _, _, extension = file_name.rpartition('.') 146 | halo_text = f"Analyzing file {i+1} of {num_files} ( {file_name} )" 147 | halo.text = halo_text 148 | 149 | if file_path.endswith('.msg'): 150 | try: 151 | raw_message = load_msg(file_path) 152 | response = api_client.analyze_message( 153 | raw_message, 154 | rules, 155 | queries) 156 | except Exception as exception: 157 | if isinstance(exception, AuthenticationError): 158 | raise exception 159 | else: 160 | halo.stop() 161 | LOGGER.warning(f"failed to analyze ({file_name}): {exception}") 162 | errors.append(exception) 163 | continue 164 | 165 | elif file_path.endswith('.eml'): 166 | try: 167 | raw_message = load_eml(file_path) 168 | response = api_client.analyze_message( 169 | raw_message, 170 | rules, 171 | queries) 172 | except Exception as exception: 173 | if isinstance(exception, AuthenticationError): 174 | raise exception 175 | else: 176 | halo.stop() 177 | LOGGER.warning(f"failed to analyze ({file_name}): {exception}") 178 | errors.append(exception) 179 | continue 180 | 181 | elif file_path.endswith('.mbox'): 182 | # in the mbox case we want to retrieve the response for each message 183 | # contained and provide a unique results key for each entry 184 | mbox_files = load_mbox(file_path, halo=halo) 185 | file_count = len(mbox_files) 186 | 187 | count = 0 188 | for subject_unique in mbox_files.keys(): 189 | count += 1 190 | halo_suffix = f" message {count} of {file_count}..." 191 | halo.text = halo_text + halo_suffix 192 | try: 193 | response = api_client.analyze_message( 194 | mbox_files[subject_unique], 195 | rules, 196 | queries) 197 | except Exception as exception: 198 | if isinstance(exception, AuthenticationError): 199 | raise exception 200 | else: 201 | halo.stop() 202 | LOGGER.warning(f"failed to analyze ({file_name}): {exception}") 203 | errors.append(exception) 204 | continue 205 | 206 | response['file_name'] = file_name 207 | response['extension'] = extension 208 | response['directory'] = file_dir 209 | response['subject'] = subject_unique 210 | results[file_path+subject_unique] = response 211 | continue 212 | 213 | else: 214 | LOGGER.error("Input file(s) must have .eml, .msg, or .mbox extension") 215 | context.exit(-1) 216 | 217 | response['file_name'] = file_name 218 | response['extension'] = extension 219 | response['directory'] = file_dir 220 | results[file_path] = response 221 | 222 | # raise the first error we saw if there were no successful results 223 | if len(results) == 0: raise errors[0] 224 | return results 225 | 226 | 227 | @binexplode_command 228 | @click.option("-v", "--verbose", count=True, help="Verbose output") 229 | def binexplode( 230 | context, 231 | api_client, 232 | api_key, 233 | input_file, 234 | output_file, 235 | output_format, 236 | verbose, 237 | ): 238 | """Scan a binary using binexplode.""" 239 | 240 | file_contents = input_file.read() 241 | 242 | with Halo(text="Scanning file...", spinner='dots') as halo: 243 | file_contents_base64_encoded = base64.b64encode(file_contents).decode('utf-8') 244 | file_name = Path(input_file.name).name 245 | result = api_client.binexplode_scan(file_contents_base64_encoded, file_name) 246 | 247 | return result 248 | 249 | 250 | @me_command 251 | @click.option("-v", "--verbose", count=True, help="Verbose output") 252 | def me( 253 | context, 254 | api_client, 255 | api_key, 256 | output_file, 257 | output_format, 258 | verbose, 259 | ): 260 | """Get information about the currently authenticated Sublime user.""" 261 | 262 | result = api_client.me() 263 | 264 | return result 265 | 266 | 267 | @feedback_command 268 | def feedback( 269 | context, 270 | api_client, 271 | feedback 272 | ): 273 | """Send feedback directly to the Sublime team. 274 | 275 | Use single quotes for 'FEEDBACK' 276 | """ 277 | 278 | result = api_client.feedback(feedback) 279 | 280 | return result 281 | 282 | 283 | @click.command() 284 | @click.option("-k", "--api-key", required=False, 285 | help="Key to include in API requests") 286 | @click.option("-s", "--save-dir", required=False, 287 | type=click.Path(resolve_path=True), 288 | help="Default save directory for items retrieved from your Sublime environment") 289 | def setup(api_key="", save_dir=""): 290 | """Configure defaults.""" 291 | config = {"api_key": api_key, "save_dir": save_dir, "permission": ""} 292 | save_config(config) 293 | click.echo("Configuration saved to {!r}".format(CONFIG_FILE)) 294 | 295 | 296 | @click.command() 297 | def version(): 298 | """Get version and OS information for your Sublime commandline installation.""" 299 | click.echo( 300 | "sublime {}\n" 301 | " Python {}\n" 302 | " {}\n".format(__version__, platform.python_version(), platform.platform()) 303 | ) 304 | -------------------------------------------------------------------------------- /src/sublime/cli/templates/analyze.txt.j2: -------------------------------------------------------------------------------- 1 | {% import "macros.txt.j2" as macros with context %} 2 | 3 | ╔═══════════════════════════╗ 4 | ║
{{ "{:^25}".format("Results") }}
║ 5 | ╚═══════════════════════════╝ 6 | {%- if flagged_messages | length > 0 %} 7 | {% set msg = flagged_messages[0] %} 8 | {%- else %} 9 | {% set msg = unflagged_messages[0] %} 10 | {%- endif %} 11 | File Name: {{msg.file_name}} 12 | {%- if msg.directory %} 13 | Directory: {{msg.directory}} 14 | {%- endif %} 15 | 16 | Total Rules: {{stats.total_rules}} 17 | Total Queries: {{stats.total_queries}} 18 | {%- if stats.total_rules > 0 -%} 19 | {# new line #} 20 | Flagged Rules: {{stats.flagged_rules}} 21 | {%- endif -%} 22 | {# new line #} 23 | {# new line #} 24 | 25 | {%- if msg.flagged_rule_results|length > 0 -%} 26 | {{ macros.flagged_rules(msg) }} 27 | {%- endif -%} 28 | 29 | {%- if msg.unflagged_rule_results|length > 0 %} 30 | {# new line #} 31 |
UNFLAGGED RULES
32 | {# new line #} 33 | {%- set max_elements = 20 %} 34 | {%- set elements_slice = msg.unflagged_rule_results[:max_elements if verbose < 1 else None] %} 35 | {%- for rule in elements_slice %} 36 | - {{ rule.rule.name }} 37 | {%- if verbose %} 38 | Source: {{ rule.source }} 39 | {# new line #} 40 | {%- endif %} 41 | {%- endfor %} 42 | {% if msg.unflagged_rule_results | length > max_elements and verbose < 1 -%} 43 | {{ " " | indent(2) }} - And {{ msg.unflagged_rule_results | length - max_elements}} more. Run again with -v for full output. 44 | {% endif -%} 45 | {%- endif %} 46 | 47 | {%- if msg.failed_rule_results|length > 0 %} 48 | {{ macros.failed_rules(msg) }} 49 | {%- endif %} 50 | 51 | {%- if true -%} 52 | {{ macros.query_results(msg) }} 53 | {%- endif -%} 54 | -------------------------------------------------------------------------------- /src/sublime/cli/templates/analyze_multi.txt.j2: -------------------------------------------------------------------------------- 1 | {% import "macros.txt.j2" as macros with context %} 2 | 3 | ╔═══════════════════════════╗ 4 | ║
{{ "{:^25}".format("Results") }}
║ 5 | ╚═══════════════════════════╝ 6 | 7 | {# new line #} 8 |
SUMMARY
9 | ================== 10 | Total Rules: {{stats.total_rules}} 11 | Total Queries: {{stats.total_queries}} 12 | Total Messages: {{stats.total_messages}} 13 | {# new line #} 14 | {%- if stats.total_rules > 0 -%} 15 | {# new line #} 16 | Flagged Rules: {{stats.flagged_rules}} 17 | {%- endif -%} 18 | {# new line #} 19 | Flagged Messages: {{stats.flagged_messages}} 20 | 21 | {%- if rules|length > 0 %} 22 | {# new line #} 23 | {# new line #} 24 |
Rules Run
25 | ------------------ 26 | {%- for rule in rules %} 27 | - {{rule.name}} 28 | {%- endfor %} 29 | {%- endif %} 30 | 31 | {%- if queries|length > 0 %} 32 | {# new line #} 33 | {# new line #} 34 |
Queries Run
35 | ------------------ 36 | {%- for query in queries %} 37 | {%- if query.name %} 38 | - {{query.name}} 39 | {%- else %} 40 | - Query {{loop.index}} 41 | {%- endif %} 42 | {%- endfor %} 43 | {%- endif %} 44 | 45 | 46 | {%- if flagged_messages|length > 0 %} 47 | {# new line #} 48 | {# new line #} 49 |
FLAGGED MESSAGES
50 | ================== 51 | {%- endif %} 52 | 53 | {%- for msg in flagged_messages %} 54 | {# new line #} 55 |
MESSAGE {{loop.index}}
56 | ------------------ 57 | {%- if msg.extension == 'mbox' %} 58 | Subject: {{msg.subject}} 59 | {%- endif%} 60 | File Name: {{msg.file_name}} 61 | {%- if msg.directory %} 62 | Directory: {{msg.directory}} 63 | {%- endif %} 64 | {# new line #} 65 | 66 | {%- if msg.flagged_rule_results|length > 0 -%} 67 | {{ macros.flagged_rules(msg) | indent(4, True) }} 68 | {%- endif -%} 69 | 70 | {%- if msg.failed_rule_results|length > 0 -%} 71 | {{ macros.failed_rules(msg) | indent(4, True) }} 72 | {%- endif -%} 73 | 74 | {%- if true -%} 75 | {{ macros.query_results(msg) | indent(4, True) }} 76 | {%- endif -%} 77 | 78 | {% endfor %} 79 | 80 | 81 | 82 | {%- if unflagged_messages|length > 0 %} 83 | {# new line #} 84 | {%- if rules|length > 0 %} 85 |
UNFLAGGED MESSAGES
86 | ================== 87 | {%- else %} 88 |
MESSAGES
89 | ================== 90 | {%- endif %} 91 | {%- endif %} 92 | 93 | {%- for msg in unflagged_messages %} 94 | {# new line #} 95 |
MESSAGE {{loop.index}}
96 | ------------------ 97 | {%- if msg.extension == 'mbox' %} 98 | Subject: {{msg.subject}} 99 | {%- endif %} 100 | File Name: {{msg.file_name}} 101 | Directory: {{msg.directory}} 102 | {# new line #} 103 | 104 | {%- if msg.failed_rule_results|length > 0 -%} 105 | {{ macros.failed_rules(msg) | indent(4, True) }} 106 | {%- endif -%} 107 | 108 | {%- if true -%} 109 | {{ macros.query_results(msg) | indent(4, True) }} 110 | {%- endif -%} 111 | 112 | {% endfor %} 113 | -------------------------------------------------------------------------------- /src/sublime/cli/templates/feedback_result.txt.j2: -------------------------------------------------------------------------------- 1 | {%- if result.first_name and result.first_name != 'Unauthenticated' %} 2 | Thank you, {{ result.first_name }}! We received your feedback. 3 | {%- else %} 4 | Thank you! We received your feedback. 5 | {%- endif %} 6 | -------------------------------------------------------------------------------- /src/sublime/cli/templates/macros.txt.j2: -------------------------------------------------------------------------------- 1 | {% macro flagged_rules(msg) %} 2 | {# new line #} 3 |
FLAGGED RULES
4 | {# new line #} 5 | {%- for rule in msg.flagged_rule_results %} 6 | - {{ rule.rule.name}} 7 | {%- if verbose %} 8 | Source: {{ rule.source }} 9 | {# new line #} 10 | {%- endif %} 11 | {%- endfor %} 12 | {% endmacro %} 13 | 14 | {% macro failed_rules(msg) %} 15 | {# new line #} 16 |
FAILED RULES
17 | {# new line #} 18 | {%- for rule in msg.failed_rule_results %} 19 | - {{ rule.rule.name }} 20 | Error: {{ rule.error }} 21 | {%- if verbose %} 22 | Source: {{ rule.source }} 23 | {# new line #} 24 | {%- endif %} 25 | {%- endfor %} 26 | {% endmacro %} 27 | 28 | {% macro query_results(msg) %} 29 | {%- if msg.normal_query_results | length > 0 or msg.falsey_query_results | length > 0 %} 30 | {# new line #} 31 |
QUERIES
32 | {# new line #} 33 | {%- endif %} 34 | 35 | {%- if msg.normal_query_results | length > 0 %} 36 | {%- for query in (msg.normal_query_results) %} 37 | {%- if query.query.name %} 38 | - {{ query.query.name }} 39 | {%- else %} 40 | - Query {{ loop.index }} 41 | {%- endif %} 42 | Result: {{ query.result }} 43 | {%- if verbose %} 44 | Source: {{ query.source }} 45 | {%- endif %} 46 | {# new line #} 47 | {%- endfor %} 48 | {%- endif %} 49 | 50 | {%- if (verbose and (msg.falsey_query_results | length > 0)) or msg.falsey_query_results|length == 1 %} 51 | {%- for query in (msg.falsey_query_results) %} 52 | {%- if query.query.name %} 53 | - {{ query.query.name }} 54 | {%- else %} 55 | - Query {{ loop.index }} 56 | {%- endif %} 57 | Result: {{ query.result }} 58 | {%- if verbose %} 59 | Source: {{ query.source }} 60 | {# new line #} 61 | {%- endif %} 62 | {%- endfor %} 63 | {%- endif %} 64 | 65 | {%- if msg.failed_query_results|length > 0 %} 66 | {# new line #} 67 | {# new line #} 68 |
FAILED QUERIES
69 | {# new line #} 70 | {%- for query in msg.failed_query_results %} 71 | {%- if query.query.name %} 72 | - {{ query.query.name }} 73 | {%- else %} 74 | - Query {{ loop.index }} 75 | {%- endif %} 76 | Error: {{ query.error }} 77 | {%- if verbose %} 78 | Source: {{ query.source }} 79 | {# new line #} 80 | {%- endif %} 81 | {%- endfor %} 82 | {%- endif %} 83 | {% endmacro %} 84 | -------------------------------------------------------------------------------- /src/sublime/cli/templates/me_result.txt.j2: -------------------------------------------------------------------------------- 1 | ╔═══════════════════════════╗ 2 | ║
{{ "{:^25}".format("Authenticated User") }}
║ 3 | ╚═══════════════════════════╝ 4 | 5 | Full Name: {{ result.first_name }} {{ result.last_name }} 6 | Email address: {{ result.email_address }} 7 | Organization: {{ result.org_name }} 8 | {%- if verbose %} 9 | User ID: {{ result.id }} 10 | Organization ID: {{ result.org_id }} 11 | {%- endif %} 12 | -------------------------------------------------------------------------------- /src/sublime/cli/templates/message_data_model.txt.j2: -------------------------------------------------------------------------------- 1 | Message ID: {{ results.message_id }} 2 | -------------------------------------------------------------------------------- /src/sublime/error.py: -------------------------------------------------------------------------------- 1 | """Sublime API client errors.""" 2 | 3 | 4 | class SublimeError(Exception): 5 | def __init__( 6 | self, 7 | message=None, 8 | status_code=None, 9 | headers=None 10 | ): 11 | super(SublimeError, self).__init__(message) 12 | 13 | self._message = message 14 | self.status_code = status_code 15 | self.headers = headers or {} 16 | self.request_id = self.headers.get("x-request-id", None) 17 | 18 | def __str__(self): 19 | msg = self._message or "" 20 | if self.request_id is not None: 21 | return u"Request {0}: {1}".format(self.request_id, msg) 22 | else: 23 | return msg 24 | 25 | @property 26 | def message(self): 27 | return self._message 28 | 29 | def __repr__(self): 30 | return "%s(message=%r, http_status=%r, request_id=%r)" % ( 31 | self.__class__.__name__, 32 | self._message, 33 | self.status_code, 34 | self.request_id, 35 | ) 36 | 37 | 38 | class InvalidRequestError(SublimeError): 39 | """Invalid request (HTTP 400 or 404).""" 40 | 41 | 42 | class RateLimitError(SublimeError): 43 | """API rate limit exceeded.""" 44 | 45 | 46 | class APIError(SublimeError): 47 | """All other failed requests.""" 48 | 49 | 50 | class AuthenticationError(SublimeError): 51 | """Invalid or missing authentiction.""" 52 | 53 | 54 | class LoadRuleError(SublimeError): 55 | """Error loading rules file.""" 56 | 57 | 58 | class LoadMessageDataModelError(SublimeError): 59 | """Error loading Message Data Model file.""" 60 | 61 | 62 | class LoadEMLError(SublimeError): 63 | """Error loading .EML.""" 64 | 65 | 66 | class LoadMSGError(SublimeError): 67 | """Error loading .MSG.""" 68 | 69 | 70 | class LoadMBOXError(SublimeError): 71 | """Error loading .MBOX.""" 72 | -------------------------------------------------------------------------------- /src/sublime/util.py: -------------------------------------------------------------------------------- 1 | """Utility and helper functions.""" 2 | 3 | import os 4 | import socket 5 | import sys 6 | import email 7 | import mailbox 8 | import base64 9 | import json 10 | 11 | import yaml 12 | import click 13 | import structlog 14 | import msg_parser 15 | from halo import Halo 16 | from six.moves.configparser import ConfigParser 17 | from pathlib import Path 18 | 19 | from sublime.error import * 20 | 21 | CONFIG_FILE = os.path.expanduser(os.path.join("~", ".config", "sublime", "setup.cfg")) 22 | LOGGER = structlog.get_logger() 23 | 24 | DEFAULT_CONFIG = {"api_key": "", "save_dir": "", "permission": ""} 25 | 26 | CONFIRMATION_MESSAGE_GENERIC = """ 27 | Messages will be sent to Sublime Security servers in order to be processed. 28 | 29 | This message is intended to preserve your privacy. You only need to accept once. 30 | 31 | Would you like to continue? 32 | 33 | """ 34 | 35 | CONFIRMATION_MESSAGE_ANALYZE = """ 36 | Messages will be sent to Sublime Security servers in order to run rules and queries. 37 | 38 | This message is intended to preserve your privacy. You only need to accept once. 39 | 40 | Would you like to continue? 41 | 42 | """ 43 | 44 | def load_config(): 45 | """Load configuration. 46 | 47 | :returns: 48 | Current configuration based on configuration file and environment variables. 49 | :rtype: dict 50 | 51 | """ 52 | config_parser = ConfigParser( 53 | {key: str(value) for key, value in DEFAULT_CONFIG.items()} 54 | ) 55 | config_parser.add_section("sublime") 56 | 57 | if os.path.isfile(CONFIG_FILE): 58 | # LOGGER.debug("Parsing configuration file: %s..." % CONFIG_FILE) 59 | with open(CONFIG_FILE) as config_file: 60 | config_parser.readfp(config_file) 61 | else: 62 | # LOGGER.debug("Configuration file not found: %s" % CONFIG_FILE) 63 | pass 64 | 65 | if "SUBLIME_API_KEY" in os.environ: 66 | api_key = os.environ["SUBLIME_API_KEY"] 67 | # LOGGER.debug("API key found in environment variable: %s", api_key, api_key=api_key) 68 | # Environment variable takes precedence over configuration file content 69 | config_parser.set("sublime", "api_key", api_key) 70 | 71 | if "SUBLIME_SAVE_DIR" in os.environ: 72 | save_dir = os.environ["SUBLIME_SAVE_DIR"] 73 | # LOGGER.debug("Save dir found in environment variable: %s", save_dir, save_dir=save_dir) 74 | # Environment variable takes precedence over configuration file content 75 | config_parser.set("sublime", "save_dir", save_dir) 76 | 77 | return { 78 | "api_key": config_parser.get("sublime", "api_key"), 79 | "save_dir": config_parser.get("sublime", "save_dir"), 80 | "permission": config_parser.get("sublime", "permission"), 81 | } 82 | 83 | 84 | def save_config(config): 85 | """Save configuration. 86 | 87 | :param config: Data to be written to the configuration file. 88 | :type config: dict 89 | 90 | """ 91 | config_parser = ConfigParser() 92 | config_parser.add_section("sublime") 93 | 94 | if len(config) == 0: 95 | click.echo('Error: no options provided. Try "sublime setup -h" for help.') 96 | click.get_current_context().exit(-1) 97 | 98 | # If either value was not specified, load the existing values saved 99 | # to ensure we don't overwrite their values to null here 100 | saved_config = load_config() 101 | if 'api_key' not in config or not config['api_key']: 102 | config['api_key'] = saved_config['api_key'] 103 | if 'save_dir' not in config or not config['save_dir']: 104 | config['save_dir'] = saved_config['save_dir'] 105 | if 'permission' not in config or not config['permission']: 106 | config['permission'] = saved_config['permission'] 107 | 108 | if config["save_dir"] and not os.path.isdir(config["save_dir"]): 109 | click.echo("Error: save directory is not a valid directory") 110 | click.get_current_context().exit(-1) 111 | 112 | config_parser.set("sublime", "api_key", config["api_key"]) 113 | config_parser.set("sublime", "save_dir", config["save_dir"]) 114 | config_parser.set("sublime", "permission", config["permission"]) 115 | 116 | config_parser_existing = ConfigParser() 117 | if os.path.isfile(CONFIG_FILE): 118 | # LOGGER.debug("Reading configuration file: %s...", CONFIG_FILE, path=CONFIG_FILE) 119 | with open(CONFIG_FILE) as config_file: 120 | config_parser_existing.readfp(config_file) 121 | 122 | # if an emailrep key exists, ensure we don't overwrite it 123 | try: 124 | emailrep_key = config_parser_existing.get("emailrep", "key") 125 | if emailrep_key: 126 | config_parser.add_section("emailrep") 127 | config_parser.set("emailrep", "key", emailrep_key) 128 | except: 129 | pass 130 | 131 | config_dir = os.path.dirname(CONFIG_FILE) 132 | if not os.path.isdir(config_dir): 133 | os.makedirs(config_dir) 134 | 135 | with open(CONFIG_FILE, "w") as config_file: 136 | config_parser.write(config_file) 137 | 138 | 139 | def request_permission(subcommand, api_key=None): 140 | config = load_config() 141 | permission = config['permission'] 142 | if not permission or permission != "True": 143 | from sublime.api import Sublime 144 | sublime_client = Sublime(api_key) 145 | 146 | message = CONFIRMATION_MESSAGE_ANALYZE if subcommand == "analyze" \ 147 | else CONFIRMATION_MESSAGE_GENERIC 148 | 149 | if click.confirm(message): 150 | config['permission'] = "True" 151 | save_config(config) 152 | sublime_client.privacy_ack(True) 153 | else: 154 | sublime_client.privacy_ack(False) 155 | print("Aborted!") 156 | sys.exit() 157 | 158 | 159 | def load_eml(input_file): 160 | """Load .EML file. 161 | 162 | :param input_file: Path to file. 163 | :type input_file: str 164 | :returns: Base64-encoded raw content 165 | :rtype: string 166 | :raises: LoadEMLError 167 | 168 | """ 169 | with open(input_file) as f: 170 | return load_eml_file_handle(f) 171 | 172 | 173 | def load_eml_file_handle(input_file): 174 | """Load .EML file. 175 | 176 | :param input_file: File handle. 177 | :type input_file: _io.TextIOWrapper 178 | :returns: Base64-encoded raw content 179 | :rtype: string 180 | :raises: LoadEMLError 181 | 182 | """ 183 | if input_file is None: 184 | raise LoadEMLError("Missing .eml file") 185 | 186 | try: 187 | message = email.message_from_file(input_file) 188 | raw_message_base64 = base64.b64encode( 189 | message.as_string().encode('utf-8')).decode('ascii') 190 | 191 | # fails for utf-8 messages: 192 | # decoded = base64.b64encode(message.as_bytes()).decode('ascii') 193 | except Exception as exception: 194 | error_message = "{}".format(exception) 195 | raise LoadEMLError(error_message) 196 | 197 | return raw_message_base64 198 | 199 | 200 | def load_msg(filepath): 201 | """Load .MSG file. 202 | 203 | :param filepath: Path to file. 204 | :type filepath: str 205 | :returns: Base64-encoded raw content 206 | :rtype: string 207 | :raises: LoadMSGError 208 | 209 | """ 210 | 211 | with open(filepath) as f: 212 | return load_msg_file_handle(f) 213 | 214 | 215 | def load_msg_file_handle(input_file): 216 | """Load .MSG file. 217 | 218 | :param input_file: File handle. 219 | :type input_file: _io.TextIOWrapper 220 | :returns: Base64-encoded raw content 221 | :rtype: string 222 | :raises: LoadMSGError 223 | 224 | """ 225 | if input_file is None: 226 | raise LoadMSGError("Missing .msg file") 227 | 228 | try: 229 | msg_obj = msg_parser.MsOxMessage(input_file.name) 230 | email_formatter = msg_parser.email_builder.EmailFormatter(msg_obj) 231 | message_str = email_formatter.build_email() 232 | 233 | raw_message_base64 = base64.b64encode( 234 | message_str.encode('utf-8')).decode('ascii') 235 | except Exception as exception: 236 | error_message = "{}".format(exception) 237 | raise LoadMSGError(error_message) 238 | 239 | return raw_message_base64 240 | 241 | def load_mbox(filepath, halo=None): 242 | """Load .MBOX file. 243 | 244 | :param filepath: Path to file. 245 | :type filepath: str 246 | :returns: Base64-encoded raw content 247 | :rtype: map of (key: subject+index, value: raw_message) 248 | :raises: LoadMBOXError 249 | 250 | """ 251 | if halo: 252 | _, _, file_name = filepath.rpartition('/') 253 | halo.text = f"Loading ({file_name}) this may take a while..." 254 | 255 | raw_messages = {} 256 | mbox = mailbox.mbox(filepath) 257 | num_messages = len(mbox) 258 | 259 | for i in range(num_messages): 260 | message = mbox[i] 261 | if halo: 262 | halo.text = f"Encoding ({file_name}) message {i+1} of {num_messages}" 263 | 264 | try: 265 | # identify a suitable key for this message 266 | instance = 0 267 | subject = message['subject'] or "[Empty Subject]" 268 | key = subject 269 | while key in raw_messages: 270 | instance += 1 271 | key = subject + f" ({instance})" 272 | 273 | # encode the raw message and return 274 | raw_message = message.as_string().encode('utf-8') 275 | raw_messages[key] = base64.b64encode(raw_message).decode('ascii') 276 | 277 | except Exception as exception: 278 | if halo: halo.stop() 279 | LOGGER.warning(f"failed to decode {key}: {exception}") 280 | if halo: halo.start() 281 | # raise LoadMBOXError(error_message) 282 | 283 | return raw_messages 284 | 285 | def load_message_data_model(filepath): 286 | """Load Message Data Model file. 287 | 288 | :param filepath: Path to file. 289 | :type filepath: str 290 | :returns: Message Data Model JSON object 291 | :rtype: dict 292 | :raises: LoadMessageDataModelError 293 | 294 | """ 295 | with open(filepath) as f: 296 | return load_message_data_model_file_handle(f) 297 | 298 | 299 | def load_message_data_model_file_handle(input_file): 300 | """Load Message Data Model file. 301 | 302 | :param input_file: File handle. 303 | :type input_file: _io.TextIOWrapper 304 | :returns: Message Data Model JSON object 305 | :rtype: dict 306 | :raises: LoadMessageDataModelError 307 | 308 | """ 309 | if input_file is None: 310 | raise LoadMessageDataModelError("Missing Message Data Model file") 311 | 312 | try: 313 | message_data_model = json.load(input_file) 314 | except Exception as exception: 315 | error_message = "{}".format(exception) 316 | raise LoadMessageDataModelError(error_message) 317 | 318 | return message_data_model 319 | 320 | 321 | def load_yml_path(files_path, ignore_errors=True): 322 | """Load rules and queries from a path. 323 | 324 | :param files_path: Path to YML files 325 | :type files_path: string 326 | :param ignore_errors: Ignore file loading errors 327 | :type ignore_errors: boolean 328 | :returns: A list of rules and a list of queries 329 | :rtype: list, list 330 | :raises: LoadRuleError 331 | 332 | """ 333 | # gather all rules files 334 | sqar_files = [] 335 | for file in Path(files_path).rglob("*.yml"): 336 | sqar_files.append(file) 337 | for file in Path(files_path).rglob("*.yaml"): 338 | sqar_files.append(file) 339 | 340 | # get all rules and queries from 341 | rules, queries = [], [] 342 | for file in sqar_files: 343 | with file.open(encoding='utf-8') as f: 344 | try: 345 | rules_tmp, queries_tmp = load_yml(f) 346 | if rules_tmp: 347 | rules.extend(rules_tmp) 348 | if queries_tmp: 349 | queries.extend(queries_tmp) 350 | except LoadRuleError as error: 351 | # Ignore errors and continue reading the remaining files. 352 | if ignore_errors: 353 | LOGGER.warning(error.message) 354 | else: 355 | raise 356 | 357 | if len(rules) == 0 and len(queries) == 0: 358 | LOGGER.warning(f"No valid YAML files found in {files_path}") 359 | 360 | return rules, queries 361 | 362 | 363 | def load_yml(yml_file, ignore_errors=True): 364 | """Load rules and queries from a file. 365 | 366 | :param yml_file: YML file 367 | :type yml_file: _io.TextIOWrapper 368 | :param ignore_errors: Ignore loading errors 369 | :type ignore_errors: boolean 370 | :returns: A list of rules and a list of queries 371 | :rtype: list, list 372 | :raises: LoadRuleError 373 | 374 | """ 375 | if yml_file is None: 376 | if ignore_errors: 377 | LOGGER.warning("Missing YML file") 378 | return [], [] 379 | else: 380 | raise LoadRuleError("Missing YML file") 381 | 382 | try: 383 | rules_and_queries_yaml = yaml.load(yml_file, Loader=yaml.SafeLoader) 384 | if not rules_and_queries_yaml or not isinstance(rules_and_queries_yaml, dict): 385 | if ignore_errors: 386 | LOGGER.warning("Invalid YML file") 387 | return [], [] 388 | else: 389 | raise LoadRuleError("Invalid YML file") 390 | except yaml.scanner.ScannerError as e: 391 | error = """File '{}' contains invalid characters: {}""".format(yml_file.name, e) 392 | raise LoadRuleError(error) 393 | except Exception as e: 394 | raise LoadRuleError(e) 395 | 396 | rules_yaml, queries_yaml = rules_and_queries_yaml.get("rules", []), rules_and_queries_yaml.get("queries", []) 397 | rules, queries = [], [] 398 | 399 | if not rules_yaml and not queries_yaml: 400 | rule_or_query_yaml = rules_and_queries_yaml 401 | 402 | # default to query 403 | if "type" not in rule_or_query_yaml: 404 | rule_or_query_yaml["type"] = "query" 405 | 406 | if rule_or_query_yaml.get("type") not in ["rule", "query"]: 407 | error_str = f'Invalid type in {yml_file.name}' 408 | if ignore_errors: 409 | LOGGER.warning(error_str) 410 | return [], [] 411 | raise LoadRuleError(error_str) 412 | 413 | if rule_or_query_yaml.get("type") == "rule": 414 | rules_yaml.append(rule_or_query_yaml) 415 | else: 416 | queries_yaml.append(rule_or_query_yaml) 417 | 418 | def safe_yaml_filter(yaml_dict): 419 | if not yaml_dict.get("source"): 420 | error_str = f"Missing source in '{yml_file.name}'" 421 | raise LoadRuleError(error_str) 422 | return { 423 | "source": yaml_dict.get("source"), 424 | "name": yaml_dict.get("name"), 425 | "severity": yaml_dict.get("severity"), 426 | } 427 | 428 | for rule_yaml in rules_yaml: 429 | if not isinstance(rule_yaml, dict): 430 | error_str = f"Invalid list of rules" 431 | raise LoadRuleError(error_str) 432 | 433 | rule = safe_yaml_filter(rule_yaml) 434 | 435 | if rule: 436 | rules.append(rule) 437 | 438 | for query_yaml in queries_yaml: 439 | if not isinstance(query_yaml, dict): 440 | error_str = f"Invalid list of queries" 441 | raise LoadRuleError(error_str) 442 | 443 | query = safe_yaml_filter(query_yaml) 444 | 445 | if query: 446 | queries.append(query) 447 | 448 | return rules, queries 449 | 450 | 451 | def get_datetime_formats(): 452 | formats=[ 453 | '%Y-%m-%d', 454 | '%Y-%m-%dT%H:%M:%S', 455 | '%Y-%m-%dT%H:%M:%S.%f%z', 456 | '%Y-%m-%d %H:%M:%S' 457 | ] 458 | 459 | return formats 460 | --------------------------------------------------------------------------------