├── plugins ├── __init__.py ├── helpers │ ├── __init__.py │ ├── common.py │ ├── plugin.py │ ├── basic_info.py │ └── writer.py ├── local_login.py ├── volume_mount.py ├── persistence.py ├── file_download.py ├── remote_login.py └── prog_exec.py ├── .gitattributes ├── .gitignore ├── images ├── demo_scenario.png └── demo_timeline.png ├── helper_tools ├── aul2madb │ ├── Cargo.toml │ ├── src │ │ └── main.rs │ └── Cargo.lock ├── README.md └── ndjson2madb.py ├── LICENSE ├── README.md └── ma2tl.py /plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /plugins/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .history/ 2 | *.pyc 3 | .DS_Store 4 | .vscode/ 5 | helper_tools/aul2madb/target/ 6 | -------------------------------------------------------------------------------- /images/demo_scenario.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mnrkbys/ma2tl/HEAD/images/demo_scenario.png -------------------------------------------------------------------------------- /images/demo_timeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mnrkbys/ma2tl/HEAD/images/demo_timeline.png -------------------------------------------------------------------------------- /helper_tools/aul2madb/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "aul2madb" 3 | version = "0.0.2" 4 | authors = ["Minoru Kobayashi "] 5 | description = "Apple Unified Logs converter for ma2tl" 6 | edition = "2021" 7 | 8 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 9 | 10 | [dependencies] 11 | macos-unifiedlogs = {git = "https://github.com/mandiant/macos-UnifiedLogs"} 12 | clap = { version = "4.*", features = ["derive"]} 13 | csv = "1.*" 14 | chrono = "0.4.*" 15 | simplelog = "0.12.1" 16 | rusqlite = "*" 17 | # csv = "1.2.1" 18 | # log = "0.4.17" 19 | dunce = "1.*" 20 | -------------------------------------------------------------------------------- /plugins/helpers/common.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2021 Minoru Kobayashi 3 | # 4 | # This file is part of ma2tl. 5 | # Usage or distribution of this code is subject to the terms of the MIT License. 6 | # 7 | 8 | import datetime 9 | 10 | 11 | def convert_apfs_time(timestamp): 12 | try: 13 | return datetime.datetime(1970, 1, 1) + datetime.timedelta(microseconds=timestamp/1000) 14 | except Exception: 15 | return None 16 | 17 | 18 | def get_timedelta(ts1, ts2): 19 | try: 20 | dt1 = datetime.datetime.strptime(ts1, '%Y-%m-%d %H:%M:%S.%f') 21 | except ValueError: 22 | dt1 = datetime.datetime.strptime(ts1 + '.000000', '%Y-%m-%d %H:%M:%S.%f') 23 | 24 | try: 25 | dt2 = datetime.datetime.strptime(ts2, '%Y-%m-%d %H:%M:%S.%f') 26 | except ValueError: 27 | dt2 = datetime.datetime.strptime(ts2 + '.000000', '%Y-%m-%d %H:%M:%S.%f') 28 | 29 | return abs(dt2 - dt1).total_seconds() 30 | 31 | 32 | if __name__ == '__main__': 33 | print('This file is part of forensic timeline generator "ma2tl". So, it cannot run separately.') 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Minoru Kobayashi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ma2tl (mac_apt to timeline) 2 | 3 | This is a DFIR tool for generating a macOS forensic timeline from the analysis result DBs of [mac_apt](https://github.com/ydkhatri/mac_apt). 4 | 5 | ## Requirements 6 | 7 | - Python 3.7.0 or later 8 | - pytz 9 | - tzlocal 10 | - xlsxwriter 11 | 12 | ## Installation 13 | 14 | ```Shell 15 | % git clone https://github.com/mnrkbys/ma2tl.git 16 | ``` 17 | 18 | ## Usage 19 | 20 | ```Shell 21 | % python ./ma2tl.py -h 22 | usage: ma2tl.py [-h] [-i INPUT] [-o OUTPUT] [-ot OUTPUT_TYPE] [-s START] [-e END] [-t TIMEZONE] [-l LOG_LEVEL] plugin [plugin ...] 23 | 24 | Forensic timeline generator using mac_apt analysis results. Supports only SQLite DBs. 25 | 26 | positional arguments: 27 | plugin Plugins to run (space separated). 28 | 29 | optional arguments: 30 | -h, --help show this help message and exit 31 | -i INPUT, --input INPUT 32 | Path to a folder that contains mac_apt DBs. 33 | -o OUTPUT, --output OUTPUT 34 | Path to a folder to save ma2tl result. 35 | -ot OUTPUT_TYPE, --output_type OUTPUT_TYPE 36 | Specify the output file type: SQLITE, XLSX, TSV (Default: SQLITE) 37 | -s START, --start START 38 | Specify start timestamp. (ex. 2021-11-05 08:30:00) 39 | -e END, --end END Specify end timestamp. 40 | -t TIMEZONE, --timezone TIMEZONE 41 | Specify Timezone: "UTC", "Asia/Tokyo", "US/Eastern", etc (Default: System Local Timezone) 42 | -l LOG_LEVEL, --log_level LOG_LEVEL 43 | Specify log level: INFO, DEBUG, WARNING, ERROR, CRITICAL (Default: INFO) 44 | 45 | The following 4 plugins are available: 46 | FILE_DOWNLOAD Extract file download activities. 47 | PERSISTENCE Extract persistence settings. 48 | PROG_EXEC Extract program execution activities. 49 | VOLUME_MOUNT Extract volume mount/unmount activities. 50 | ---------------------------------------------------------------------------- 51 | ALL Run all plugins 52 | ``` 53 | 54 | ## Generated timeline example 55 | 56 | ![Scenario](images/demo_scenario.png) 57 | ![Timeline](images/demo_timeline.png) 58 | 59 | ## Notes 60 | Unfortunately, the latest version of mac_apt cannot parse Unified Logs files correctly. So you have to replace UnifiedLogs.db with a database created by [helper tools](https://github.com/mnrkbys/ma2tl/tree/main/helper_tools). 61 | 62 | ## Presentation 63 | 64 | This tool was published on [Japan Security Analyst Conference 2022](https://jsac.jpcert.or.jp/en/index.html) (JSAC2022). 65 | 66 | Slides are available below: 67 | 68 | - [Japanese version](https://jsac.jpcert.or.jp/archive/2022/pdf/JSAC2022_2_kobayashi_jp.pdf) 69 | - [English version](https://jsac.jpcert.or.jp/archive/2022/pdf/JSAC2022_2_kobayashi_en.pdf) 70 | 71 | ## Author 72 | 73 | [Minoru Kobayashi](https://twitter.com/unkn0wnbit) 74 | 75 | ## License 76 | 77 | [MIT](http://opensource.org/licenses/mit-license.php) 78 | -------------------------------------------------------------------------------- /plugins/helpers/plugin.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2021 Minoru Kobayashi 3 | # 4 | # This file is part of ma2tl. 5 | # Usage or distribution of this code is subject to the terms of the MIT License. 6 | # 7 | # -------------------------------------------------- 8 | # This code is based on mac_apt's plugin.py 9 | # 10 | 11 | import logging 12 | import os 13 | import sys 14 | import traceback 15 | from importlib import import_module 16 | 17 | 18 | def import_plugins(plugins): 19 | plugin_path = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "plugins") 20 | sys.path.append(plugin_path) 21 | imported_plugin_name = [] 22 | 23 | try: 24 | dir_list = os.listdir(plugin_path) 25 | for filename in dir_list: 26 | if filename.endswith(".py") and not filename.startswith("_"): 27 | try: 28 | plugin = import_module(filename.replace(".py", "")) 29 | if _check_plugin_validation(plugin): 30 | if plugin.PLUGIN_NAME not in imported_plugin_name: 31 | plugins.append(plugin) 32 | imported_plugin_name.append(plugin.PLUGIN_NAME) 33 | else: 34 | print(f"Failed to import plugin - {filename} : Plugin name {plugin.PLUGIN_NAME} is already in use. This plugin is skipped.") 35 | else: 36 | print(f"Failed to import plugin - {filename} : Plugin is missing a required variable") 37 | 38 | except Exception as ex: 39 | print(f"Failed to import plugin - {filename}") 40 | print(f"Plugin import exception details: {str(ex)}") 41 | continue 42 | 43 | except Exception as ex: 44 | print("Error: Does plugin directory exist?") 45 | print(f"Exception details: {str(ex)}") 46 | 47 | plugins.sort(key=lambda plugin: plugin.PLUGIN_NAME) 48 | return len(plugins) 49 | 50 | 51 | def _check_plugin_validation(plugin): 52 | for attr in ('PLUGIN_NAME', 'PLUGIN_DESCRIPTION', 'PLUGIN_ACTIVITY_TYPE', 'PLUGIN_AUTHOR', 'PLUGIN_AUTHOR_EMAIL'): 53 | try: 54 | _ = getattr(plugin, attr) 55 | except Exception: 56 | print(f"Plugin {plugin} does not have {attr}.") 57 | return False 58 | 59 | return True 60 | 61 | 62 | def check_user_specified_plugin_name(plugins_to_run, plugins): 63 | for user_specified_plugin in plugins_to_run: 64 | found = False 65 | for plugin in plugins: 66 | if plugin.PLUGIN_NAME == user_specified_plugin: 67 | found = True 68 | break 69 | 70 | if not found: 71 | print(f"Error: Plugin name not found : {user_specified_plugin}") 72 | return False 73 | 74 | return True 75 | 76 | 77 | def setup_logger(log_file_path, name, log_level=logging.INFO): 78 | try: 79 | logger = logging.getLogger(name) 80 | 81 | log_file_handler = logging.FileHandler(log_file_path, encoding='UTF-8') 82 | log_file_format = logging.Formatter('%(asctime)s|%(name)s|%(levelname)s|%(message)s', datefmt='%Y-%m-%d %H:%M:%S') 83 | log_file_handler.setFormatter(log_file_format) 84 | logger.addHandler(log_file_handler) 85 | 86 | log_console_handler = logging.StreamHandler() 87 | log_console_handler.setLevel(log_level) 88 | log_console_format = logging.Formatter('%(name)s-%(levelname)s-%(message)s') 89 | log_console_handler.setFormatter(log_console_format) 90 | logger.addHandler(log_console_handler) 91 | 92 | except Exception as ex: 93 | print("Error while trying to create log file\nError Details:\n") 94 | traceback.print_exc() 95 | sys.exit("Program aborted..could not create log file!") 96 | 97 | return logger 98 | 99 | 100 | if __name__ == '__main__': 101 | print('This file is part of forensic timeline generator "ma2tl". So, it cannot run separately.') 102 | -------------------------------------------------------------------------------- /helper_tools/README.md: -------------------------------------------------------------------------------- 1 | # ma2tl Helper Tools 2 | 3 | Unfortunately, the current version of mac_apt is not able to parse Unified Logs correctly since macOS 11. However, there are several measures against this problem. 4 | 5 | ## Measure 1 (aul2madb) 6 | 7 | As mentioned above, the current version of mac_apt cannot parse Unified Logs. On the other hand, exporting Unified Logs can be done correctly. So, we can parse the exported Unified Logs with other tools. For example, [macos-UnifiedLogs](https://github.com/mandiant/macos-UnifiedLogs) can parse them which come from macOS 10.12 or later. 8 | 9 | aul2madb uses macos-UnifiedLogs as a library and can convert the exported Unified Logs by mac_apt to the database in the same format as mac_apt. macos-UnifiedLogs is written in Rust. Therefore, this tool is also written in Rust, not Python. 10 | 11 | ### Building aul2madb 12 | 13 | ```zsh 14 | % cd aul2madb 15 | % cargo build --release 16 | ``` 17 | 18 | ### Help of aul2madb 19 | 20 | ```zsh 21 | % ./target/release/aul2madb -h 22 | Unified Logs converter for ma2tl 23 | 24 | Usage: aul2madb [OPTIONS] --input 25 | 26 | Options: 27 | -i, --input Path to a logarchive or to a directory that contains exported Unified Logs 28 | -f, --output-format Output format [default: sqlite] [possible values: sqlite, tsv] 29 | -o, --output Path to output file [default: ./UnifiedLogs.db] 30 | -h, --help Print help 31 | -V, --version Print version 32 | ``` 33 | 34 | ### How to run aul2madb 35 | 36 | ```zsh 37 | % ./target/release/aul2madb --input ~/Desktop/system_logs.logarchive --output-format sqlite --output UnifiedLogs.db 38 | Staring Unified Logs converter... 39 | Processing as a logarchive. 40 | Parsing: /Users/macforensics/Desktop/system_logs.logarchive/Persist/0000000000000fbf.tracev3 41 | Parsing: /Users/macforensics/Desktop/system_logs.logarchive/Persist/0000000000000fcf.tracev3 42 | Parsing: /Users/macforensics/Desktop/system_logs.logarchive/Persist/0000000000000fa9.tracev3 43 | Parsing: /Users/macforensics/Desktop/system_logs.logarchive/Persist/0000000000000fd4.tracev3 44 | Parsing: /Users/macforensics/Desktop/system_logs.logarchive/Persist/0000000000000fb6.tracev3 45 | (snip) 46 | ``` 47 | 48 | ## Measure 2 (ndjson2madb.py) 49 | 50 | Actually, the log command of macOS can display log entries as ndjson with a lot of attributions. The ndjson data contains almost the same information as the database created by mac_apt. Therefore, ndjson2madb.py can convert the ndjson data to the database in the same format as mac_apt. 51 | 52 | ### Installing required packages 53 | 54 | ```zsh 55 | % pip3 install ndjson 56 | ``` 57 | 58 | ### Help of ndjson2madb.py 59 | 60 | ```zsh 61 | % python3 ./ndjson2madb.py -h 62 | usage: ndjson2madb.py [-h] [-i INPUT] -o OUTPUT 63 | 64 | Convert the exported Unified Logs with ndjson style to mac_apt UnifiedLogs.db. 65 | 66 | options: 67 | -h, --help show this help message and exit 68 | -i INPUT, --input INPUT 69 | Path to an exported Unified Logs file (Default: - (STDIN)) 70 | -o OUTPUT, --output OUTPUT 71 | Path to an output database file (Default: UnifiedLogs.db) 72 | 73 | [Exporting Unified Logs Tips] 74 | Exporting all entries of Unified Logs takes a lot of disk space. I recommend using zip command along with to reduce the file size. 75 | % log show --info --debug --style ndjson --timezone 'UTC' | zip ~/Desktop/unifiedlogs_ndjson.zip - 76 | 77 | Zipped file can be converted to a database like below: 78 | % unzip -q -c ~/Desktop/unifiedlogs_ndjson.zip | python3 ./ndjson2ma.py -o ./UnifiedLogs.db 79 | 80 | [Timezone] 81 | This script does NOT consider timezone. So, you need to run the log command like below: 82 | % log show --info --debug --style ndjson --timezone 'UTC' > /path/to/unifiedlogs.ndjson 83 | ``` 84 | 85 | ### How to run ndjson2madb.py 86 | 87 | ``` 88 | % log show --info --debug --style ndjson --timezone 'UTC' | zip ~/Desktop/unifiedlogs_ndjson.zip - 89 | % unzip -q -c ~/Desktop/unifiedlogs_ndjson.zip | python3 ./ndjson2ma.py -o ./UnifiedLogs.db 90 | ``` 91 | -------------------------------------------------------------------------------- /plugins/local_login.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2023 Minoru Kobayashi 3 | # 4 | # This file is part of ma2tl. 5 | # Usage or distribution of this code is subject to the terms of the MIT License. 6 | # 7 | 8 | from __future__ import annotations 9 | 10 | import logging 11 | import os 12 | import re 13 | 14 | from plugins.helpers.basic_info import BasicInfo, MacAptDBType 15 | 16 | PLUGIN_NAME = os.path.splitext(os.path.basename(__file__))[0].upper() 17 | PLUGIN_DESCRIPTION = "Extract local login activities." 18 | PLUGIN_ACTIVITY_TYPE = "Local Login" 19 | PLUGIN_VERSION = "20230830" 20 | PLUGIN_AUTHOR = "Minoru Kobayashi" 21 | PLUGIN_AUTHOR_EMAIL = "unknownbit@gmail.com" 22 | 23 | log = None 24 | 25 | 26 | # Extract local authentication, OS restart, and OS shutdown logs 27 | # This function is confirmed to work correctly for macOS 13+ 28 | def extract_local_authentication(basic_info: BasicInfo, timeline_events: list) -> bool: 29 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 30 | return False 31 | 32 | run_query = basic_info.mac_apt_dbs.run_query 33 | start_ts, end_ts = basic_info.get_between_dates_utc() 34 | sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 35 | (ProcessName == "loginwindow" AND \ 36 | Message LIKE "-[SessionAgentNotificationCenter %" AND \ 37 | Message LIKE "%sendDistributedNotification%" AND \ 38 | NOT (\ 39 | Message LIKE "%com.apple.system.sessionagent.sessionstatechanged%" OR \ 40 | Message LIKE "%com.apple.system.loginwindow.likely%"\ 41 | )\ 42 | ) \ 43 | ORDER BY TimeUtc;' 44 | regex = r'^-\[SessionAgentNotificationCenter .+ \| .+: (?P.+), with userID:(?P\d+)' 45 | actions = { 46 | 'sessionDidLogin': 'Logged in', 47 | 'screenIsLocked': 'Screen is locked', 48 | 'screenIsUnlocked': 'Screen is unlocked', 49 | 'sessionDidMoveOffConsole': 'Moved to Fast User Switching mode', 50 | 'sessionDidMoveOnConsole': 'Moved from Fast User Switching mode', 51 | 'logoutInitiated': 'Logout initiated', 52 | 'restartInitiated': 'OS restart initiated', 53 | 'shutdownInitiated': 'OS shutdown initiated', 54 | 'logoutCancelled': 'Cancelled', 55 | 'logoutContinued': 'Continued' 56 | } 57 | 58 | state = "" 59 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql): 60 | if result := re.match(regex, row['Message']): 61 | for action in actions.keys(): 62 | msg = "" 63 | if result['notified_action'].endswith(action): 64 | if action == 'logoutInitiated': 65 | state = 'logout' 66 | elif action == 'restartInitiated': 67 | state = 'OS restart' 68 | elif action == 'shutdownInitiated': 69 | state = 'OS shutdown' 70 | 71 | if state and action in ('logoutCancelled', 'logoutContinued'): 72 | msg = f"{actions[action]} {state} with uid={result['uid']}" 73 | state = "" 74 | else: 75 | msg = f"{actions[action]} with uid={result['uid']}" 76 | 77 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 78 | timeline_events.append(event) 79 | break 80 | 81 | return True 82 | 83 | 84 | def run(basic_info: BasicInfo) -> bool: 85 | global log 86 | log = logging.getLogger(basic_info.output_params.logger_root + '.PLUGINS.' + PLUGIN_NAME) 87 | timeline_events = [] 88 | extract_local_authentication(basic_info, timeline_events) 89 | 90 | log.info(f"Detected {len(timeline_events)} events.") 91 | if len(timeline_events) > 0: 92 | basic_info.data_writer.write_data_rows(timeline_events) 93 | return True 94 | 95 | return False 96 | 97 | 98 | if __name__ == '__main__': 99 | print('This file is part of forensic timeline generator "ma2tl". So, it cannot run separately.') 100 | -------------------------------------------------------------------------------- /plugins/volume_mount.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2021-2023 Minoru Kobayashi 3 | # 4 | # This file is part of ma2tl. 5 | # Usage or distribution of this code is subject to the terms of the MIT License. 6 | # 7 | 8 | from __future__ import annotations 9 | 10 | import logging 11 | import os 12 | import re 13 | 14 | from plugins.helpers.basic_info import BasicInfo, MacAptDBType 15 | 16 | PLUGIN_NAME = os.path.splitext(os.path.basename(__file__))[0].upper() 17 | PLUGIN_DESCRIPTION = "Extract volume mount/unmount activities." 18 | PLUGIN_ACTIVITY_TYPE = "Volume Mount" 19 | PLUGIN_VERSION = "20230830" 20 | PLUGIN_AUTHOR = "Minoru Kobayashi" 21 | PLUGIN_AUTHOR_EMAIL = "unknownbit@gmail.com" 22 | 23 | log = None 24 | 25 | 26 | # Extract volume mount/unmount logs 27 | def extract_volume_mount_logs_hfs_apfs(basic_info: BasicInfo, timeline_events: list) -> bool: 28 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 29 | return False 30 | 31 | run_query = basic_info.mac_apt_dbs.run_query 32 | start_ts, end_ts = basic_info.get_between_dates_utc() 33 | sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 34 | (ProcessName = "kernel" AND \ 35 | (Message LIKE "%mounted%" OR \ 36 | Message LIKE "%unmount%" OR \ 37 | Message LIKE "%mounting volume%" OR \ 38 | Message LIKE "%unmounting volume%"\ 39 | )\ 40 | ) \ 41 | ORDER BY TimeUtc;' 42 | 43 | # ignore_volumes = ('Macintosh HD', 'Macintosh HD - Data', 'VM', 'Update', 'Preboot', 'Recovery', 'Boot OS X', 'macOS Base System', 'com.apple.TimeMachine.') 44 | ignore_volumes = ('Macintosh HD', 'Macintosh HD - Data', 'VM', 'Update', 'Preboot', 'Recovery', 'Boot OS X', 'macOS Base System') 45 | 46 | # macOS 13+ APFS mount: apfs_log_mount_unmount:2039: disk5s1 mounting volume Mount Test, requested by: mount_apfs (pid 52313); parent: mount (pid 52312) 47 | # unmount: apfs_log_mount_unmount:2039: disk5s1 unmounting volume Mount Test, requested by: diskarbitrationd (pid 122); parent: launchd (pid 1) 48 | regex_dic = { 49 | 'mount_hfs': r'hfs: mounted (.+) on device (.+)', # macOS 10.15+ 50 | 'unmount_hfs': r'hfs: unmount initiated on (.+) on device (.+)', # macOS 10.15+ 51 | 'mount_apfs': r'apfs_vfsop_mount:\d+: .+: mounted volume: (.+)', # macOS 10.15 - 12 52 | 'unmount_apfs': r'apfs_vfsop_unmount:\d+: .+: unmounting volume (.+)', # macOS 10.15 - 12 53 | 'mount_apfs_13': r'apfs_log_.+:\d+: disk.+ mounting volume (.+), requested by:', # macOS 13+ 54 | 'unmount_apfs_13': r'apfs_log_.+:\d+: disk.+ unmounting volume (.+), requested by:' # macOS 13+ 55 | } 56 | 57 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql): 58 | for reg_type, regex in regex_dic.items(): 59 | result = re.match(regex, row['Message']) 60 | if result: 61 | volume = result.group(1) 62 | # ignore_flag = False 63 | # for ignore_volume in ignore_volumes: 64 | # if volume.startswith(ignore_volume): 65 | # ignore_flag = True 66 | # break 67 | 68 | # if ignore_flag: 69 | # break 70 | 71 | if volume in ignore_volumes or volume.startswith("com.apple.TimeMachine."): 72 | break 73 | 74 | if reg_type.startswith('mount'): 75 | mount_status = 'Volume Mount' 76 | elif reg_type.startswith('unmount'): 77 | mount_status = 'Volume Unmount' 78 | 79 | if reg_type.endswith('hfs'): 80 | fs = 'hfs' 81 | elif reg_type.endswith('apfs') or reg_type.endswith('apfs_13'): 82 | fs = 'apfs' 83 | 84 | event = [row['TimeUtc'], mount_status, f"{result.group(1)} ({fs})", PLUGIN_NAME] 85 | timeline_events.append(event) 86 | break 87 | 88 | return True 89 | 90 | 91 | def extract_volume_mount_smbfs(basic_info): 92 | pass 93 | # start_ts, end_ts = basic_info.get_between_dates_utc() 94 | # sql = 'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{}" AND "{}" AND \ 95 | # (ProcessName = "NetAuthSysAgent" and Category = "NetFS" and Message like "%URL = %") \ 96 | # ORDER BY TimeUtc;'.format(start_ts, end_ts) 97 | # regex_smbfs = r'\s+URL = (.+)' 98 | 99 | 100 | # Extract msdos(fat32)/exfat volume mount logs 101 | def extract_volume_mount_fat(basic_info): 102 | pass 103 | # start_ts, end_ts = basic_info.get_between_dates_utc() 104 | # sql = 'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{}" AND "{}" AND \ 105 | # (ProcessName = "deleted" AND Message like "totalAvailable thresholds for %") AND \ 106 | # (Message like "%[msdos]%" or Message like "%[exfat]%") \ 107 | # ORDER BY TimeUtc;'.format(start_ts, end_ts) 108 | # regex_fat = r'.+PRIMARY at: (.+) \[(.+)\] .+' 109 | 110 | 111 | def extract_volume_mount_ntfs(basic_info): 112 | pass 113 | # start_ts, end_ts = basic_info.get_between_dates_utc() 114 | # sql = 'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{}" AND "{}" AND \ 115 | # (ProcessName = "kernel" AND SenderName = "ntfs" AND Message like "NTFS volume name %") \ 116 | # ORDER BY TimeUtc;'.format(start_ts, end_ts) 117 | # regex_ntfs = r'NTFS volume name (.+), .+' 118 | 119 | 120 | def extract_volume_unmount(basic_info): 121 | pass 122 | 123 | 124 | def run(basic_info: BasicInfo) -> bool: 125 | global log 126 | log = logging.getLogger(basic_info.output_params.logger_root + '.PLUGINS.' + PLUGIN_NAME) 127 | timeline_events = [] 128 | extract_volume_mount_logs_hfs_apfs(basic_info, timeline_events) 129 | 130 | log.info(f"Detected {len(timeline_events)} events.") 131 | if len(timeline_events) > 0: 132 | basic_info.data_writer.write_data_rows(timeline_events) 133 | return True 134 | 135 | return False 136 | 137 | 138 | if __name__ == '__main__': 139 | print('This file is part of forensic timeline generator "ma2tl". So, it cannot run separately.') 140 | -------------------------------------------------------------------------------- /plugins/persistence.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2021-2023 Minoru Kobayashi 3 | # 4 | # This file is part of ma2tl. 5 | # Usage or distribution of this code is subject to the terms of the MIT License. 6 | # 7 | 8 | from __future__ import annotations 9 | 10 | import datetime 11 | import logging 12 | import os 13 | 14 | from plugins.helpers.basic_info import BasicInfo, MacAptDBType 15 | from plugins.helpers.common import convert_apfs_time 16 | 17 | PLUGIN_NAME = os.path.splitext(os.path.basename(__file__))[0].upper() 18 | PLUGIN_DESCRIPTION = "Extract persistence settings." 19 | PLUGIN_ACTIVITY_TYPE = "Persistence" 20 | PLUGIN_VERSION = "20230830" 21 | PLUGIN_AUTHOR = "Minoru Kobayashi" 22 | PLUGIN_AUTHOR_EMAIL = "unknownbit@gmail.com" 23 | 24 | log = None 25 | 26 | # TODO 27 | # Resolve symbolic link in AppPath. 28 | 29 | std_apppath_system_vol = ( 30 | '/System/Applications/', 31 | '/System/Library/CoreServices/', 32 | '/System/Library/Extensions/', 33 | '/System/Library/Frameworks/', 34 | '/System/Library/PrivateFrameworks/', 35 | '/System/Library/CryptoTokenKit/', 36 | '/System/Library/Filesystems/', 37 | '/System/Library/Image Capture/', 38 | '/System/Library/Input Methods/', 39 | '/System/Library/PreferencePanes/', 40 | '/System/Library/Services/', 41 | '/System/iOSSupport/', 42 | '/System/Installation/', 43 | '/usr/libexec/', 44 | '/usr/bin/', 45 | '/usr/sbin/', 46 | '/bin/', 47 | '/sbin/' 48 | ) 49 | 50 | std_persistence_system_vol = ( 51 | '/System/Library/LaunchDaemons/', 52 | '/System/Library/LaunchAgents/' 53 | ) 54 | 55 | std_apppath_data_vol = ( 56 | '/Applications/', 57 | '/Library/Apple/', 58 | '/Library/Application Support/', 59 | '/Library/Extensions/' 60 | ) 61 | 62 | 63 | def _check_between_ts(check_ts, start_ts, end_ts): 64 | check_dt = datetime.datetime.strptime(check_ts, '%Y-%m-%d %H:%M:%S.%f') 65 | start_dt = datetime.datetime.strptime(start_ts, '%Y-%m-%d %H:%M:%S') 66 | end_dt = datetime.datetime.strptime(end_ts, '%Y-%m-%d %H:%M:%S') 67 | 68 | if start_dt <= check_dt and check_dt <= end_dt: 69 | return True 70 | else: 71 | return False 72 | 73 | 74 | def extract_autostart(basic_info: BasicInfo, timeline_events: list) -> bool: 75 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.MACAPT_DB | MacAptDBType.APFS_VOLUMES): 76 | return False 77 | 78 | run_query = basic_info.mac_apt_dbs.run_query 79 | start_ts, end_ts = basic_info.get_between_dates_utc() 80 | sql_users = 'SELECT Username, UID FROM Users;' 81 | sql = 'SELECT * FROM AutoStart WHERE AppPath != "";' 82 | sql_combined = 'SELECT * FROM Combined_Paths LEFT JOIN Combined_Inodes \ 83 | ON Combined_Paths.CNID = Combined_Inodes.CNID \ 84 | WHERE Combined_Paths.Path = "{}" LIMIT 1;' 85 | 86 | users = {} 87 | for row in run_query(MacAptDBType.MACAPT_DB, sql_users): 88 | users[row['Username']] = int(row['UID']) 89 | 90 | persistence_entries = [] 91 | for row in basic_info.mac_apt_dbs.run_query(MacAptDBType.MACAPT_DB, sql): 92 | skip_flag = False 93 | for apppath_prefix in std_apppath_system_vol: 94 | if row['AppPath'].startswith(apppath_prefix): 95 | skip_flag = True 96 | break 97 | 98 | for persistence_prefix in std_persistence_system_vol: 99 | if row['Source'].startswith(persistence_prefix): 100 | skip_flag = True 101 | break 102 | 103 | if skip_flag: 104 | continue 105 | 106 | persistence_entries.append({'Source': row['Source'], 'AppPath': row['AppPath']}) 107 | 108 | for persistence_entry in persistence_entries: 109 | persistence_file = persistence_entry['Source'] 110 | persistence_app = persistence_entry['AppPath'] 111 | 112 | non_std_apppath = True 113 | for apppath_prefix in std_apppath_data_vol: 114 | if persistence_app.startswith(apppath_prefix): 115 | log.debug(f"AppPath: {persistence_app} , Prefix: {apppath_prefix}") 116 | non_std_apppath = False 117 | break 118 | 119 | ts_app_create_utc = '' 120 | msg = '' 121 | event_persistence_app = None 122 | for row in run_query(MacAptDBType.APFS_VOLUMES, sql_combined.format(persistence_app)): 123 | ts_app_create_utc = convert_apfs_time(row['Created']).strftime('%Y-%m-%d %H:%M:%S.%f') 124 | msg = persistence_app 125 | if non_std_apppath: 126 | msg = '[Non-standard AppPath] ' + msg 127 | event_persistence_app = [ts_app_create_utc, PLUGIN_ACTIVITY_TYPE + ' App Creation', msg, PLUGIN_NAME] 128 | 129 | ts_file_create_utc = '' 130 | msg = '' 131 | event_persistence_file = None 132 | for row in run_query(MacAptDBType.APFS_VOLUMES, sql_combined.format(persistence_file)): 133 | ts_file_create_utc = convert_apfs_time(row['Created']).strftime('%Y-%m-%d %H:%M:%S.%f') 134 | msg = f"{persistence_file} (AppPath: {persistence_app})" 135 | if non_std_apppath: 136 | msg = '[Non-standard AppPath] ' + msg 137 | event_persistence_file = [ts_file_create_utc, PLUGIN_ACTIVITY_TYPE + ' File Creation', msg, PLUGIN_NAME] 138 | 139 | if event_persistence_app and event_persistence_file and \ 140 | (_check_between_ts(ts_app_create_utc, start_ts, end_ts) or _check_between_ts(ts_file_create_utc, start_ts, end_ts)): 141 | timeline_events.append(event_persistence_app) 142 | timeline_events.append(event_persistence_file) 143 | 144 | return True 145 | 146 | 147 | def run(basic_info: BasicInfo) -> bool: 148 | global log 149 | log = logging.getLogger(basic_info.output_params.logger_root + '.PLUGINS.' + PLUGIN_NAME) 150 | timeline_events = [] 151 | extract_autostart(basic_info, timeline_events) 152 | 153 | log.info(f"Detected {len(timeline_events)} events.") 154 | if len(timeline_events) > 0: 155 | basic_info.data_writer.write_data_rows(timeline_events) 156 | return True 157 | 158 | return False 159 | 160 | 161 | if __name__ == '__main__': 162 | print('This file is part of forensic timeline generator "ma2tl". So, it cannot run separately.') 163 | -------------------------------------------------------------------------------- /plugins/helpers/basic_info.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2021-2023 Minoru Kobayashi 3 | # 4 | # This file is part of ma2tl. 5 | # Usage or distribution of this code is subject to the terms of the MIT License. 6 | # 7 | # -------------------------------------------------- 8 | # This code is based on mac_apt's macinfo.py 9 | # 10 | 11 | from __future__ import annotations 12 | 13 | import datetime 14 | import sqlite3 15 | import sys 16 | from enum import Enum, Flag, auto 17 | 18 | import pytz 19 | 20 | from plugins.helpers.writer import TLEventWriter 21 | 22 | 23 | # class MacAptDBType(Enum): 24 | class MacAptDBType(Flag): 25 | NONE = auto() 26 | MACAPT_DB = auto() 27 | UNIFIED_LOGS = auto() 28 | APFS_VOLUMES = auto() 29 | ALL = MACAPT_DB | UNIFIED_LOGS | APFS_VOLUMES 30 | 31 | 32 | class OutputParams: 33 | def __init__(self): 34 | self.logger_root = '' 35 | self.output_path = '' 36 | self.use_sqlite = False 37 | self.use_xlsx = False 38 | self.use_tsv = False 39 | 40 | 41 | class ExistDbs(Flag): 42 | NONE = auto() 43 | MACAPT_DB = auto() 44 | UNIFIED_LOGS = auto() 45 | APFS_VOLUMES = auto() 46 | ALL = MACAPT_DB | UNIFIED_LOGS | APFS_VOLUMES 47 | 48 | 49 | class MacAptDbs: 50 | def __init__(self, mac_apt_db='', unifiedlogs_db='', apfs_volumes_db=''): 51 | self.mac_apt_db_path = mac_apt_db 52 | self.mac_apt_db_conn = None 53 | self.mac_apt_db_cursor = None 54 | 55 | self.unifiedlogs_db_path = unifiedlogs_db 56 | self.unifiedlogs_db_conn = None 57 | self.unifiedlogs_db_cursor = None 58 | 59 | self.apfs_volumes_db_path = apfs_volumes_db 60 | self.apfs_volumes_db_conn = None 61 | self.apfs_volumes_db_cursor = None 62 | 63 | self.has_mac_apt_db = False 64 | self.has_unifiedlogs_db = False 65 | self.has_apfs_volumes_db = False 66 | 67 | def open_dbs(self): 68 | if self.mac_apt_db_path: 69 | # self.mac_apt_db_conn = sqlite3.connect(self.mac_apt_db_path) 70 | self.mac_apt_db_conn = sqlite3.connect(f"file:{self.mac_apt_db_path}?mode=ro", uri=True) 71 | self.mac_apt_db_conn.row_factory = sqlite3.Row 72 | self.mac_apt_db_cursor = self.mac_apt_db_conn.cursor() 73 | self.has_mac_apt_db = True 74 | 75 | if self.unifiedlogs_db_path: 76 | # self.unifiedlogs_db_conn = sqlite3.connect(self.unifiedlogs_db_path) 77 | self.unifiedlogs_db_conn = sqlite3.connect(f"file:{self.unifiedlogs_db_path}?mode=ro", uri=True) 78 | self.unifiedlogs_db_conn.row_factory = sqlite3.Row 79 | self.unifiedlogs_db_cursor = self.unifiedlogs_db_conn.cursor() 80 | self.has_unifiedlogs_db = True 81 | 82 | if self.apfs_volumes_db_path: 83 | # self.apfs_volumes_db_conn = sqlite3.connect(self.apfs_volumes_db_path) 84 | self.apfs_volumes_db_conn = sqlite3.connect(f"file:{self.apfs_volumes_db_path}?mode=ro", uri=True) 85 | self.apfs_volumes_db_conn.row_factory = sqlite3.Row 86 | self.apfs_volumes_db_cursor = self.apfs_volumes_db_conn.cursor() 87 | self.has_apfs_volumes_db = True 88 | 89 | def close_dbs(self): 90 | if self.mac_apt_db_conn: 91 | self.mac_apt_db_path = '' 92 | self.mac_apt_db_conn.close() 93 | self.has_mac_apt_db = False 94 | 95 | if self.unifiedlogs_db_conn: 96 | self.unifiedlogs_db_path = '' 97 | self.unifiedlogs_db_conn.close() 98 | self.has_unifiedlogs_db = False 99 | 100 | if self.apfs_volumes_db_conn: 101 | self.apfs_volumes_db_path = '' 102 | self.apfs_volumes_db_conn.close() 103 | self.has_apfs_volumes_db = False 104 | 105 | def has_dbs(self, db_type: MacAptDBType) -> MacAptDBType: 106 | result = MacAptDBType.NONE 107 | if db_type | MacAptDBType.MACAPT_DB and self.has_mac_apt_db: 108 | result = MacAptDBType.MACAPT_DB 109 | if db_type | MacAptDBType.UNIFIED_LOGS and self.has_unifiedlogs_db: 110 | if result == MacAptDBType.NONE: 111 | result = MacAptDBType.UNIFIED_LOGS 112 | else: 113 | result |= MacAptDBType.UNIFIED_LOGS 114 | if db_type | MacAptDBType.APFS_VOLUMES and self.has_apfs_volumes_db: 115 | if result == MacAptDBType.NONE: 116 | result = MacAptDBType.APFS_VOLUMES 117 | else: 118 | result |= MacAptDBType.APFS_VOLUMES 119 | 120 | if db_type == result: 121 | return True 122 | else: 123 | return False 124 | 125 | def run_query(self, db_type: MacAptDBType, query: str) -> sqlite3.Row | tuple: 126 | cursor = None 127 | if db_type == MacAptDBType.MACAPT_DB and self.has_mac_apt_db: 128 | cursor = self.mac_apt_db_cursor 129 | if db_type == MacAptDBType.UNIFIED_LOGS and self.has_unifiedlogs_db: 130 | cursor = self.unifiedlogs_db_cursor 131 | if db_type == MacAptDBType.APFS_VOLUMES and self.has_apfs_volumes_db: 132 | cursor = self.apfs_volumes_db_cursor 133 | 134 | if cursor: 135 | return cursor.execute(query) 136 | else: 137 | return tuple() 138 | 139 | def is_table_exist(self, db_type: MacAptDBType, table_name: str) -> bool: 140 | if db_type == MacAptDBType.MACAPT_DB: 141 | cursor = self.mac_apt_db_cursor 142 | if db_type == MacAptDBType.UNIFIED_LOGS: 143 | cursor = self.unifiedlogs_db_cursor 144 | if db_type == MacAptDBType.APFS_VOLUMES: 145 | cursor = self.apfs_volumes_db_cursor 146 | 147 | cursor.execute(f'SELECT * FROM sqlite_master WHERE type="table" and name="{table_name}"') 148 | if cursor.fetchone(): 149 | return True 150 | else: 151 | return False 152 | 153 | 154 | class BasicInfo: 155 | def __init__(self, mac_apt_dbs: MacAptDbs, output_params, start_ts, end_ts, timezone='UTC'): 156 | self.mac_apt_dbs = mac_apt_dbs 157 | self.output_params = output_params 158 | # self.analyzing_unifiedlogs_only = False 159 | self.data_writer = TLEventWriter(output_params, 'ma2tl', 'ma2tl', timezone) 160 | 161 | try: 162 | self.tzinfo_user = pytz.timezone(timezone) 163 | self.tzinfo_utc = pytz.timezone('UTC') 164 | except pytz.exceptions.UnknownTimeZoneError as ex: 165 | sys.exit(f"Unknown TimeZone: {ex}") 166 | 167 | self.start_dt_usertz = self._convert_ts_to_usertz(start_ts) 168 | self.end_dt_usertz = self._convert_ts_to_usertz(end_ts) 169 | self.start_dt_utc = self._convert_ts_to_utc(start_ts) 170 | self.end_dt_utc = self._convert_ts_to_utc(end_ts) 171 | 172 | def _convert_ts_to_usertz(self, ts): 173 | dt_naive = datetime.datetime.strptime(ts, '%Y-%m-%d %H:%M:%S') 174 | return self.tzinfo_user.normalize(self.tzinfo_user.localize(dt_naive)) 175 | 176 | def _convert_ts_to_utc(self, ts): 177 | dt_aware_usertz = self._convert_ts_to_usertz(ts) 178 | return dt_aware_usertz.astimezone(pytz.timezone('UTC')) 179 | 180 | def get_between_dates_usertz(self): 181 | fmt = '%Y-%m-%d %H:%M:%S' 182 | return [self.start_dt_usertz.strftime(fmt), self.end_dt_usertz.strftime(fmt)] 183 | 184 | def get_between_dates_utc(self): 185 | fmt = '%Y-%m-%d %H:%M:%S' 186 | return [self.start_dt_utc.strftime(fmt), self.end_dt_utc.strftime(fmt)] 187 | 188 | 189 | if __name__ == '__main__': 190 | print('This file is part of forensic timeline generator "ma2tl". So, it cannot run separately.') 191 | -------------------------------------------------------------------------------- /plugins/file_download.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2021-2023 Minoru Kobayashi 3 | # 4 | # This file is part of ma2tl. 5 | # Usage or distribution of this code is subject to the terms of the MIT License. 6 | # 7 | 8 | from __future__ import annotations 9 | 10 | import logging 11 | import os 12 | 13 | from plugins.helpers.basic_info import BasicInfo, MacAptDBType 14 | from plugins.helpers.common import get_timedelta 15 | 16 | PLUGIN_NAME = os.path.splitext(os.path.basename(__file__))[0].upper() 17 | PLUGIN_DESCRIPTION = "Extract file download activities." 18 | PLUGIN_ACTIVITY_TYPE = "File Download" 19 | PLUGIN_VERSION = "20230830" 20 | PLUGIN_AUTHOR = "Minoru Kobayashi" 21 | PLUGIN_AUTHOR_EMAIL = "unknownbit@gmail.com" 22 | 23 | log = None 24 | 25 | 26 | class FileDownloadEvent: 27 | def __init__(self, ts, data_url, origin_url, local_path, agent=''): 28 | self.ts = ts 29 | self.data_url = data_url 30 | self.origin_url = origin_url 31 | self.local_path = local_path 32 | self.agent = agent 33 | 34 | 35 | def extract_spotlight_dataview_file_download(basic_info: BasicInfo, filedownload_events: list) -> bool: 36 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.MACAPT_DB): 37 | return False 38 | 39 | run_query = basic_info.mac_apt_dbs.run_query 40 | sql = 'SELECT * FROM "{}" WHERE kMDItemDownloadedDate BETWEEN "{}" AND "{}" \ 41 | ORDER BY kMDItemDownloadedDate;' 42 | sql_tableinfo = 'PRAGMA table_info("{}");' 43 | tables = { 44 | 'SpotlightDataView-1-store': False, 45 | 'SpotlightDataView-1-.store-DIFF': False 46 | } 47 | 48 | for table in tables.keys(): 49 | for column in run_query(MacAptDBType.MACAPT_DB, sql_tableinfo.format(table)).fetchall(): 50 | if column[1] == 'kMDItemDownloadedDate': 51 | tables[table] = True 52 | break 53 | 54 | start_ts, end_ts = basic_info.get_between_dates_utc() 55 | for table, has_downloaddeddate in tables.items(): 56 | if has_downloaddeddate: 57 | for row in run_query(MacAptDBType.MACAPT_DB, sql.format(table, start_ts, end_ts)): 58 | skip_flag = False 59 | ts = row['kMDItemDownloadedDate'] 60 | data_url = row['kMDItemWhereFroms'] # If this column have multiple URLs, it should be split with comma(,). First one is DataUrl, second one is OriginUrl. 61 | local_path = row['FullPath'] 62 | agent = 'N/A' 63 | 64 | if len(data_url.split(', ')) == 2: 65 | data_url, origin_url = data_url.split(', ') 66 | else: 67 | data_url = data_url.split(', ')[0] 68 | origin_url = 'N/A' 69 | 70 | for event in filedownload_events: 71 | if event.ts == ts and event.data_url == data_url and event.origin_url == origin_url and event.local_path == local_path: 72 | skip_flag = True 73 | break 74 | 75 | if not skip_flag: 76 | filedownload_events.append(FileDownloadEvent(ts, data_url, origin_url, local_path, agent)) 77 | 78 | return True 79 | 80 | 81 | def extract_safari_quarantine_file_download(basic_info: BasicInfo, filedownload_events: list) -> bool: 82 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.MACAPT_DB): 83 | return False 84 | 85 | run_query = basic_info.mac_apt_dbs.run_query 86 | start_ts, end_ts = basic_info.get_between_dates_utc() 87 | sql = f'SELECT Quarantine.TimeStamp, Quarantine.AgentName, Quarantine.DataUrl, Quarantine.OriginUrl, Safari.Other_Info FROM Quarantine \ 88 | INNER JOIN Safari ON Safari.Type = "DOWNLOAD" AND Quarantine.DataUrl = Safari.URL \ 89 | WHERE Quarantine.TimeStamp BETWEEN "{start_ts}" AND "{end_ts}" AND \ 90 | Quarantine.AgentName = "Safari" \ 91 | ORDER BY TimeStamp;' 92 | 93 | for row in run_query(MacAptDBType.MACAPT_DB, sql): 94 | skip_flag = False 95 | ts = row['TimeStamp'] 96 | data_url = row['DataUrl'] 97 | origin_url = row['OriginUrl'] 98 | local_path = row['Other_Info'] 99 | agent = row['AgentName'] 100 | 101 | for event in filedownload_events: 102 | if event.data_url == data_url and event.local_path == local_path and get_timedelta(event.ts, ts) <= 1: 103 | log.debug(f"{event.ts}, {event.data_url}, {event.origin_url}, {event.local_path}, {event.agent}") 104 | log.debug(f"{ts}, {data_url}, {origin_url}, {local_path}, {agent}") 105 | if event.agent in (None, '', 'N/A'): 106 | event.agent = agent 107 | skip_flag = True 108 | break 109 | 110 | if not skip_flag: 111 | filedownload_events.append(FileDownloadEvent(ts, data_url, origin_url, local_path, agent)) 112 | 113 | return True 114 | 115 | 116 | def extract_chrome_file_download(basic_info: BasicInfo, filedownload_events: list) -> bool: 117 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.MACAPT_DB): 118 | return False 119 | 120 | table_name = 'Chrome' 121 | if not basic_info.mac_apt_dbs.is_table_exist(MacAptDBType.MACAPT_DB, table_name): 122 | log.info(f"{table_name} table does not exist.") 123 | return False 124 | 125 | run_query = basic_info.mac_apt_dbs.run_query 126 | start_ts, end_ts = basic_info.get_between_dates_utc() 127 | sql = f'SELECT * FROM Chrome WHERE Type = "DOWNLOAD" AND Date BETWEEN "{start_ts}" AND "{end_ts}" ORDER BY Date;' 128 | 129 | for row in run_query(MacAptDBType.MACAPT_DB, sql): 130 | ts = row['Date'] 131 | data_url = row['URL'] 132 | origin_url = row['Referrer or Previous Page'] 133 | local_path = row['Local Path'] 134 | agent = 'Chrome' 135 | filedownload_events.append(FileDownloadEvent(ts, data_url, origin_url, local_path, agent)) 136 | 137 | return True 138 | 139 | 140 | def extract_quarantine_file_download(basic_info: BasicInfo, filedownload_events: list) -> bool: 141 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.MACAPT_DB): 142 | return False 143 | 144 | run_query = basic_info.mac_apt_dbs.run_query 145 | start_ts, end_ts = basic_info.get_between_dates_utc() 146 | sql = f'SELECT TimeStamp, AgentName, DataUrl, OriginUrl FROM Quarantine \ 147 | WHERE TimeStamp BETWEEN "{start_ts}" AND "{end_ts}" \ 148 | ORDER BY TimeStamp;' 149 | 150 | for row in run_query(MacAptDBType.MACAPT_DB, sql): 151 | skip_flag = False 152 | ts = row['TimeStamp'] 153 | data_url = row['DataUrl'] 154 | origin_url = row['OriginUrl'] 155 | local_path = 'N/A' 156 | agent = row['AgentName'] 157 | 158 | for event in filedownload_events: 159 | if event.data_url == data_url and get_timedelta(event.ts, ts) <= 1: 160 | log.debug(f"{event.ts}, {event.data_url}, {event.origin_url}, {event.local_path}, {event.agent}") 161 | log.debug(f"{ts}, {data_url}, {origin_url}, {local_path}, {agent}") 162 | if event.agent in (None, '', 'N/A'): 163 | event.agent = agent + '?' 164 | skip_flag = True 165 | break 166 | 167 | if not skip_flag: 168 | filedownload_events.append(FileDownloadEvent(ts, data_url, origin_url, local_path, agent)) 169 | 170 | return True 171 | 172 | 173 | def run(basic_info: BasicInfo) -> bool: 174 | global log 175 | log = logging.getLogger(basic_info.output_params.logger_root + '.PLUGINS.' + PLUGIN_NAME) 176 | timeline_events = [] 177 | filedownload_events = [] 178 | extract_spotlight_dataview_file_download(basic_info, filedownload_events) 179 | extract_safari_quarantine_file_download(basic_info, filedownload_events) 180 | extract_chrome_file_download(basic_info, filedownload_events) 181 | extract_quarantine_file_download(basic_info, filedownload_events) 182 | 183 | for event in filedownload_events: 184 | if event.local_path in (None, '', 'N/A'): 185 | event = [event.ts, PLUGIN_ACTIVITY_TYPE, f"From {event.data_url} , Origin: {event.origin_url} , Agent: {event.agent})", PLUGIN_NAME] 186 | else: 187 | event = [event.ts, PLUGIN_ACTIVITY_TYPE, f"{event.local_path} (From {event.data_url} , Origin: {event.origin_url} , Agent: {event.agent})", PLUGIN_NAME] 188 | timeline_events.append(event) 189 | 190 | log.info(f"Detected {len(timeline_events)} events.") 191 | if len(timeline_events) > 0: 192 | basic_info.data_writer.write_data_rows(timeline_events) 193 | return True 194 | 195 | return False 196 | 197 | 198 | if __name__ == '__main__': 199 | print('This file is part of forensic timeline generator "ma2tl". So, it cannot run separately.') 200 | -------------------------------------------------------------------------------- /helper_tools/ndjson2madb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2023 Minoru Kobayashi (@unkn0wnbit) 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | from __future__ import annotations 19 | 20 | import argparse 21 | import os 22 | import sqlite3 23 | import sys 24 | from typing import NoReturn 25 | 26 | import ndjson 27 | 28 | 29 | class UnifiedLogsDbWriter: 30 | def __init__(self) -> None: 31 | self.db_path: str = "" 32 | self.conn: sqlite3.Connection = None 33 | self.cursor: sqlite3.Cursor = None 34 | self.table_name: str = "" 35 | self.column_list: list[dict] = None 36 | self.sql_executemany: str = "" 37 | 38 | def open_db(self, db_path: str) -> bool | NoReturn: 39 | self.db_path = db_path 40 | try: 41 | if self.db_path and not os.path.exists(self.db_path): 42 | self.conn = sqlite3.connect(self.db_path) 43 | self.conn.execute("PRAGMA journal_mode=WAL") 44 | return True 45 | else: 46 | print(f"Specified SQLite file has been existed: {self.db_path}") 47 | return False 48 | 49 | except (OSError, sqlite3.Error) as ex: 50 | print(f"Failed to open/create sqlite db at path {self.db_path}") 51 | print(f"Error details: {str(ex)}") 52 | raise ex 53 | 54 | def close_db(self) -> NoReturn: 55 | if self.conn: 56 | self.conn.close() 57 | self.conn = None 58 | 59 | def _build_create_table_query(self) -> str: 60 | sql = 'CREATE TABLE "' + self.table_name + '" (' 61 | # for column_name in self.column_list: 62 | # sql += f'"{column_name}" TEXT,' 63 | for column_pair in self.column_list: 64 | for column_name, column_type in column_pair.items(): 65 | sql += f'"{column_name}" {column_type},' 66 | 67 | sql = sql[:-1] # remove the last comma 68 | sql += ")" 69 | return sql 70 | 71 | def create_table(self, table_name="UnifiedLogs", column_list: list[dict] = list()) -> bool | NoReturn: 72 | try: 73 | self.table_name = table_name 74 | self.column_list = column_list 75 | if not self.column_list: 76 | self.column_list = [ 77 | {"File": "TEXT"}, 78 | {"DecompFilePos": "INTEGER"}, 79 | {"ContinuousTime": "TEXT"}, 80 | {"TimeUtc": "TEXT"}, 81 | {"Thread": "INTEGER"}, 82 | {"Type": "TEXT"}, 83 | {"ActivityID": "INTEGER"}, 84 | {"ParentActivityID": "INTEGER"}, 85 | {"ProcessID": "INTEGER"}, 86 | {"EffectiveUID": "INTEGER"}, 87 | {"TTL": "INTEGER"}, 88 | {"ProcessName": "TEXT"}, 89 | {"SenderName": "TEXT"}, 90 | {"Subsystem": "TEXT"}, 91 | {"Category": "TEXT"}, 92 | {"SignpostName": "TEXT"}, 93 | {"SignpostInfo": "TEXT"}, 94 | {"ImageOffset": "INTEGER"}, 95 | {"SenderUUID": "TEXT"}, 96 | {"ProcessImageUUID": "TEXT"}, 97 | {"SenderImagePath": "TEXT"}, 98 | {"ProcessImagePath": "TEXT"}, 99 | {"Message": "TEXT"}, 100 | ] 101 | self.cursor = self.conn.cursor() 102 | self.cursor.execute(self._build_create_table_query()) 103 | self.conn.commit() 104 | self.sql_executemany = ( 105 | 'INSERT INTO "' + self.table_name + '" VALUES (?' + ",?" * (len(self.column_list) - 1) + ")" 106 | ) 107 | return True 108 | 109 | except sqlite3.Error as ex: 110 | print(f"Error creating SQLite table: {self.table_name}") 111 | print(f"Error details: {str(ex)}") 112 | raise ex 113 | 114 | def write_rows(self, rows: list[list | tuple]) -> bool | NoReturn: 115 | try: 116 | self.cursor.executemany(self.sql_executemany, rows) 117 | self.conn.commit() 118 | return True 119 | 120 | except sqlite3.Error as ex: 121 | print(f"Error writing to SQLite table: {self.table_name}") 122 | print(f"Error details: {str(ex)}") 123 | raise ex 124 | 125 | 126 | def parse_arguments() -> argparse.ArgumentParser: 127 | epilog = ( 128 | "[Exporting Unified Logs Tips]\n" 129 | + "Exporting all entries of Unified Logs takes a lot of disk space. I recommend using zip command along with to reduce the file size.\n" 130 | + "% log show --info --debug --style ndjson --timezone 'UTC' | zip ~/Desktop/unifiedlogs_ndjson.zip -\n\n" 131 | + "Zipped file can be converted to a database like below:\n" 132 | + "% unzip -q -c ~/Desktop/unifiedlogs_ndjson.zip | python3 ./ndjson2ma.py -o ./UnifiedLogs.db\n" 133 | + "\n\n" 134 | + "[Timezone]\n" 135 | + "This script does NOT consider timezone. So, you need to run the log command like below:\n" 136 | + "% log show --info --debug --style ndjson --timezone 'UTC' > /path/to/unifiedlogs.ndjson" 137 | ) 138 | 139 | parser = argparse.ArgumentParser( 140 | description="Convert the exported Unified Logs with ndjson style to mac_apt UnifiedLogs.db.\n", 141 | epilog=epilog, 142 | formatter_class=argparse.RawTextHelpFormatter, 143 | ) 144 | parser.add_argument( 145 | "-i", "--input", action="store", default="-", help="Path to an exported Unified Logs file (Default: - (STDIN))" 146 | ) 147 | parser.add_argument( 148 | "-o", 149 | "--output", 150 | action="store", 151 | default="UnifiedLogs.db", 152 | required=True, 153 | help="Path to an output database file (Default: UnifiedLogs.db)", 154 | ) 155 | return parser.parse_args() 156 | 157 | 158 | def parse_log_entry(log_entry: dict) -> list: 159 | file = "" 160 | decomp_file_pos = 0 161 | continuous_time = "0" 162 | # time_utc: str = log_entry.get('machTimestamp', '0000-00-00 00:00:00.000000+0000') 163 | time_utc: str = log_entry.get("timestamp", "0000-00-00 00:00:00.000000+0000") 164 | thread = log_entry.get("threadID", 0) 165 | type = log_entry.get("messageType", "") 166 | activity_id = log_entry.get("activityIdentifier", 0) 167 | parent_activity_id = log_entry.get("parentActivityIdentifier", 0) 168 | process_id = log_entry.get("processID", 0) 169 | effective_uid = 0 170 | ttl = 0 171 | process_name: str = log_entry.get("processImagePath", "") 172 | sender_name: str = log_entry.get("senderImagePath", "") 173 | subsystem = log_entry.get("subsystem", "") 174 | category = log_entry.get("category", "") 175 | signpost_name = "" 176 | signpost_info = "" 177 | image_offset = 0 178 | sender_uuid = log_entry.get("senderImageUUID", "") 179 | process_image_uuid = log_entry.get("processImageUUID", "") 180 | sender_image_path = log_entry.get("senderImagePath", "") 181 | process_image_path = log_entry.get("processImagePath", "") 182 | message = log_entry.get("eventMessage", "") 183 | 184 | time_utc = time_utc.split("+")[0] 185 | process_name = process_name.split("/")[-1] 186 | sender_name = sender_name.split("/")[-1] 187 | 188 | unifiedlogs_db_entry = [ 189 | file, 190 | decomp_file_pos, 191 | continuous_time, 192 | time_utc, 193 | thread, 194 | type, 195 | activity_id, 196 | parent_activity_id, 197 | process_id, 198 | effective_uid, 199 | ttl, 200 | process_name, 201 | sender_name, 202 | subsystem, 203 | category, 204 | signpost_name, 205 | signpost_info, 206 | image_offset, 207 | sender_uuid, 208 | process_image_uuid, 209 | sender_image_path, 210 | process_image_path, 211 | message, 212 | ] 213 | 214 | return unifiedlogs_db_entry 215 | 216 | 217 | def main(): 218 | args = parse_arguments() 219 | 220 | if os.path.exists(args.output): 221 | print("{} is already exist.".format(args.output)) 222 | sys.exit(1) 223 | 224 | db_writer = UnifiedLogsDbWriter() 225 | if not db_writer.open_db(args.output): 226 | sys.exit(1) 227 | db_writer.create_table() 228 | 229 | if args.input == "-": 230 | f = sys.stdin 231 | else: 232 | f = open(args.input, "rt") 233 | 234 | log_reader = ndjson.reader(f) 235 | unifiedlogs_db_entries = list() 236 | 237 | for log_entry in log_reader: 238 | parsed_entry = parse_log_entry(log_entry) 239 | unifiedlogs_db_entries.append(parsed_entry) 240 | if len(unifiedlogs_db_entries) == 1000: 241 | db_writer.write_rows(unifiedlogs_db_entries) 242 | unifiedlogs_db_entries = list() 243 | 244 | if args.input != "-": 245 | f.close() 246 | 247 | if len(unifiedlogs_db_entries) > 0: 248 | db_writer.write_rows(unifiedlogs_db_entries) 249 | db_writer.close_db() 250 | 251 | 252 | if __name__ == "__main__": 253 | main() 254 | -------------------------------------------------------------------------------- /plugins/helpers/writer.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2021 Minoru Kobayashi 3 | # 4 | # This file is part of ma2tl. 5 | # Usage or distribution of this code is subject to the terms of the MIT License. 6 | # 7 | # -------------------------------------------------- 8 | # This code is based on mac_apt's writer.py 9 | # 10 | 11 | import csv 12 | import datetime 13 | import logging 14 | import os 15 | import sqlite3 16 | 17 | import pytz 18 | import xlsxwriter 19 | 20 | log = logging.getLogger('MA2TL.HELPERS.WRITER') 21 | 22 | 23 | class TLEventWriter: 24 | def __init__(self, output_params, base_name, table_name, timezone): 25 | self.output_path = output_params.output_path 26 | self.table_name = table_name 27 | self.tzinfo_user_str = timezone 28 | self.use_sqlite = False 29 | self.sqlite_writer = None 30 | self.sqlite_db_path = os.path.join(self.output_path, base_name + '.db') 31 | self.use_xlsx = False 32 | self.xlsx_writer = None 33 | self.xlsx_file_path = os.path.join(self.output_path, base_name + '.xlsx') 34 | self.use_tsv = False 35 | self.tsv_writer = None 36 | self.tsv_file_path = os.path.join(self.output_path, base_name + '.tsv') 37 | 38 | if output_params.use_sqlite: 39 | self.use_sqlite = True 40 | self.sqlite_writer = SqliteWriter() 41 | self.sqlite_writer.open_db(self.sqlite_db_path) 42 | if output_params.use_xlsx: 43 | self.use_xlsx = True 44 | self.xlsx_writer = XlsxWriter() 45 | self.xlsx_writer.create_xlsx_file(self.xlsx_file_path) 46 | if output_params.use_tsv: 47 | self.use_tsv = True 48 | self.tsv_writer = TsvWriter() 49 | self.tsv_writer.create_tsv_file(self.tsv_file_path) 50 | 51 | def write_data_header(self, header_list): 52 | if self.use_sqlite: 53 | self.sqlite_writer.create_table(self.table_name, header_list) 54 | if self.use_xlsx: 55 | self.xlsx_writer.create_sheet(self.table_name) 56 | self.xlsx_writer.add_header_row(header_list) 57 | if self.use_tsv: 58 | self.tsv_writer.write_rows(header_list, header=True) 59 | 60 | def _convert_ts_microsec_to_usertz(self, ts_with_microsecond): 61 | try: 62 | dt_naive = datetime.datetime.strptime(ts_with_microsecond, '%Y-%m-%d %H:%M:%S.%f') 63 | except ValueError as ex: 64 | dt_naive = datetime.datetime.strptime(ts_with_microsecond + '.000000', '%Y-%m-%d %H:%M:%S.%f') 65 | tzinfo_utc = pytz.timezone('UTC') 66 | dt_aware_utc = tzinfo_utc.localize(dt_naive) 67 | return dt_aware_utc.astimezone(pytz.timezone(self.tzinfo_user_str)).strftime('%Y-%m-%d %H:%M:%S.%f') 68 | 69 | def write_data_rows(self, rows): 70 | if len(rows) == 0: 71 | return 72 | 73 | # Insert user timezone timestamp. 74 | for row in rows: 75 | row.insert(1, self._convert_ts_microsec_to_usertz(row[0])) 76 | 77 | if self.use_sqlite: 78 | self.sqlite_writer.write_rows(rows) 79 | if self.use_xlsx: 80 | self.xlsx_writer.write_rows(rows) 81 | if self.use_tsv: 82 | self.tsv_writer.write_rows(rows) 83 | 84 | def close_writer(self): 85 | if self.use_sqlite: 86 | self.sqlite_writer.close_db() 87 | if self.use_xlsx: 88 | self.xlsx_writer.close_xlsx_file() 89 | if self.use_tsv: 90 | self.tsv_writer.close_tsv_file() 91 | 92 | 93 | class SqliteWriter: 94 | def __init__(self): 95 | self.db_path = '' 96 | self.conn = None 97 | self.cursor = None 98 | self.table_name = '' 99 | self.column_list = None 100 | self.sql_executemany = '' 101 | 102 | def open_db(self, db_path): 103 | self.db_path = db_path 104 | try: 105 | if self.db_path and not os.path.exists(self.db_path): 106 | self.conn = sqlite3.connect(self.db_path) 107 | return True 108 | else: 109 | log.error(f"Specified SQLite files has been existed: {self.db_path}") 110 | return False 111 | 112 | except (OSError, sqlite3.Error) as ex: 113 | log.error(f"Failed to open/create sqlite db at path {self.db_path}") 114 | log.exception(f"Error details: {str(ex)}") 115 | raise ex 116 | 117 | def close_db(self): 118 | if self.conn: 119 | self.conn.close() 120 | self.conn = None 121 | 122 | def _build_create_table_query(self): 123 | sql = 'CREATE TABLE "' + self.table_name + '" (' 124 | for column_name in self.column_list: 125 | sql += f'"{column_name}" TEXT,' 126 | 127 | sql = sql[:-1] # remove the last comma 128 | sql += ')' 129 | return sql 130 | 131 | def create_table(self, table_name, column_list): 132 | try: 133 | self.table_name = table_name 134 | self.column_list = column_list 135 | self.cursor = self.conn.cursor() 136 | self.cursor.execute(self._build_create_table_query()) 137 | self.conn.commit() 138 | self.sql_executemany = 'INSERT INTO "' + self.table_name + '" VALUES (?' + ',?'*(len(self.column_list) - 1) + ')' 139 | return True 140 | 141 | except sqlite3.Error as ex: 142 | log.error(f"Error creating SQLite table: {self.table_name}") 143 | log.exception(f"Error details: {str(ex)}") 144 | raise ex 145 | 146 | def write_rows(self, rows): 147 | try: 148 | self.cursor.executemany(self.sql_executemany, rows) 149 | self.conn.commit() 150 | return True 151 | 152 | except sqlite3.Error as ex: 153 | log.error(f"Error writing to SQLite table: {self.table_name}") 154 | log.exception(f"Error details: {str(ex)}") 155 | raise ex 156 | 157 | 158 | class XlsxWriter: 159 | def __init__(self): 160 | self.file_path = '' 161 | self.workbook = None 162 | self.sheet = None 163 | self.max_allowed_rows = 1000000 164 | self.row_index = 0 165 | self.sheet_name = '' 166 | self.max_row_index = 0 167 | self.max_col_index = 0 168 | self.col_width_list = None 169 | 170 | def create_xlsx_file(self, file_path): 171 | self.file_path = file_path 172 | try: 173 | self.workbook = xlsxwriter.Workbook(self.file_path, {'strings_to_urls': False, 'constant_memory': True}) 174 | except (xlsxwriter.exceptions.XlsxWriterException, OSError) as ex: 175 | log.error(f"Failed to create xlsx file at path {self.file_path}") 176 | log.exception(f"Error details: {str(ex)}") 177 | raise ex 178 | 179 | def _beautify_columns(self): 180 | sheet = self.workbook.get_worksheet_by_name(self.sheet_name) 181 | sheet.freeze_panes(1, 0) # Freeze 1st row 182 | # Set column widths 183 | col_index = 0 184 | for col_width in self.col_width_list: 185 | if col_index == 0 or col_index == 1: 186 | col_width = 25 187 | if col_width > 100: 188 | col_width = 100 189 | sheet.set_column(col_index, col_index, col_width) 190 | col_index += 1 191 | # Autofilter 192 | sheet.autofilter(0, 0, self.max_row_index, self.max_col_index) 193 | 194 | def close_xlsx_file(self): 195 | self._beautify_columns() 196 | if self.workbook: 197 | self.workbook.close() 198 | self.workbook = None 199 | self.file_path = '' 200 | 201 | def create_sheet(self, sheet_name): 202 | if len(sheet_name) > 31: 203 | log.warning(f"Sheet name \"{sheet_name}\" is longer than the Excel limit of 31 char. It will be truncated to 31 char!") 204 | sheet_name = sheet_name[0:31] 205 | try: 206 | self.sheet = self.workbook.add_worksheet(sheet_name) 207 | except xlsxwriter.exceptions.XlsxWriterException as ex: 208 | log.exception(f"Unknown error while adding sheet {sheet_name}") 209 | raise ex 210 | self.row_index = 0 211 | self.sheet_name = sheet_name 212 | 213 | def add_header_row(self, header_list): 214 | column_index = 0 215 | for column_name in header_list: 216 | column_width = 8.43 217 | self.sheet.write_string(self.row_index, column_index, column_name, self.workbook.add_format({'bold': True})) 218 | self.sheet.set_column(column_index, column_index, column_width) 219 | column_index += 1 220 | self.row_index += 1 221 | self.max_col_index = column_index - 1 222 | self.max_row_index = self.row_index - 1 223 | self.col_width_list = [len(col_name)+3 for col_name in header_list] 224 | 225 | def _store_column_width(self, row): 226 | column_index = 0 227 | for item in row: 228 | width = len(item) + 1 229 | if width > self.col_width_list[column_index]: 230 | self.col_width_list[column_index] = width 231 | column_index += 1 232 | 233 | def _write_row(self, row): 234 | column_index = 0 235 | if self.row_index > self.max_allowed_rows: 236 | log.exception("Error trying to add sheet for overflow data (>1 million rows)") 237 | raise xlsxwriter.exceptions.XlsxWriterException 238 | 239 | try: 240 | row_str = tuple(map(str, row)) 241 | for item in row_str: 242 | try: 243 | self.sheet.write(self.row_index, column_index, item) 244 | except (TypeError, ValueError, xlsxwriter.exceptions.XlsxWriterException): 245 | log.exception(f"Error writing data:{item} of type:{type(row[column_index])} in excel row:{self.row_index}") 246 | column_index += 1 247 | 248 | self.row_index += 1 249 | self.max_row_index = self.row_index - 1 250 | self._store_column_width(row_str) 251 | except xlsxwriter.exceptions.XlsxWriterException as ex: 252 | log.exception(f"Error writing excel row {self.row_index}") 253 | 254 | def write_rows(self, rows): 255 | for row in rows: 256 | self._write_row(row) 257 | 258 | 259 | class TsvWriter: 260 | def __init__(self): 261 | self.file_path = '' 262 | self.file_handle = None 263 | self.tsv_writer = None 264 | 265 | def create_tsv_file(self, file_path): 266 | try: 267 | self.file_handle = open(file_path, 'wt', encoding='UTF-8', newline='') 268 | self.tsv_writer = csv.writer(self.file_handle, delimiter='\t') 269 | except (OSError, csv.Error) as ex: 270 | log.error(f"Failed to create file at path {self.file_path}") 271 | log.exception(f"Error details: {str(ex)}") 272 | raise ex 273 | 274 | def close_tsv_file(self): 275 | if self.tsv_writer: 276 | self.file_handle.close() 277 | self.tsv_writer = None 278 | self.file_path = '' 279 | 280 | def write_rows(self, rows, header=False): 281 | if header: 282 | self.tsv_writer.writerow(rows) 283 | else: 284 | self.tsv_writer.writerows(rows) 285 | 286 | 287 | if __name__ == '__main__': 288 | print('This file is part of forensic timeline generator "ma2tl". So, it cannot run separately.') 289 | -------------------------------------------------------------------------------- /plugins/remote_login.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2023 Minoru Kobayashi 3 | # 4 | # This file is part of ma2tl. 5 | # Usage or distribution of this code is subject to the terms of the MIT License. 6 | # 7 | 8 | from __future__ import annotations 9 | 10 | import logging 11 | import os 12 | import re 13 | 14 | from plugins.helpers.basic_info import BasicInfo, MacAptDBType 15 | 16 | PLUGIN_NAME = os.path.splitext(os.path.basename(__file__))[0].upper() 17 | PLUGIN_DESCRIPTION = "Extract remote login activities." 18 | PLUGIN_ACTIVITY_TYPE = "Remote Login" 19 | PLUGIN_VERSION = "20230830" 20 | PLUGIN_AUTHOR = "Minoru Kobayashi" 21 | PLUGIN_AUTHOR_EMAIL = "unknownbit@gmail.com" 22 | 23 | log = None 24 | 25 | 26 | # Extract sshd authentication logs 27 | # This function is confirmed to work correctly for macOS 13+ 28 | def extract_remote_authentication_sshd(basic_info: BasicInfo, timeline_events: list) -> bool: 29 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 30 | return False 31 | 32 | run_query = basic_info.mac_apt_dbs.run_query 33 | start_ts, end_ts = basic_info.get_between_dates_utc() 34 | # sshd log samples 35 | ### accepted login and logout 36 | # [Default] fatal: Timeout before authentication for 172.16.114.1 port 62211 37 | # [Info] Accepted keyboard-interactive/pam for macforensics from 172.16.114.1 port 60341 ssh2 38 | # [Info] Received disconnect from 172.16.114.1 port 60341:11: disconnected by user 39 | # [Info] Disconnected from user macforensics 172.16.114.1 port 60341 40 | ### user is existing but not valid password 41 | # [Default] error: PAM: authentication error for macforensics from 172.16.114.1 42 | # [Info] Failed none for macforensics from 172.16.114.1 port 62312 ssh2 43 | # [Info] Failed password for macforensics from 172.16.114.1 port 59703 ssh2 44 | # [Info] Connection closed by authenticating user macforensics 172.16.114.1 port 59703 [preauth] 45 | # [Default] error: maximum authentication attempts exceeded for macforensics from 172.16.114.1 port 62312 ssh2 [preauth] 46 | # [Info] Disconnecting authenticating user macforensics 172.16.114.1 port 62312: Too many authentication failures [preauth] 47 | ### invalid user 48 | # [Info] Invalid user ZZZZZ from 172.16.114.1 port 59701 49 | # [Info] Postponed keyboard-interactive for invalid user ZZZZZ from 172.16.114.1 port 59701 ssh2 [preauth] 50 | # [Default] error: PAM: unknown user for illegal user ZZZZZ from 172.16.114.1 51 | # [Info] Failed keyboard-interactive/pam for invalid user ZZZZZ from 172.16.114.1 port 59701 ssh2 52 | # [Info] Failed none for invalid user ZZZZZ from 172.16.114.1 port 59701 ssh2 53 | # [Info] Failed password for invalid user ZZZZZ from 172.16.114.1 port 59701 ssh2 54 | # [Info: Connection closed by invalid user ZZZZZ 172.16.114.1 port 62588 [preauth]] 55 | # [Default] error: maximum authentication attempts exceeded for invalid user ZZZZZ from 172.16.114.1 port 59701 ssh2 [preauth] 56 | # [Info] Disconnecting invalid user ZZZZZ 172.16.114.1 port 59701: Too many authentication failures [preauth] 57 | sql_loginout = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 58 | (ProcessName == "sshd" AND SenderName == "sshd" AND \ 59 | (Message LIKE "fatal: Timeout before authentication for %" OR \ 60 | Message LIKE "Accepted % for % from %" OR \ 61 | Message LIKE "Disconnected from %"\ 62 | )\ 63 | ) \ 64 | ORDER BY TimeUtc;' 65 | sql_invalid_password = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 66 | (ProcessName == "sshd" AND SenderName == "sshd" AND \ 67 | (Message LIKE "error: PAM: authentication error for %" OR \ 68 | Message LIKE "Failed password for % from % port %" OR \ 69 | Message LIKE "Connection closed by authenticating user %"\ 70 | )\ 71 | ) \ 72 | ORDER BY TimeUtc;' 73 | sql_invalid_user = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 74 | (ProcessName == "sshd" AND SenderName == "sshd" AND \ 75 | (Message LIKE "Invalid user %" OR \ 76 | Message LIKE "error: PAM: unknown user for illegal user %" OR \ 77 | Message LIKE "Failed % for invalid user % from % port %" OR \ 78 | Message LIKE "Connection closed by invalid user %" OR \ 79 | Message LIKE "error: maximum authentication attempts %" OR \ 80 | Message LIKE "Disconnecting invalid user %"\ 81 | )\ 82 | ) \ 83 | ORDER BY TimeUtc;' 84 | 85 | regex_loginout = ( 86 | r'^fatal: Timeout before authentication for (?P
.+) port (?P.+)', 87 | r'^Accepted .+ for (?P.+) from (?P
.+) port (?P.+) .+', 88 | r'^Disconnected from user (?P.+) (?P
.+) port (?P.+)' 89 | ) 90 | regex_invalid_password = ( 91 | r'^error: PAM: authentication error for (?P.+) from (?P
.+)', 92 | r'^Failed password for (?P\w+) from (?P
.+) port (?P.+)', 93 | r'^Connection closed by authenticating user (?P.+) (?P
.+) port (?P.+) .+', 94 | r'^error: maximum authentication attempts exceeded for (?P\w+) from (?P
.+) port (?P.+) .+', 95 | r'^Disconnecting authenticating user (?P.+) (?P
.+) port (?P.+): Too many authentication failures .+' 96 | ) 97 | regex_invalid_user = ( 98 | r'^Invalid user (?P.+) from (?P
.+) port (?P.+)', 99 | r'^error: PAM: unknown user for illegal user (?P.+) from (?P
.+)', 100 | r'^Failed password for invalid user (?P.+) from (?P
.+) port (?P.+)', 101 | r'^Connection closed by invalid user (?P.+) (?P
.+) port (?P.+) .+', 102 | r'^error: maximum authentication attempts exceeded for invalid user (?P.+) from (?P
.+) port (?P.+) .+', 103 | r'^Disconnecting invalid user (?P.+) (?P
.+) port (?P.+): Too many authentication failures .+' 104 | ) 105 | 106 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql_loginout): 107 | for idx, regex in enumerate(regex_loginout): 108 | if result := re.match(regex, row['Message']): 109 | msg = "" 110 | if idx == 0: 111 | msg = f"SSHD: Authentication timeout addr={result['address']}, port={result['port']}" 112 | elif idx == 1: 113 | msg = f"SSHD: Accepted user={result['username']}, addr={result['address']}, port={result['port']}" 114 | elif idx == 2: 115 | msg = f"SSHD: Disconnected user={result['username']}, addr={result['address']}, port={result['port']}" 116 | 117 | if msg: 118 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 119 | timeline_events.append(event) 120 | 121 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql_invalid_password): 122 | for idx, regex in enumerate(regex_invalid_password): 123 | if result := re.match(regex, row['Message']): 124 | msg = "" 125 | if idx == 0: 126 | msg = f"SSHD: Authentication error user={result['username']}, addr={result['address']}" 127 | elif idx == 1: 128 | msg = f"SSHD: Failed password user={result['username']}, addr={result['address']}, port={result['port']}" 129 | elif idx == 2: 130 | msg = f"SSHD: Connection closed user={result['username']}, addr={result['address']}, port={result['port']}" 131 | elif idx == 3: 132 | msg = f"SSHD: Maximum authentication attempts exceeded user={result['username']}, addr={result['address']}, port={result['port']}" 133 | elif idx == 4: 134 | msg = f"SSHD: Disconnecting user={result['username']}, addr={result['address']}, port={result['port']}" 135 | 136 | if msg: 137 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 138 | timeline_events.append(event) 139 | 140 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql_invalid_user): 141 | for idx, regex in enumerate(regex_invalid_user): 142 | if result := re.match(regex, row['Message']): 143 | msg = "" 144 | if idx == 0: 145 | msg = f"SSHD: Invalid user={result['username']}, addr={result['address']}, port={result['port']}" 146 | elif idx == 1: 147 | msg = f"SSHD: Authentication error invalid user={result['username']}, addr={result['address']}" 148 | elif idx == 2: 149 | msg = f"SSHD: Failed password invalid user={result['username']}, addr={result['address']}, port={result['port']}" 150 | elif idx == 3: 151 | msg = f"SSHD: Connection closed invalid user={result['username']}, addr={result['address']}, port={result['port']}" 152 | elif idx == 4: 153 | msg = f"SSHD: Maximum authentication attempts exceeded invalid user={result['username']}, addr={result['address']}, port={result['port']}" 154 | elif idx == 5: 155 | msg = f"SSHD: Disconnecting invalid user={result['username']}, addr={result['address']}, port={result['port']}" 156 | 157 | if msg: 158 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 159 | timeline_events.append(event) 160 | 161 | return True 162 | 163 | 164 | # Extract screensharingd authentication logs 165 | # This function is confirmed to work correctly for macOS 13+ 166 | def extract_remote_authentication_screensharing(basic_info: BasicInfo, timeline_events: list) -> bool: 167 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 168 | return False 169 | 170 | run_query = basic_info.mac_apt_dbs.run_query 171 | start_ts, end_ts = basic_info.get_between_dates_utc() 172 | sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 173 | (ProcessName == "screensharingd" AND \ 174 | Message LIKE "Authentication: %"\ 175 | ) \ 176 | ORDER BY TimeUtc;' 177 | regex = r'^Authnetication: (?P.+) :: (?P.+) :: User Name: (?P.+) :: Viewer Address: (?P
.+) :: Type: (?P.+)' 178 | 179 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql): 180 | if result := re.match(regex, row['Message']): 181 | msg = f"Screen Sharing: authentication={result['auth_result']}, user={result['username']}, addr={result['address']}, type={result['type']}" 182 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 183 | timeline_events.append(event) 184 | 185 | return True 186 | 187 | 188 | def run(basic_info: BasicInfo) -> bool: 189 | global log 190 | log = logging.getLogger(basic_info.output_params.logger_root + '.PLUGINS.' + PLUGIN_NAME) 191 | timeline_events = [] 192 | extract_remote_authentication_sshd(basic_info, timeline_events) 193 | extract_remote_authentication_screensharing(basic_info, timeline_events) 194 | 195 | log.info(f"Detected {len(timeline_events)} events.") 196 | if len(timeline_events) > 0: 197 | basic_info.data_writer.write_data_rows(timeline_events) 198 | return True 199 | 200 | return False 201 | 202 | 203 | if __name__ == '__main__': 204 | print('This file is part of forensic timeline generator "ma2tl". So, it cannot run separately.') 205 | -------------------------------------------------------------------------------- /ma2tl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # ma2tl.py 4 | # Generate a forensic timeline form the result DBs of mac_apt analysis. 5 | # 6 | # 7 | # MIT License 8 | # 9 | # Copyright (c) 2021-2023 Minoru Kobayashi (@unkn0wnbit) 10 | # 11 | # Permission is hereby granted, free of charge, to any person obtaining a copy 12 | # of this software and associated documentation files (the "Software"), to deal 13 | # in the Software without restriction, including without limitation the rights 14 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 15 | # copies of the Software, and to permit persons to whom the Software is 16 | # furnished to do so, subject to the following conditions: 17 | # 18 | # The above copyright notice and this permission notice shall be included in all 19 | # copies or substantial portions of the Software. 20 | # 21 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 22 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 23 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 24 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 25 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 26 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 27 | # SOFTWARE. 28 | # 29 | 30 | from __future__ import annotations 31 | 32 | import argparse 33 | import glob 34 | import logging 35 | import os 36 | import re 37 | import sys 38 | import textwrap 39 | import time 40 | 41 | import tzlocal 42 | 43 | import plugins.helpers.basic_info as basicinfo 44 | from plugins.helpers.plugin import (check_user_specified_plugin_name, 45 | import_plugins, setup_logger) 46 | 47 | log = None 48 | MA2TL_VERSION = '20230830' 49 | 50 | 51 | def parse_arguments(plugins: list) -> argparse.ArgumentParser: 52 | plugin_name_list = ['ALL'] 53 | plugins_info = f"The following {len(plugins)} plugins are available:" 54 | 55 | for plugin in plugins: 56 | plugins_info += "\n {:<20}{}".format(plugin.PLUGIN_NAME, textwrap.fill(plugin.PLUGIN_DESCRIPTION, subsequent_indent=' '*24, initial_indent=' '*24, width=80)[24:]) 57 | plugin_name_list.append(plugin.PLUGIN_NAME) 58 | 59 | plugins_info += "\n " + "-"*76 + "\n" +\ 60 | " "*4 + "ALL" + " "*17 + "Run all plugins" 61 | 62 | parser = argparse.ArgumentParser( 63 | description='Forensic timeline generator using mac_apt analysis results. Supports only SQLite DBs.', 64 | epilog=plugins_info, formatter_class=argparse.RawTextHelpFormatter 65 | ) 66 | parser.add_argument('-i', '--input', action='store', default=None, help='Path to a folder that contains mac_apt DBs') 67 | parser.add_argument('-o', '--output', action='store', default=None, help='Path to a folder to save ma2tl result') 68 | parser.add_argument('-ot', '--output_type', action='store', default='SQLITE', help='Specify the output file type: SQLITE, XLSX, TSV (Default: SQLITE)') 69 | # parser.add_argument('-f', '--force', action='store_true', default=False, help='Overwrite an output file.') 70 | # parser.add_argument('-u', '--unifiedlogs_only', action='store_true', default=False, help='Analyze UnifiedLogs.db only (Default: False)') 71 | parser.add_argument('-s', '--start', action='store', default=None, help='Specify start timestamp (ex. 2021-11-05 08:30:00)') 72 | parser.add_argument('-e', '--end', action='store', default=None, help='Specify end timestamp') 73 | parser.add_argument('-t', '--timezone', action='store', default=None, help='Specify Timezone: "UTC", "Asia/Tokyo", "US/Eastern", etc (Default: System Local Timezone)') 74 | parser.add_argument('-l', '--log_level', action='store', default='INFO', help='Specify log level: INFO, DEBUG, WARNING, ERROR, CRITICAL (Default: INFO)') 75 | parser.add_argument('plugin', nargs="+", help="Plugins to run (space separated).") 76 | return parser.parse_args() 77 | 78 | 79 | def expand_to_abspath(path): 80 | if path.startswith('~/') or path == '~': 81 | path = os.path.expanduser(path) 82 | return os.path.abspath(path) 83 | 84 | 85 | def check_input_path(input_path: str, macapt_dbs: basicinfo.MacAptDbs) -> bool: 86 | try: 87 | if os.path.isdir(input_path): 88 | db_list = glob.glob(os.path.join(input_path, '*.db')) 89 | for db_path in db_list: 90 | if os.path.isfile(db_path): 91 | if os.path.basename(db_path) == 'mac_apt.db': 92 | macapt_dbs.mac_apt_db_path = db_path 93 | elif os.path.basename(db_path) == 'UnifiedLogs.db': 94 | macapt_dbs.unifiedlogs_db_path = db_path 95 | elif re.match(r'APFS_Volumes_\w{8}-\w{4}-\w{4}-\w{4}-\w{12}\.db', os.path.basename(db_path)): 96 | macapt_dbs.apfs_volumes_db_path = db_path 97 | # if macapt_dbs.mac_apt_db_path and macapt_dbs.unifiedlogs_db_path and macapt_dbs.apfs_volumes_db_path: 98 | if macapt_dbs.mac_apt_db_path or macapt_dbs.unifiedlogs_db_path or macapt_dbs.apfs_volumes_db_path: 99 | return True 100 | # else: 101 | print("Error: mac_apt analysis result DBs are insufficient.") 102 | return False 103 | else: 104 | print("Error: the input path is not a directory.") 105 | return False 106 | 107 | except Exception as ex: 108 | print(f"Error: Unknown exception, error details are: {str(ex)}") 109 | return False 110 | 111 | 112 | def check_output_path(output_path, force_flag=False): 113 | try: 114 | if os.path.isdir(output_path): 115 | for filename in os.listdir(output_path): 116 | if filename.startswith('ma2tl.'): 117 | print(f"Error: There is already a file that starts with \"ma2tl.\" : {filename}") 118 | return False 119 | return True 120 | 121 | else: 122 | if os.path.isfile(output_path): 123 | print(f"Error: The file already exists : {output_path}") 124 | return False 125 | 126 | else: 127 | try: 128 | os.makedirs(output_path) 129 | return True 130 | except Exception as ex: 131 | print(f"Error: Cannot create an output folder : {output_path}\nError Details: {str(ex)}") 132 | return False 133 | 134 | except Exception as ex: 135 | print(f"Exception occurred: {str(ex)}") 136 | return False 137 | 138 | 139 | def exit_(message=''): 140 | global log 141 | if log and (len(message) > 0): 142 | log.info(message) 143 | sys.exit() 144 | else: 145 | sys.exit(message) 146 | 147 | 148 | def main(): 149 | global log 150 | plugins = [] 151 | if import_plugins(plugins) == 0: 152 | exit_("Error: No plugins could be added.") 153 | 154 | # 155 | # Check arguments 156 | # 157 | args = parse_arguments(plugins) 158 | 159 | if args.output: 160 | args.output = expand_to_abspath(args.output) 161 | print(f"Output path: {args.output}") 162 | if not check_output_path(args.output): 163 | exit_() 164 | else: 165 | exit_('Specify a folder path to store ma2tl result files.') 166 | 167 | args.log_level = args.log_level.upper() 168 | if args.log_level not in ('INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'): 169 | exit_("Invalid input type for log level. Valid values are INFO, DEBUG, WARNING, ERROR, CRITICAL") 170 | else: 171 | if args.log_level == "INFO": 172 | args.log_level = logging.INFO 173 | elif args.log_level == "DEBUG": 174 | args.log_level = logging.DEBUG 175 | elif args.log_level == "WARNING": 176 | args.log_level = logging.WARNING 177 | elif args.log_level == "ERROR": 178 | args.log_level = logging.ERROR 179 | elif args.log_level == "CRITICAL": 180 | args.log_level = logging.CRITICAL 181 | 182 | # 183 | # Start analysis 184 | # 185 | started_time = time.time() 186 | logger_root = os.path.splitext(os.path.basename(__file__))[0].upper() 187 | log = setup_logger(os.path.join(args.output, f"ma2tl_log_{time.strftime('%Y%m%d-%H%M%S')}.txt"), logger_root, args.log_level) 188 | log.setLevel(args.log_level) 189 | log.info(f"ma2tl (mac_apt to timeline) ver.{MA2TL_VERSION}: Started at {time.strftime('%H:%M:%S', time.localtime(started_time))}") 190 | log.info(f"Command line: {' '.join(sys.argv)}") 191 | 192 | plugins_to_run = [x.upper() for x in args.plugin] 193 | if 'ALL' in plugins_to_run: 194 | process_all = True 195 | else: 196 | process_all = False 197 | 198 | if not process_all: 199 | if not check_user_specified_plugin_name(plugins_to_run, plugins): 200 | exit_("Error: Specified plugin name is not found.") 201 | 202 | output_params = basicinfo.OutputParams() 203 | output_params.logger_root = logger_root 204 | output_params.output_path = args.output 205 | if args.output_type: 206 | args.output_type = args.output_type.upper() 207 | if args.output_type not in ('SQLITE', 'XLSX', 'TSV'): 208 | exit_(f"Error: Unsupported output type: {args.output_type}") 209 | 210 | if args.output_type == 'SQLITE': 211 | output_params.use_sqlite = True 212 | elif args.output_type == 'XLSX': 213 | output_params.use_xlsx = True 214 | elif args.output_type == 'TSV': 215 | output_params.use_tsv = True 216 | 217 | macapt_dbs = basicinfo.MacAptDbs() 218 | if args.input: 219 | args.input = expand_to_abspath(args.input) 220 | log.info(f"Input path : {args.input}") 221 | if not check_input_path(args.input, macapt_dbs): 222 | exit_() 223 | else: 224 | exit_('Error: Specify mac_apt result DBs folder.') 225 | 226 | if args.start and args.end: 227 | regex_ts = r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' 228 | if not (re.match(regex_ts, args.start) and re.match(regex_ts, args.end)): 229 | exit_('Error: Start timestamp or end timestamp cannot be recognized.') 230 | else: 231 | exit_('Error: Specify both start and end timestamp.') 232 | 233 | # 234 | # Prepare BasicInfo object 235 | # 236 | if args.timezone: 237 | tz = args.timezone 238 | else: 239 | tz = str(tzlocal.get_localzone()) 240 | basic_info = basicinfo.BasicInfo(macapt_dbs, output_params, args.start, args.end, tz) 241 | basic_info.mac_apt_dbs.open_dbs() 242 | 243 | # 244 | # Write data header 245 | # 246 | header_list = ['Timestamp (UTC)', f"Timestamp ({tz})", 'ActivityType', 'Message', 'PluginName'] 247 | basic_info.data_writer.write_data_header(header_list) 248 | 249 | # 250 | # Run plugins!! 251 | # 252 | for plugin in plugins: 253 | if process_all or (plugin.PLUGIN_NAME in plugins_to_run): 254 | log.info("-"*50) 255 | log.info(f"Running plugin - {plugin.PLUGIN_NAME}") 256 | try: 257 | plugin.run(basic_info) 258 | except Exception: 259 | log.exception(f"An exception occurred while running plugin - {plugin.PLUGIN_NAME}") 260 | 261 | # 262 | # Close mac_apt DBs 263 | # 264 | basic_info.mac_apt_dbs.close_dbs() 265 | 266 | # 267 | # Close TLEventWriter object 268 | # 269 | basic_info.data_writer.close_writer() 270 | 271 | ended_time = time.time() 272 | log.info("Finished.") 273 | log.info(f"Processing time: {time.strftime('%H:%M:%S', time.gmtime(ended_time - started_time))}") 274 | 275 | 276 | if __name__ == "__main__": 277 | if sys.version_info.major >= 3 and sys.version_info.minor >= 7: 278 | main() 279 | else: 280 | sys.exit('Need to install Python 3.7.0 or later to run this script.') 281 | -------------------------------------------------------------------------------- /helper_tools/aul2madb/src/main.rs: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2023 Minoru Kobayashi (@unkn0wnbit) 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | 17 | // 18 | // parse_log_archive() and parse_trace_file() are borrowed from macos-UnifiedLogs' unifiedlog_parser. 19 | // https://github.com/mandiant/macos-UnifiedLogs/blob/main/examples/unifiedlog_parser/src/main.rs 20 | // 21 | 22 | use chrono::{SecondsFormat, TimeZone, Utc}; 23 | use clap::{Parser, ValueEnum}; 24 | use csv::Writer; 25 | use macos_unifiedlogs::dsc::SharedCacheStrings; 26 | use macos_unifiedlogs::parser::{ 27 | build_log, collect_shared_strings, collect_strings, collect_timesync, parse_log, 28 | }; 29 | use macos_unifiedlogs::timesync::TimesyncBoot; 30 | use macos_unifiedlogs::unified_log::{LogData, UnifiedLogData}; 31 | use macos_unifiedlogs::uuidtext::UUIDText; 32 | use rusqlite::{params, Connection, Result}; 33 | use simplelog::{Config, SimpleLogger}; 34 | use std::error::Error; 35 | use std::fs; 36 | use std::fs::OpenOptions; 37 | use std::path::{Path, PathBuf}; 38 | use std::process; 39 | // use log::LevelFilter; 40 | 41 | #[derive(Parser, Debug)] 42 | // #[clap(version, about, long_about = None)] 43 | #[command(version, about, long_about = None)] 44 | struct Args { 45 | /// Path to a logarchive or to a directory that contains exported Unified Logs 46 | #[clap(short, long)] 47 | input: String, 48 | 49 | /// Output format 50 | // #[arg(short = 'f', long, default_value = "sqlite")] 51 | #[clap(short = 'f', long, default_value = "sqlite")] 52 | output_format: OutputFormat, 53 | 54 | /// Path to output file 55 | #[clap(short, long, default_value = "./UnifiedLogs.db")] 56 | output: String, 57 | } 58 | 59 | #[derive(ValueEnum, Clone, Debug)] 60 | enum OutputFormat { 61 | SQLITE, 62 | // CSV, 63 | TSV, 64 | } 65 | 66 | fn main() { 67 | SimpleLogger::init(simplelog::LevelFilter::Warn, Config::default()) 68 | .expect("Failed to initialize simple logger"); 69 | 70 | let _args = Args::parse(); 71 | 72 | // let input_path = Path::new(&_args.input).canonicalize().unwrap(); 73 | let input_path = dunce::canonicalize(Path::new(&_args.input)).unwrap(); 74 | let output_path = Path::new(&_args.output); 75 | 76 | if !input_path.is_dir() { 77 | println!( 78 | "{} is not a logarchive or a directory.", 79 | &input_path.display() 80 | ); 81 | process::exit(1); 82 | } 83 | 84 | if output_path.is_file() { 85 | println!("{} has been exist.", &output_path.display()); 86 | process::exit(1); 87 | } 88 | 89 | println!("Staring Unified Logs converter..."); 90 | 91 | output_header().unwrap(); 92 | 93 | if input_path.display().to_string().ends_with(".logarchive") { 94 | println!("Processing as a logarchive."); 95 | parse_log_archive(&input_path.display().to_string()); 96 | } else { 97 | println!("Processing as exported Unified Logs."); 98 | parse_exported_logs(&input_path.display().to_string()); 99 | } 100 | 101 | println!( 102 | "\nFinished parsing Unified Log data. Saved results to: {}", 103 | &output_path.display() 104 | ); 105 | } 106 | 107 | fn parse_exported_logs(path: &str) { 108 | let mut exported_path = PathBuf::from(path); 109 | 110 | exported_path.push("uuidtext"); 111 | let string_results = collect_strings(&exported_path.display().to_string()).unwrap(); 112 | 113 | exported_path.push("dsc"); 114 | let shared_strings_result = 115 | collect_shared_strings(&exported_path.display().to_string()).unwrap(); 116 | exported_path.pop(); 117 | exported_path.pop(); 118 | 119 | exported_path.push("diagnostics"); 120 | exported_path.push("timesync"); 121 | let timesync_data = collect_timesync(&exported_path.display().to_string()).unwrap(); 122 | exported_path.pop(); 123 | 124 | parse_trace_file( 125 | &string_results, 126 | &shared_strings_result, 127 | ×ync_data, 128 | &exported_path.display().to_string(), 129 | ); 130 | } 131 | 132 | // Parse a provided directory path. Currently expect the path to follow macOS log collect structure 133 | fn parse_log_archive(path: &str) { 134 | let mut archive_path = PathBuf::from(path); 135 | 136 | // Parse all UUID files which contain strings and other metadata 137 | let string_results = collect_strings(&archive_path.display().to_string()).unwrap(); 138 | 139 | archive_path.push("dsc"); 140 | // Parse UUID cache files which also contain strings and other metadata 141 | let shared_strings_results = 142 | collect_shared_strings(&archive_path.display().to_string()).unwrap(); 143 | archive_path.pop(); 144 | 145 | archive_path.push("timesync"); 146 | // Parse all timesync files 147 | let timesync_data = collect_timesync(&archive_path.display().to_string()).unwrap(); 148 | archive_path.pop(); 149 | 150 | // Keep UUID, UUID cache, timesync files in memory while we parse all tracev3 files 151 | // Allows for faster lookups 152 | parse_trace_file( 153 | &string_results, 154 | &shared_strings_results, 155 | ×ync_data, 156 | path, 157 | ); 158 | 159 | // println!("\nFinished parsing Unified Log data. Saved results to: output.csv"); 160 | } 161 | 162 | // Use the provided strings, shared strings, timesync data to parse the Unified Log data at provided path. 163 | // Currently expect the path to follow macOS log collect structure 164 | fn parse_trace_file( 165 | string_results: &[UUIDText], 166 | shared_strings_results: &[SharedCacheStrings], 167 | timesync_data: &[TimesyncBoot], 168 | path: &str, 169 | ) { 170 | // We need to persist the Oversize log entries (they contain large strings that don't fit in normal log entries) 171 | // Some log entries have Oversize strings located in different tracev3 files. 172 | // This is very rare. Seen in ~20 log entries out of ~700,000. Seen in ~700 out of ~18 million 173 | let mut oversize_strings = UnifiedLogData { 174 | header: Vec::new(), 175 | catalog_data: Vec::new(), 176 | oversize: Vec::new(), 177 | }; 178 | 179 | // Exclude missing data from returned output. Keep separate until we parse all oversize entries. 180 | // Then at end, go through all missing data and check all parsed oversize entries again 181 | let mut exclude_missing = true; 182 | let mut missing_data: Vec = Vec::new(); 183 | 184 | let mut archive_path = PathBuf::from(path); 185 | archive_path.push("Persist"); 186 | 187 | let mut log_count = 0; 188 | if archive_path.exists() { 189 | let paths = fs::read_dir(&archive_path).unwrap(); 190 | 191 | // Loop through all tracev3 files in Persist directory 192 | for log_path in paths { 193 | let data = log_path.unwrap(); 194 | let full_path = data.path().display().to_string(); 195 | println!("Parsing: {}", full_path); 196 | 197 | let log_data = if data.path().exists() { 198 | parse_log(&full_path).unwrap() 199 | } else { 200 | println!("File {} no longer on disk", full_path); 201 | continue; 202 | }; 203 | 204 | // Get all constructed logs and any log data that failed to get constrcuted (exclude_missing = true) 205 | let (results, missing_logs) = build_log( 206 | &log_data, 207 | string_results, 208 | shared_strings_results, 209 | timesync_data, 210 | exclude_missing, 211 | ); 212 | // Track Oversize entries 213 | oversize_strings 214 | .oversize 215 | .append(&mut log_data.oversize.to_owned()); 216 | 217 | // Track missing logs 218 | missing_data.push(missing_logs); 219 | log_count += results.len(); 220 | output(&results).unwrap(); 221 | } 222 | } 223 | 224 | archive_path.pop(); 225 | archive_path.push("Special"); 226 | 227 | if archive_path.exists() { 228 | let paths = fs::read_dir(&archive_path).unwrap(); 229 | 230 | // Loop through all tracev3 files in Special directory 231 | for log_path in paths { 232 | let data = log_path.unwrap(); 233 | let full_path = data.path().display().to_string(); 234 | println!("Parsing: {}", full_path); 235 | 236 | let mut log_data = if data.path().exists() { 237 | parse_log(&full_path).unwrap() 238 | } else { 239 | println!("File {} no longer on disk", full_path); 240 | continue; 241 | }; 242 | 243 | // Append our old Oversize entries in case these logs point to other Oversize entries the previous tracev3 files 244 | log_data.oversize.append(&mut oversize_strings.oversize); 245 | let (results, missing_logs) = build_log( 246 | &log_data, 247 | string_results, 248 | shared_strings_results, 249 | timesync_data, 250 | exclude_missing, 251 | ); 252 | // Track Oversize entries 253 | oversize_strings.oversize = log_data.oversize; 254 | // Track missing logs 255 | missing_data.push(missing_logs); 256 | log_count += results.len(); 257 | 258 | output(&results).unwrap(); 259 | } 260 | } 261 | 262 | archive_path.pop(); 263 | archive_path.push("Signpost"); 264 | 265 | if archive_path.exists() { 266 | let paths = fs::read_dir(&archive_path).unwrap(); 267 | 268 | // Loop through all tracev3 files in Signpost directory 269 | for log_path in paths { 270 | let data = log_path.unwrap(); 271 | let full_path = data.path().display().to_string(); 272 | println!("Parsing: {}", full_path); 273 | 274 | let log_data = if data.path().exists() { 275 | parse_log(&full_path).unwrap() 276 | } else { 277 | println!("File {} no longer on disk", full_path); 278 | continue; 279 | }; 280 | 281 | let (results, missing_logs) = build_log( 282 | &log_data, 283 | string_results, 284 | shared_strings_results, 285 | timesync_data, 286 | exclude_missing, 287 | ); 288 | 289 | // Signposts have not been seen with Oversize entries 290 | missing_data.push(missing_logs); 291 | log_count += results.len(); 292 | 293 | output(&results).unwrap(); 294 | } 295 | } 296 | archive_path.pop(); 297 | archive_path.push("HighVolume"); 298 | 299 | if archive_path.exists() { 300 | let paths = fs::read_dir(&archive_path).unwrap(); 301 | 302 | // Loop through all tracev3 files in HighVolume directory 303 | for log_path in paths { 304 | let data = log_path.unwrap(); 305 | let full_path = data.path().display().to_string(); 306 | println!("Parsing: {}", full_path); 307 | 308 | let log_data = if data.path().exists() { 309 | parse_log(&full_path).unwrap() 310 | } else { 311 | println!("File {} no longer on disk", full_path); 312 | continue; 313 | }; 314 | let (results, missing_logs) = build_log( 315 | &log_data, 316 | string_results, 317 | shared_strings_results, 318 | timesync_data, 319 | exclude_missing, 320 | ); 321 | 322 | // Oversize entries have not been seen in logs in HighVolume 323 | missing_data.push(missing_logs); 324 | log_count += results.len(); 325 | 326 | output(&results).unwrap(); 327 | } 328 | } 329 | archive_path.pop(); 330 | 331 | archive_path.push("logdata.LiveData.tracev3"); 332 | 333 | // Check if livedata exists. We only have it if 'log collect' was used 334 | if archive_path.exists() { 335 | println!("Parsing: logdata.LiveData.tracev3"); 336 | let mut log_data = parse_log(&archive_path.display().to_string()).unwrap(); 337 | log_data.oversize.append(&mut oversize_strings.oversize); 338 | let (results, missing_logs) = build_log( 339 | &log_data, 340 | string_results, 341 | shared_strings_results, 342 | timesync_data, 343 | exclude_missing, 344 | ); 345 | // Track missing data 346 | missing_data.push(missing_logs); 347 | log_count += results.len(); 348 | 349 | output(&results).unwrap(); 350 | // Track oversize entries 351 | oversize_strings.oversize = log_data.oversize; 352 | archive_path.pop(); 353 | } 354 | 355 | exclude_missing = false; 356 | 357 | // Since we have all Oversize entries now. Go through any log entries that we were not able to build before 358 | for mut leftover_data in missing_data { 359 | // Add all of our previous oversize data to logs for lookups 360 | leftover_data 361 | .oversize 362 | .append(&mut oversize_strings.oversize.to_owned()); 363 | 364 | // Exclude_missing = false 365 | // If we fail to find any missing data its probably due to the logs rolling 366 | // Ex: tracev3A rolls, tracev3B references Oversize entry in tracev3A will trigger missing data since tracev3A is gone 367 | let (results, _) = build_log( 368 | &leftover_data, 369 | string_results, 370 | shared_strings_results, 371 | timesync_data, 372 | exclude_missing, 373 | ); 374 | log_count += results.len(); 375 | 376 | output(&results).unwrap(); 377 | } 378 | println!("Parsed {} log entries", log_count); 379 | } 380 | 381 | // Create csv file and create headers 382 | fn output_header() -> Result<(), Box> { 383 | let args = Args::parse(); 384 | 385 | match args.output_format { 386 | OutputFormat::SQLITE => { 387 | output_header_sqlite(&args.output)?; 388 | } 389 | // OutputFormat::CSV => { 390 | // let csv_file = OpenOptions::new() 391 | // .append(true) 392 | // .create(true) 393 | // .open(args.output)?; 394 | // let writer = csv::Writer::from_writer(csv_file); 395 | // output_header_csv(writer)?; 396 | // } 397 | OutputFormat::TSV => { 398 | let csv_file = OpenOptions::new() 399 | .append(true) 400 | .create(true) 401 | .open(args.output)?; 402 | let writer = csv::WriterBuilder::new() 403 | .delimiter(b'\t') 404 | .from_writer(csv_file); 405 | output_header_csv(writer)?; 406 | } 407 | } 408 | Ok(()) 409 | } 410 | 411 | fn output_header_sqlite(path: &str) -> Result<(), Box> { 412 | let conn = Connection::open(path)?; 413 | conn.pragma_update(None, "journal_mode", &"WAL")?; 414 | conn.execute( 415 | "CREATE TABLE UnifiedLogs ( 416 | File TEXT, 417 | DecompFilePos INTEGER, 418 | ContinuousTime TEXT, 419 | TimeUtc TEXT, 420 | Thread INTEGER, 421 | Type TEXT, 422 | ActivityID INTEGER, 423 | ParentActivityID INTEGER, 424 | ProcessID INTEGER, 425 | EffectiveUID INTEGER, 426 | TTL INTEGER, 427 | ProcessName TEXT, 428 | SenderName TEXT, 429 | Subsystem TEXT, 430 | Category TEXT, 431 | SignpostName TEXT, 432 | SignpostInfo TEXT, 433 | ImageOffset INTEGER, 434 | SenderUUID TEXT, 435 | ProcessImageUUID TEXT, 436 | SenderImagePath TEXT, 437 | ProcessImagePath TEXT, 438 | Message TEXT 439 | );", 440 | params![], 441 | )?; 442 | Ok(()) 443 | } 444 | 445 | fn output_header_csv(mut writer: Writer) -> Result<(), Box> { 446 | writer.write_record(&[ 447 | "File", 448 | "DecompFilePos", 449 | "ContinuousTime", 450 | "TimeUTC", 451 | "Thread", 452 | "Type", 453 | "ActivityID", 454 | "ParentActivityID", 455 | "ProcessID", 456 | "EffectiveUID", 457 | "TTL", 458 | "ProcessName", 459 | "SenderName", 460 | "Subsystem", 461 | "Category", 462 | "SignpostName", 463 | "SignpostInfo", 464 | "ImageOffset", 465 | "SenderUUID", 466 | "ProcessImageUUID", 467 | "SenderImagePath", 468 | "ProcessImagePath", 469 | "Message", 470 | ])?; 471 | Ok(()) 472 | } 473 | 474 | // Append or create csv file 475 | fn output(results: &Vec) -> Result<(), Box> { 476 | let args = Args::parse(); 477 | 478 | match args.output_format { 479 | OutputFormat::SQLITE => { 480 | // writer = csv::Writer::from_writer(csv_file); 481 | output_sqlite(&args.output, results)?; 482 | } 483 | // OutputFormat::CSV => { 484 | // let csv_file = OpenOptions::new() 485 | // .append(true) 486 | // .create(true) 487 | // .open(args.output)?; 488 | // // let mut writer = csv::Writer::from_writer(csv_file); 489 | // let writer = csv::Writer::from_writer(csv_file); 490 | // output_csv(writer, results)?; 491 | // } 492 | OutputFormat::TSV => { 493 | let csv_file = OpenOptions::new() 494 | .append(true) 495 | .create(true) 496 | .open(args.output)?; 497 | // let mut writer = csv::WriterBuilder::new() 498 | let writer = csv::WriterBuilder::new() 499 | .delimiter(b'\t') 500 | .from_writer(csv_file); 501 | output_csv(writer, results)?; 502 | } 503 | } 504 | Ok(()) 505 | } 506 | 507 | fn output_sqlite(path: &str, results: &Vec) -> Result<(), Box> { 508 | let conn = Connection::open(path)?; 509 | let mut stmt = conn.prepare("INSERT INTO UnifiedLogs ( 510 | File, \ 511 | DecompFilePos, \ 512 | ContinuousTime, \ 513 | TimeUtc, \ 514 | Thread, \ 515 | Type, \ 516 | ActivityID, \ 517 | ParentActivityID, \ 518 | ProcessID, \ 519 | EffectiveUID, \ 520 | TTL, \ 521 | ProcessName, \ 522 | SenderName, \ 523 | Subsystem, \ 524 | Category, \ 525 | SignpostName, \ 526 | SignpostInfo, \ 527 | ImageOffset, \ 528 | SenderUUID, \ 529 | ProcessImageUUID, \ 530 | SenderImagePath, \ 531 | ProcessImagePath, \ 532 | Message 533 | ) 534 | VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, \ 535 | ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, \ 536 | ?20, ?21, ?22, ?23);")?; 537 | 538 | for data in results { 539 | let date_time = Utc.timestamp_nanos(data.time as i64); 540 | 541 | let process_path = data.process.split("/").collect::>(); 542 | let process_name = process_path.last().unwrap().to_string(); 543 | 544 | let library_path = data.library.split("/").collect::>(); 545 | let library_name = library_path.last().unwrap().to_string(); 546 | 547 | let mut library_uuid = String::new(); 548 | if !data.library_uuid.is_empty() { 549 | library_uuid = String::from(&data.library_uuid[0..8]) 550 | + "-" 551 | + &data.library_uuid[8..12] 552 | + "-" 553 | + &data.library_uuid[12..16] 554 | + "-" 555 | + &data.library_uuid[16..20] 556 | + "-" 557 | + &data.library_uuid[20..]; 558 | } 559 | 560 | let mut process_uuid = String::new(); 561 | if !data.process_uuid.is_empty() { 562 | process_uuid = String::from(&data.process_uuid[0..8]) 563 | + "-" 564 | + &data.process_uuid[8..12] 565 | + "-" 566 | + &data.process_uuid[12..16] 567 | + "-" 568 | + &data.process_uuid[16..20] 569 | + "-" 570 | + &data.process_uuid[20..]; 571 | } 572 | 573 | stmt.execute(params![ 574 | "".to_string(), 575 | 0.to_string(), 576 | "0".to_string(), 577 | date_time.to_rfc3339_opts(SecondsFormat::Micros, true), 578 | data.thread_id.to_string(), 579 | data.log_type.to_owned(), 580 | data.activity_id.to_string(), 581 | 0.to_string(), 582 | data.pid.to_string(), 583 | data.euid.to_string(), 584 | 0.to_string(), 585 | process_name, 586 | library_name, 587 | data.subsystem.to_owned(), 588 | data.category.to_owned(), 589 | "".to_string(), 590 | "".to_string(), 591 | 0.to_string(), 592 | library_uuid.to_string(), 593 | process_uuid.to_string(), 594 | data.library.to_owned(), 595 | data.process.to_owned(), 596 | data.message.to_owned(), 597 | ])?; 598 | } 599 | stmt.finalize()?; 600 | conn.close().unwrap(); 601 | Ok(()) 602 | } 603 | 604 | fn output_csv( 605 | mut writer: Writer, 606 | results: &Vec, 607 | ) -> Result<(), Box> { 608 | for data in results { 609 | let date_time = Utc.timestamp_nanos(data.time as i64); 610 | 611 | let process_path = data.process.split("/").collect::>(); 612 | let process_name = process_path.last().unwrap().to_string(); 613 | 614 | let library_path = data.library.split("/").collect::>(); 615 | let library_name = library_path.last().unwrap().to_string(); 616 | 617 | let mut library_uuid = String::new(); 618 | if !data.library_uuid.is_empty() { 619 | library_uuid = String::from(&data.library_uuid[0..8]) 620 | + "-" 621 | + &data.library_uuid[8..12] 622 | + "-" 623 | + &data.library_uuid[12..16] 624 | + "-" 625 | + &data.library_uuid[16..20] 626 | + "-" 627 | + &data.library_uuid[20..]; 628 | } 629 | 630 | let mut process_uuid = String::new(); 631 | if !data.process_uuid.is_empty() { 632 | process_uuid = String::from(&data.process_uuid[0..8]) 633 | + "-" 634 | + &data.process_uuid[8..12] 635 | + "-" 636 | + &data.process_uuid[12..16] 637 | + "-" 638 | + &data.process_uuid[16..20] 639 | + "-" 640 | + &data.process_uuid[20..]; 641 | } 642 | 643 | writer.write_record(&[ 644 | "".to_string(), 645 | "0".to_string(), 646 | "0".to_string(), 647 | date_time.to_rfc3339_opts(SecondsFormat::Micros, true), 648 | data.thread_id.to_string(), 649 | data.log_type.to_owned(), 650 | data.activity_id.to_string(), 651 | "0".to_string(), 652 | data.pid.to_string(), 653 | data.euid.to_string(), 654 | "0".to_string(), 655 | process_name, 656 | library_name, 657 | data.subsystem.to_owned(), 658 | data.category.to_owned(), 659 | "".to_string(), 660 | "".to_string(), 661 | "0".to_string(), 662 | library_uuid.to_string(), 663 | process_uuid.to_string(), 664 | data.library.to_owned(), 665 | data.process.to_owned(), 666 | data.message.to_owned(), 667 | ])?; 668 | } 669 | Ok(()) 670 | } 671 | -------------------------------------------------------------------------------- /helper_tools/aul2madb/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "ahash" 7 | version = "0.8.3" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" 10 | dependencies = [ 11 | "cfg-if", 12 | "once_cell", 13 | "version_check", 14 | ] 15 | 16 | [[package]] 17 | name = "aho-corasick" 18 | version = "1.0.2" 19 | source = "registry+https://github.com/rust-lang/crates.io-index" 20 | checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" 21 | dependencies = [ 22 | "memchr", 23 | ] 24 | 25 | [[package]] 26 | name = "allocator-api2" 27 | version = "0.2.14" 28 | source = "registry+https://github.com/rust-lang/crates.io-index" 29 | checksum = "c4f263788a35611fba42eb41ff811c5d0360c58b97402570312a350736e2542e" 30 | 31 | [[package]] 32 | name = "android-tzdata" 33 | version = "0.1.1" 34 | source = "registry+https://github.com/rust-lang/crates.io-index" 35 | checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" 36 | 37 | [[package]] 38 | name = "android_system_properties" 39 | version = "0.1.5" 40 | source = "registry+https://github.com/rust-lang/crates.io-index" 41 | checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" 42 | dependencies = [ 43 | "libc", 44 | ] 45 | 46 | [[package]] 47 | name = "anstream" 48 | version = "0.3.2" 49 | source = "registry+https://github.com/rust-lang/crates.io-index" 50 | checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" 51 | dependencies = [ 52 | "anstyle", 53 | "anstyle-parse", 54 | "anstyle-query", 55 | "anstyle-wincon", 56 | "colorchoice", 57 | "is-terminal", 58 | "utf8parse", 59 | ] 60 | 61 | [[package]] 62 | name = "anstyle" 63 | version = "1.0.0" 64 | source = "registry+https://github.com/rust-lang/crates.io-index" 65 | checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" 66 | 67 | [[package]] 68 | name = "anstyle-parse" 69 | version = "0.2.0" 70 | source = "registry+https://github.com/rust-lang/crates.io-index" 71 | checksum = "e765fd216e48e067936442276d1d57399e37bce53c264d6fefbe298080cb57ee" 72 | dependencies = [ 73 | "utf8parse", 74 | ] 75 | 76 | [[package]] 77 | name = "anstyle-query" 78 | version = "1.0.0" 79 | source = "registry+https://github.com/rust-lang/crates.io-index" 80 | checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" 81 | dependencies = [ 82 | "windows-sys", 83 | ] 84 | 85 | [[package]] 86 | name = "anstyle-wincon" 87 | version = "1.0.1" 88 | source = "registry+https://github.com/rust-lang/crates.io-index" 89 | checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" 90 | dependencies = [ 91 | "anstyle", 92 | "windows-sys", 93 | ] 94 | 95 | [[package]] 96 | name = "aul2madb" 97 | version = "0.0.2" 98 | dependencies = [ 99 | "chrono", 100 | "clap", 101 | "csv", 102 | "dunce", 103 | "macos-unifiedlogs", 104 | "rusqlite", 105 | "simplelog", 106 | ] 107 | 108 | [[package]] 109 | name = "autocfg" 110 | version = "1.1.0" 111 | source = "registry+https://github.com/rust-lang/crates.io-index" 112 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 113 | 114 | [[package]] 115 | name = "base64" 116 | version = "0.21.2" 117 | source = "registry+https://github.com/rust-lang/crates.io-index" 118 | checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" 119 | 120 | [[package]] 121 | name = "bitflags" 122 | version = "1.3.2" 123 | source = "registry+https://github.com/rust-lang/crates.io-index" 124 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 125 | 126 | [[package]] 127 | name = "bitflags" 128 | version = "2.3.1" 129 | source = "registry+https://github.com/rust-lang/crates.io-index" 130 | checksum = "6776fc96284a0bb647b615056fc496d1fe1644a7ab01829818a6d91cae888b84" 131 | 132 | [[package]] 133 | name = "bumpalo" 134 | version = "3.13.0" 135 | source = "registry+https://github.com/rust-lang/crates.io-index" 136 | checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" 137 | 138 | [[package]] 139 | name = "byteorder" 140 | version = "1.4.3" 141 | source = "registry+https://github.com/rust-lang/crates.io-index" 142 | checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" 143 | 144 | [[package]] 145 | name = "cc" 146 | version = "1.0.79" 147 | source = "registry+https://github.com/rust-lang/crates.io-index" 148 | checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" 149 | 150 | [[package]] 151 | name = "cfg-if" 152 | version = "1.0.0" 153 | source = "registry+https://github.com/rust-lang/crates.io-index" 154 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 155 | 156 | [[package]] 157 | name = "chrono" 158 | version = "0.4.26" 159 | source = "registry+https://github.com/rust-lang/crates.io-index" 160 | checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" 161 | dependencies = [ 162 | "android-tzdata", 163 | "iana-time-zone", 164 | "js-sys", 165 | "num-traits", 166 | "time 0.1.45", 167 | "wasm-bindgen", 168 | "winapi", 169 | ] 170 | 171 | [[package]] 172 | name = "clap" 173 | version = "4.3.3" 174 | source = "registry+https://github.com/rust-lang/crates.io-index" 175 | checksum = "ca8f255e4b8027970e78db75e78831229c9815fdbfa67eb1a1b777a62e24b4a0" 176 | dependencies = [ 177 | "clap_builder", 178 | "clap_derive", 179 | "once_cell", 180 | ] 181 | 182 | [[package]] 183 | name = "clap_builder" 184 | version = "4.3.3" 185 | source = "registry+https://github.com/rust-lang/crates.io-index" 186 | checksum = "acd4f3c17c83b0ba34ffbc4f8bbd74f079413f747f84a6f89292f138057e36ab" 187 | dependencies = [ 188 | "anstream", 189 | "anstyle", 190 | "bitflags 1.3.2", 191 | "clap_lex", 192 | "strsim", 193 | ] 194 | 195 | [[package]] 196 | name = "clap_derive" 197 | version = "4.3.2" 198 | source = "registry+https://github.com/rust-lang/crates.io-index" 199 | checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" 200 | dependencies = [ 201 | "heck", 202 | "proc-macro2", 203 | "quote", 204 | "syn", 205 | ] 206 | 207 | [[package]] 208 | name = "clap_lex" 209 | version = "0.5.0" 210 | source = "registry+https://github.com/rust-lang/crates.io-index" 211 | checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" 212 | 213 | [[package]] 214 | name = "colorchoice" 215 | version = "1.0.0" 216 | source = "registry+https://github.com/rust-lang/crates.io-index" 217 | checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" 218 | 219 | [[package]] 220 | name = "core-foundation-sys" 221 | version = "0.8.4" 222 | source = "registry+https://github.com/rust-lang/crates.io-index" 223 | checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" 224 | 225 | [[package]] 226 | name = "csv" 227 | version = "1.2.2" 228 | source = "registry+https://github.com/rust-lang/crates.io-index" 229 | checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" 230 | dependencies = [ 231 | "csv-core", 232 | "itoa", 233 | "ryu", 234 | "serde", 235 | ] 236 | 237 | [[package]] 238 | name = "csv-core" 239 | version = "0.1.10" 240 | source = "registry+https://github.com/rust-lang/crates.io-index" 241 | checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" 242 | dependencies = [ 243 | "memchr", 244 | ] 245 | 246 | [[package]] 247 | name = "dunce" 248 | version = "1.0.4" 249 | source = "registry+https://github.com/rust-lang/crates.io-index" 250 | checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" 251 | 252 | [[package]] 253 | name = "errno" 254 | version = "0.3.1" 255 | source = "registry+https://github.com/rust-lang/crates.io-index" 256 | checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" 257 | dependencies = [ 258 | "errno-dragonfly", 259 | "libc", 260 | "windows-sys", 261 | ] 262 | 263 | [[package]] 264 | name = "errno-dragonfly" 265 | version = "0.1.2" 266 | source = "registry+https://github.com/rust-lang/crates.io-index" 267 | checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" 268 | dependencies = [ 269 | "cc", 270 | "libc", 271 | ] 272 | 273 | [[package]] 274 | name = "fallible-iterator" 275 | version = "0.2.0" 276 | source = "registry+https://github.com/rust-lang/crates.io-index" 277 | checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" 278 | 279 | [[package]] 280 | name = "fallible-streaming-iterator" 281 | version = "0.1.9" 282 | source = "registry+https://github.com/rust-lang/crates.io-index" 283 | checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" 284 | 285 | [[package]] 286 | name = "hashbrown" 287 | version = "0.12.3" 288 | source = "registry+https://github.com/rust-lang/crates.io-index" 289 | checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 290 | 291 | [[package]] 292 | name = "hashbrown" 293 | version = "0.14.0" 294 | source = "registry+https://github.com/rust-lang/crates.io-index" 295 | checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" 296 | dependencies = [ 297 | "ahash", 298 | "allocator-api2", 299 | ] 300 | 301 | [[package]] 302 | name = "hashlink" 303 | version = "0.8.3" 304 | source = "registry+https://github.com/rust-lang/crates.io-index" 305 | checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" 306 | dependencies = [ 307 | "hashbrown 0.14.0", 308 | ] 309 | 310 | [[package]] 311 | name = "heck" 312 | version = "0.4.1" 313 | source = "registry+https://github.com/rust-lang/crates.io-index" 314 | checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" 315 | 316 | [[package]] 317 | name = "hermit-abi" 318 | version = "0.3.1" 319 | source = "registry+https://github.com/rust-lang/crates.io-index" 320 | checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" 321 | 322 | [[package]] 323 | name = "iana-time-zone" 324 | version = "0.1.57" 325 | source = "registry+https://github.com/rust-lang/crates.io-index" 326 | checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" 327 | dependencies = [ 328 | "android_system_properties", 329 | "core-foundation-sys", 330 | "iana-time-zone-haiku", 331 | "js-sys", 332 | "wasm-bindgen", 333 | "windows", 334 | ] 335 | 336 | [[package]] 337 | name = "iana-time-zone-haiku" 338 | version = "0.1.2" 339 | source = "registry+https://github.com/rust-lang/crates.io-index" 340 | checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" 341 | dependencies = [ 342 | "cc", 343 | ] 344 | 345 | [[package]] 346 | name = "indexmap" 347 | version = "1.9.3" 348 | source = "registry+https://github.com/rust-lang/crates.io-index" 349 | checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" 350 | dependencies = [ 351 | "autocfg", 352 | "hashbrown 0.12.3", 353 | ] 354 | 355 | [[package]] 356 | name = "io-lifetimes" 357 | version = "1.0.11" 358 | source = "registry+https://github.com/rust-lang/crates.io-index" 359 | checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" 360 | dependencies = [ 361 | "hermit-abi", 362 | "libc", 363 | "windows-sys", 364 | ] 365 | 366 | [[package]] 367 | name = "is-terminal" 368 | version = "0.4.7" 369 | source = "registry+https://github.com/rust-lang/crates.io-index" 370 | checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" 371 | dependencies = [ 372 | "hermit-abi", 373 | "io-lifetimes", 374 | "rustix", 375 | "windows-sys", 376 | ] 377 | 378 | [[package]] 379 | name = "itoa" 380 | version = "1.0.6" 381 | source = "registry+https://github.com/rust-lang/crates.io-index" 382 | checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" 383 | 384 | [[package]] 385 | name = "js-sys" 386 | version = "0.3.63" 387 | source = "registry+https://github.com/rust-lang/crates.io-index" 388 | checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" 389 | dependencies = [ 390 | "wasm-bindgen", 391 | ] 392 | 393 | [[package]] 394 | name = "libc" 395 | version = "0.2.146" 396 | source = "registry+https://github.com/rust-lang/crates.io-index" 397 | checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" 398 | 399 | [[package]] 400 | name = "libsqlite3-sys" 401 | version = "0.26.0" 402 | source = "registry+https://github.com/rust-lang/crates.io-index" 403 | checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" 404 | dependencies = [ 405 | "pkg-config", 406 | "vcpkg", 407 | ] 408 | 409 | [[package]] 410 | name = "line-wrap" 411 | version = "0.1.1" 412 | source = "registry+https://github.com/rust-lang/crates.io-index" 413 | checksum = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9" 414 | dependencies = [ 415 | "safemem", 416 | ] 417 | 418 | [[package]] 419 | name = "linux-raw-sys" 420 | version = "0.3.8" 421 | source = "registry+https://github.com/rust-lang/crates.io-index" 422 | checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" 423 | 424 | [[package]] 425 | name = "log" 426 | version = "0.4.19" 427 | source = "registry+https://github.com/rust-lang/crates.io-index" 428 | checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" 429 | 430 | [[package]] 431 | name = "lz4_flex" 432 | version = "0.10.0" 433 | source = "registry+https://github.com/rust-lang/crates.io-index" 434 | checksum = "8b8c72594ac26bfd34f2d99dfced2edfaddfe8a476e3ff2ca0eb293d925c4f83" 435 | dependencies = [ 436 | "twox-hash", 437 | ] 438 | 439 | [[package]] 440 | name = "macos-unifiedlogs" 441 | version = "0.1.0" 442 | source = "git+https://github.com/mandiant/macos-UnifiedLogs#f6e60500bd65b65f44a4ff38cfb635be97fa44ab" 443 | dependencies = [ 444 | "base64", 445 | "byteorder", 446 | "chrono", 447 | "log", 448 | "lz4_flex", 449 | "nom", 450 | "plist", 451 | "regex", 452 | "serde", 453 | "serde_json", 454 | ] 455 | 456 | [[package]] 457 | name = "memchr" 458 | version = "2.5.0" 459 | source = "registry+https://github.com/rust-lang/crates.io-index" 460 | checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" 461 | 462 | [[package]] 463 | name = "minimal-lexical" 464 | version = "0.2.1" 465 | source = "registry+https://github.com/rust-lang/crates.io-index" 466 | checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" 467 | 468 | [[package]] 469 | name = "nom" 470 | version = "7.1.3" 471 | source = "registry+https://github.com/rust-lang/crates.io-index" 472 | checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" 473 | dependencies = [ 474 | "memchr", 475 | "minimal-lexical", 476 | ] 477 | 478 | [[package]] 479 | name = "num-traits" 480 | version = "0.2.15" 481 | source = "registry+https://github.com/rust-lang/crates.io-index" 482 | checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" 483 | dependencies = [ 484 | "autocfg", 485 | ] 486 | 487 | [[package]] 488 | name = "num_threads" 489 | version = "0.1.6" 490 | source = "registry+https://github.com/rust-lang/crates.io-index" 491 | checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" 492 | dependencies = [ 493 | "libc", 494 | ] 495 | 496 | [[package]] 497 | name = "once_cell" 498 | version = "1.18.0" 499 | source = "registry+https://github.com/rust-lang/crates.io-index" 500 | checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" 501 | 502 | [[package]] 503 | name = "pkg-config" 504 | version = "0.3.27" 505 | source = "registry+https://github.com/rust-lang/crates.io-index" 506 | checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" 507 | 508 | [[package]] 509 | name = "plist" 510 | version = "1.4.3" 511 | source = "registry+https://github.com/rust-lang/crates.io-index" 512 | checksum = "9bd9647b268a3d3e14ff09c23201133a62589c658db02bb7388c7246aafe0590" 513 | dependencies = [ 514 | "base64", 515 | "indexmap", 516 | "line-wrap", 517 | "quick-xml", 518 | "serde", 519 | "time 0.3.22", 520 | ] 521 | 522 | [[package]] 523 | name = "proc-macro2" 524 | version = "1.0.60" 525 | source = "registry+https://github.com/rust-lang/crates.io-index" 526 | checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" 527 | dependencies = [ 528 | "unicode-ident", 529 | ] 530 | 531 | [[package]] 532 | name = "quick-xml" 533 | version = "0.28.2" 534 | source = "registry+https://github.com/rust-lang/crates.io-index" 535 | checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" 536 | dependencies = [ 537 | "memchr", 538 | ] 539 | 540 | [[package]] 541 | name = "quote" 542 | version = "1.0.28" 543 | source = "registry+https://github.com/rust-lang/crates.io-index" 544 | checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" 545 | dependencies = [ 546 | "proc-macro2", 547 | ] 548 | 549 | [[package]] 550 | name = "regex" 551 | version = "1.8.4" 552 | source = "registry+https://github.com/rust-lang/crates.io-index" 553 | checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" 554 | dependencies = [ 555 | "aho-corasick", 556 | "memchr", 557 | "regex-syntax", 558 | ] 559 | 560 | [[package]] 561 | name = "regex-syntax" 562 | version = "0.7.2" 563 | source = "registry+https://github.com/rust-lang/crates.io-index" 564 | checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" 565 | 566 | [[package]] 567 | name = "rusqlite" 568 | version = "0.29.0" 569 | source = "registry+https://github.com/rust-lang/crates.io-index" 570 | checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" 571 | dependencies = [ 572 | "bitflags 2.3.1", 573 | "fallible-iterator", 574 | "fallible-streaming-iterator", 575 | "hashlink", 576 | "libsqlite3-sys", 577 | "smallvec", 578 | ] 579 | 580 | [[package]] 581 | name = "rustix" 582 | version = "0.37.20" 583 | source = "registry+https://github.com/rust-lang/crates.io-index" 584 | checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" 585 | dependencies = [ 586 | "bitflags 1.3.2", 587 | "errno", 588 | "io-lifetimes", 589 | "libc", 590 | "linux-raw-sys", 591 | "windows-sys", 592 | ] 593 | 594 | [[package]] 595 | name = "ryu" 596 | version = "1.0.13" 597 | source = "registry+https://github.com/rust-lang/crates.io-index" 598 | checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" 599 | 600 | [[package]] 601 | name = "safemem" 602 | version = "0.3.3" 603 | source = "registry+https://github.com/rust-lang/crates.io-index" 604 | checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" 605 | 606 | [[package]] 607 | name = "serde" 608 | version = "1.0.164" 609 | source = "registry+https://github.com/rust-lang/crates.io-index" 610 | checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" 611 | dependencies = [ 612 | "serde_derive", 613 | ] 614 | 615 | [[package]] 616 | name = "serde_derive" 617 | version = "1.0.164" 618 | source = "registry+https://github.com/rust-lang/crates.io-index" 619 | checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" 620 | dependencies = [ 621 | "proc-macro2", 622 | "quote", 623 | "syn", 624 | ] 625 | 626 | [[package]] 627 | name = "serde_json" 628 | version = "1.0.96" 629 | source = "registry+https://github.com/rust-lang/crates.io-index" 630 | checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" 631 | dependencies = [ 632 | "itoa", 633 | "ryu", 634 | "serde", 635 | ] 636 | 637 | [[package]] 638 | name = "simplelog" 639 | version = "0.12.1" 640 | source = "registry+https://github.com/rust-lang/crates.io-index" 641 | checksum = "acee08041c5de3d5048c8b3f6f13fafb3026b24ba43c6a695a0c76179b844369" 642 | dependencies = [ 643 | "log", 644 | "termcolor", 645 | "time 0.3.22", 646 | ] 647 | 648 | [[package]] 649 | name = "smallvec" 650 | version = "1.10.0" 651 | source = "registry+https://github.com/rust-lang/crates.io-index" 652 | checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" 653 | 654 | [[package]] 655 | name = "static_assertions" 656 | version = "1.1.0" 657 | source = "registry+https://github.com/rust-lang/crates.io-index" 658 | checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" 659 | 660 | [[package]] 661 | name = "strsim" 662 | version = "0.10.0" 663 | source = "registry+https://github.com/rust-lang/crates.io-index" 664 | checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" 665 | 666 | [[package]] 667 | name = "syn" 668 | version = "2.0.18" 669 | source = "registry+https://github.com/rust-lang/crates.io-index" 670 | checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" 671 | dependencies = [ 672 | "proc-macro2", 673 | "quote", 674 | "unicode-ident", 675 | ] 676 | 677 | [[package]] 678 | name = "termcolor" 679 | version = "1.1.3" 680 | source = "registry+https://github.com/rust-lang/crates.io-index" 681 | checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" 682 | dependencies = [ 683 | "winapi-util", 684 | ] 685 | 686 | [[package]] 687 | name = "time" 688 | version = "0.1.45" 689 | source = "registry+https://github.com/rust-lang/crates.io-index" 690 | checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" 691 | dependencies = [ 692 | "libc", 693 | "wasi", 694 | "winapi", 695 | ] 696 | 697 | [[package]] 698 | name = "time" 699 | version = "0.3.22" 700 | source = "registry+https://github.com/rust-lang/crates.io-index" 701 | checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" 702 | dependencies = [ 703 | "itoa", 704 | "libc", 705 | "num_threads", 706 | "serde", 707 | "time-core", 708 | "time-macros", 709 | ] 710 | 711 | [[package]] 712 | name = "time-core" 713 | version = "0.1.1" 714 | source = "registry+https://github.com/rust-lang/crates.io-index" 715 | checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" 716 | 717 | [[package]] 718 | name = "time-macros" 719 | version = "0.2.9" 720 | source = "registry+https://github.com/rust-lang/crates.io-index" 721 | checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" 722 | dependencies = [ 723 | "time-core", 724 | ] 725 | 726 | [[package]] 727 | name = "twox-hash" 728 | version = "1.6.3" 729 | source = "registry+https://github.com/rust-lang/crates.io-index" 730 | checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" 731 | dependencies = [ 732 | "cfg-if", 733 | "static_assertions", 734 | ] 735 | 736 | [[package]] 737 | name = "unicode-ident" 738 | version = "1.0.9" 739 | source = "registry+https://github.com/rust-lang/crates.io-index" 740 | checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" 741 | 742 | [[package]] 743 | name = "utf8parse" 744 | version = "0.2.1" 745 | source = "registry+https://github.com/rust-lang/crates.io-index" 746 | checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" 747 | 748 | [[package]] 749 | name = "vcpkg" 750 | version = "0.2.15" 751 | source = "registry+https://github.com/rust-lang/crates.io-index" 752 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 753 | 754 | [[package]] 755 | name = "version_check" 756 | version = "0.9.4" 757 | source = "registry+https://github.com/rust-lang/crates.io-index" 758 | checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" 759 | 760 | [[package]] 761 | name = "wasi" 762 | version = "0.10.0+wasi-snapshot-preview1" 763 | source = "registry+https://github.com/rust-lang/crates.io-index" 764 | checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" 765 | 766 | [[package]] 767 | name = "wasm-bindgen" 768 | version = "0.2.86" 769 | source = "registry+https://github.com/rust-lang/crates.io-index" 770 | checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" 771 | dependencies = [ 772 | "cfg-if", 773 | "wasm-bindgen-macro", 774 | ] 775 | 776 | [[package]] 777 | name = "wasm-bindgen-backend" 778 | version = "0.2.86" 779 | source = "registry+https://github.com/rust-lang/crates.io-index" 780 | checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" 781 | dependencies = [ 782 | "bumpalo", 783 | "log", 784 | "once_cell", 785 | "proc-macro2", 786 | "quote", 787 | "syn", 788 | "wasm-bindgen-shared", 789 | ] 790 | 791 | [[package]] 792 | name = "wasm-bindgen-macro" 793 | version = "0.2.86" 794 | source = "registry+https://github.com/rust-lang/crates.io-index" 795 | checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" 796 | dependencies = [ 797 | "quote", 798 | "wasm-bindgen-macro-support", 799 | ] 800 | 801 | [[package]] 802 | name = "wasm-bindgen-macro-support" 803 | version = "0.2.86" 804 | source = "registry+https://github.com/rust-lang/crates.io-index" 805 | checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" 806 | dependencies = [ 807 | "proc-macro2", 808 | "quote", 809 | "syn", 810 | "wasm-bindgen-backend", 811 | "wasm-bindgen-shared", 812 | ] 813 | 814 | [[package]] 815 | name = "wasm-bindgen-shared" 816 | version = "0.2.86" 817 | source = "registry+https://github.com/rust-lang/crates.io-index" 818 | checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" 819 | 820 | [[package]] 821 | name = "winapi" 822 | version = "0.3.9" 823 | source = "registry+https://github.com/rust-lang/crates.io-index" 824 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 825 | dependencies = [ 826 | "winapi-i686-pc-windows-gnu", 827 | "winapi-x86_64-pc-windows-gnu", 828 | ] 829 | 830 | [[package]] 831 | name = "winapi-i686-pc-windows-gnu" 832 | version = "0.4.0" 833 | source = "registry+https://github.com/rust-lang/crates.io-index" 834 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 835 | 836 | [[package]] 837 | name = "winapi-util" 838 | version = "0.1.5" 839 | source = "registry+https://github.com/rust-lang/crates.io-index" 840 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 841 | dependencies = [ 842 | "winapi", 843 | ] 844 | 845 | [[package]] 846 | name = "winapi-x86_64-pc-windows-gnu" 847 | version = "0.4.0" 848 | source = "registry+https://github.com/rust-lang/crates.io-index" 849 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 850 | 851 | [[package]] 852 | name = "windows" 853 | version = "0.48.0" 854 | source = "registry+https://github.com/rust-lang/crates.io-index" 855 | checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" 856 | dependencies = [ 857 | "windows-targets", 858 | ] 859 | 860 | [[package]] 861 | name = "windows-sys" 862 | version = "0.48.0" 863 | source = "registry+https://github.com/rust-lang/crates.io-index" 864 | checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" 865 | dependencies = [ 866 | "windows-targets", 867 | ] 868 | 869 | [[package]] 870 | name = "windows-targets" 871 | version = "0.48.0" 872 | source = "registry+https://github.com/rust-lang/crates.io-index" 873 | checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" 874 | dependencies = [ 875 | "windows_aarch64_gnullvm", 876 | "windows_aarch64_msvc", 877 | "windows_i686_gnu", 878 | "windows_i686_msvc", 879 | "windows_x86_64_gnu", 880 | "windows_x86_64_gnullvm", 881 | "windows_x86_64_msvc", 882 | ] 883 | 884 | [[package]] 885 | name = "windows_aarch64_gnullvm" 886 | version = "0.48.0" 887 | source = "registry+https://github.com/rust-lang/crates.io-index" 888 | checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" 889 | 890 | [[package]] 891 | name = "windows_aarch64_msvc" 892 | version = "0.48.0" 893 | source = "registry+https://github.com/rust-lang/crates.io-index" 894 | checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" 895 | 896 | [[package]] 897 | name = "windows_i686_gnu" 898 | version = "0.48.0" 899 | source = "registry+https://github.com/rust-lang/crates.io-index" 900 | checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" 901 | 902 | [[package]] 903 | name = "windows_i686_msvc" 904 | version = "0.48.0" 905 | source = "registry+https://github.com/rust-lang/crates.io-index" 906 | checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" 907 | 908 | [[package]] 909 | name = "windows_x86_64_gnu" 910 | version = "0.48.0" 911 | source = "registry+https://github.com/rust-lang/crates.io-index" 912 | checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" 913 | 914 | [[package]] 915 | name = "windows_x86_64_gnullvm" 916 | version = "0.48.0" 917 | source = "registry+https://github.com/rust-lang/crates.io-index" 918 | checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" 919 | 920 | [[package]] 921 | name = "windows_x86_64_msvc" 922 | version = "0.48.0" 923 | source = "registry+https://github.com/rust-lang/crates.io-index" 924 | checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" 925 | -------------------------------------------------------------------------------- /plugins/prog_exec.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2021-2023 Minoru Kobayashi 3 | # 4 | # This file is part of ma2tl. 5 | # Usage or distribution of this code is subject to the terms of the MIT License. 6 | # 7 | 8 | from __future__ import annotations 9 | 10 | import datetime 11 | import json 12 | import logging 13 | import os 14 | import re 15 | 16 | from plugins.helpers.basic_info import BasicInfo, MacAptDBType 17 | from plugins.helpers.common import get_timedelta 18 | 19 | PLUGIN_NAME = os.path.splitext(os.path.basename(__file__))[0].upper() 20 | PLUGIN_DESCRIPTION = "Extract program execution activities." 21 | PLUGIN_ACTIVITY_TYPE = "Program Execution" 22 | PLUGIN_VERSION = "20230830" 23 | PLUGIN_AUTHOR = "Minoru Kobayashi" 24 | PLUGIN_AUTHOR_EMAIL = "unknownbit@gmail.com" 25 | 26 | log = None 27 | ignore_processes = ('activateSettings', 'QuickLookUIService', 'com.apple.dock.extra') 28 | ignore_tccd_processes = ( 29 | '/Library/Application Support/VMware Tools/vmware-tools-daemon', 30 | '/usr/libexec/UserEventAgent', 31 | '/System/Library/CoreServices/Installer Progress.app/Contents/MacOS/Installer Progress', 32 | '/System/Library/PreferencePanes/Displays.prefPane/Contents/Resources/MirrorDisplays.app/Contents/MacOS/MirrorDisplays', 33 | '/System/Library/PrivateFrameworks/AmbientDisplay.framework/Versions/A/XPCServices/com.apple.AmbientDisplayAgent.xpc/Contents/MacOS/com.apple.AmbientDisplayAgent', 34 | '/System/Library/Frameworks/Security.framework/Versions/A/MachServices/SecurityAgent.bundle/Contents/MacOS/SecurityAgent', 35 | '/System/Library/PrivateFrameworks/SystemAdministration.framework/Versions/A/Resources/activateSettings', 36 | '/System/Library/CoreServices/SystemUIServer.app/Contents/MacOS/SystemUIServer', 37 | '/System/Library/CoreServices/talagent', 38 | '/System/Library/CoreServices/ControlCenter.app/Contents/MacOS/ControlCenter', 39 | '/System/Library/CoreServices/CoreLocationAgent.app/Contents/MacOS/CoreLocationAgent', 40 | '/System/Library/PrivateFrameworks/DoNotDisturbServer.framework/Support/donotdisturbd', 41 | '/System/Library/CoreServices/Dock.app/Contents/XPCServices/com.apple.dock.extra.xpc/Contents/MacOS/com.apple.dock.extra', 42 | '/System/Library/CoreServices/Spotlight.app/Contents/MacOS/Spotlight', 43 | '/usr/sbin/cfprefsd', 44 | '/System/Library/Frameworks/QuickLookUI.framework/Versions/A/XPCServices/QuickLookUIService.xpc/Contents/MacOS/QuickLookUIService', 45 | '/System/Library/CoreServices/TextInputMenuAgent.app/Contents/MacOS/TextInputMenuAgent', 46 | '/System/Library/CoreServices/AirPlayUIAgent.app/Contents/MacOS/AirPlayUIAgent', 47 | '/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/Metadata.framework/Versions/A/Support/corespotlightd', 48 | '/System/Library/PrivateFrameworks/CalendarDaemon.framework/Support/calaccessd', 49 | '/usr/libexec/sharingd', 50 | '/System/Library/PrivateFrameworks/CoreSuggestions.framework/Versions/A/Support/suggestd', 51 | '/usr/sbin/universalaccessd', 52 | '/System/Library/PrivateFrameworks/AppSSO.framework/Support/AppSSOAgent.app/Contents/MacOS/AppSSOAgent', 53 | '/System/Applications/Calendar.app/Contents/PlugIns/CalendarWidgetExtension.appex/Contents/MacOS/CalendarWidgetExtension', 54 | '/System/Library/PrivateFrameworks/TelephonyUtilities.framework/callservicesd', 55 | '/usr/libexec/knowledge-agent', 56 | '/System/Applications/Stocks.app/Contents/PlugIns/StocksWidget.appex/Contents/MacOS/StocksWidget', 57 | '/System/Applications/Weather.app/Contents/PlugIns/WeatherWidget.appex/Contents/MacOS/WeatherWidget', 58 | '/System/Library/PrivateFrameworks/iCloudNotification.framework/iCloudNotificationAgent', 59 | '/usr/libexec/rapportd', 60 | '/System/Library/PrivateFrameworks/AssistantServices.framework/Versions/A/Support/assistantd', 61 | '/usr/libexec/ContinuityCaptureAgent', 62 | '/System/Library/PrivateFrameworks/AOSKit.framework/Versions/A/Helpers/AOSHeartbeat.app/Contents/MacOS/AOSHeartbeat', 63 | '/System/Library/CoreServices/Keychain Circle Notification.app/Contents/MacOS/Keychain Circle', 64 | '/usr/libexec/studentd', 65 | '/System/Library/Frameworks/AddressBook.framework/Versions/A/Helpers/AddressBookManager.app/Contents/MacOS/AddressBookManager', 66 | '/usr/libexec/routined', 67 | '/System/Library/PrivateFrameworks/ContactsDonation.framework/Versions/A/Support/contactsdonationagent', 68 | '/System/Library/PrivateFrameworks/HearingCore.framework/heard', 69 | '/System/Library/PrivateFrameworks/IDS.framework/identityservicesd.app/Contents/MacOS/identityservicesd', 70 | '/System/Applications/Clock.app/Contents/PlugIns/WorldClockWidget.appex/Contents/MacOS/WorldClockWidget', 71 | '/System/Library/CoreServices/CoreServicesUIAgent.app/Contents/MacOS/CoreServicesUIAgent', 72 | '/System/Library/CoreServices/Finder.app/Contents/MacOS/Finder', 73 | '/System/Library/CoreServices/Keychain Circle Notification.app/Contents/MacOS/Keychain Circle Notification', 74 | '/System/Library/CoreServices/Screen Time.app/Contents/PlugIns/ScreenTimeWidgetExtension.appex/Contents/MacOS/ScreenTimeWidgetExtension', 75 | '/System/Library/CoreServices/UIKitSystem.app/Contents/MacOS/UIKitSystem', 76 | '/System/Library/CoreServices/UserNotificationCenter.app/Contents/MacOS/UserNotificationCenter', 77 | '/System/Library/CoreServices/WiFiAgent.app/Contents/MacOS/WiFiAgent', 78 | '/System/Library/CoreServices/diagnostics_agent', 79 | '/System/Library/Frameworks/AddressBook.framework/Versions/A/Helpers/AddressBookSourceSync.app/Contents/MacOS/AddressBookSourceSync', 80 | '/System/Library/Frameworks/AppKit.framework/Versions/C/XPCServices/com.apple.appkit.xpc.openAndSavePanelService.xpc/Contents/MacOS/com.apple.appkit.xpc.openAndSavePanelService', 81 | '/System/Library/PrivateFrameworks/AMPLibrary.framework/Versions/A/Support/AMPLibraryAgent', 82 | '/System/Library/PrivateFrameworks/AppSSO.framework/Support/AppSSODaemon', 83 | '/System/Library/PrivateFrameworks/AppStoreDaemon.framework/Support/appstoreagent', 84 | '/System/Library/PrivateFrameworks/AppleMediaServices.framework/Versions/A/Resources/amsaccountsd', 85 | '/System/Library/PrivateFrameworks/AskPermission.framework/Versions/A/Resources/askpermissiond', 86 | '/System/Library/PrivateFrameworks/BookKit.framework/Versions/A/XPCServices/com.apple.BKAgentService.xpc/Contents/MacOS/com.apple.BKAgentService', 87 | '/System/Library/PrivateFrameworks/CallHistory.framework/Support/CallHistoryPluginHelper', 88 | '/System/Library/PrivateFrameworks/DataAccess.framework/Support/dataaccessd', 89 | '/System/Library/PrivateFrameworks/ExchangeSync.framework/Versions/A/exchangesyncd', 90 | '/System/Library/PrivateFrameworks/FamilyCircle.framework/Versions/A/Resources/familycircled', 91 | '/System/Library/PrivateFrameworks/IMCore.framework/imagent.app/Contents/MacOS/imagent', 92 | '/System/Library/PrivateFrameworks/IMDPersistence.framework/XPCServices/IMDPersistenceAgent.xpc/Contents/MacOS/IMDPersistenceAgent', 93 | '/System/Library/PrivateFrameworks/MediaAnalysis.framework/Versions/A/mediaanalysisd', 94 | '/System/Library/PrivateFrameworks/NewDeviceOutreach.framework/ndoagent', 95 | '/System/Library/PrivateFrameworks/Noticeboard.framework/Versions/A/Resources/nbagent.app/Contents/MacOS/nbagent', 96 | '/System/Library/PrivateFrameworks/PassKitCore.framework/passd', 97 | '/System/Library/PrivateFrameworks/People.framework/peopled', 98 | '/System/Library/PrivateFrameworks/PhotoAnalysis.framework/Versions/A/Support/photoanalysisd', 99 | '/System/Library/PrivateFrameworks/PhotoLibraryServices.framework/Versions/A/Support/photolibraryd', 100 | '/System/Library/PrivateFrameworks/SafariSafeBrowsing.framework/Versions/A/com.apple.Safari.SafeBrowsing.Service', 101 | '/System/Library/PrivateFrameworks/SoftwareUpdate.framework/Versions/A/Resources/SoftwareUpdateNotificationManager.app/Contents/MacOS/SoftwareUpdateNotificationManager', 102 | '/System/Library/PrivateFrameworks/Translation.framework/translationd', 103 | '/System/Library/PrivateFrameworks/VoiceShortcuts.framework/Versions/A/Support/siriactionsd', 104 | '/System/Library/PrivateFrameworks/iTunesCloud.framework/Support/itunescloudd', 105 | '/System/Library/Services/AppleSpell.service/Contents/MacOS/AppleSpell', 106 | '/System/Volumes/Preboot/Cryptexes/App/System/Applications/Safari.app/Contents/MacOS/Safari', 107 | '/System/Volumes/Preboot/Cryptexes/App/System/Applications/Safari.app/Contents/XPCServices/com.apple.Safari.SandboxBroker.xpc/Contents/MacOS/com.apple.Safari.SandboxBroker', 108 | '/System/Volumes/Preboot/Cryptexes/Incoming/OS/System/Library/Frameworks/WebKit.framework/Versions/A/XPCServices/com.apple.WebKit.GPU.xpc/Contents/MacOS/com.apple.WebKit.GPU', 109 | '/usr/libexec/AssetCache/AssetCache', 110 | '/usr/libexec/DataDetectorsLocalSources', 111 | '/usr/libexec/biomesyncd', 112 | '/usr/libexec/periodic-wrapper', 113 | '/usr/libexec/siriknowledged', 114 | '/usr/libexec/tipsd', 115 | '/System/Library/Frameworks/CoreMediaIO.framework/Versions/A/Resources/VDC.plugin/Contents/Resources/VDCAssistant', 116 | '/System/Library/CoreServices/NotificationCenter.app/Contents/MacOS/NotificationCenter', 117 | '/System/Library/CoreServices/Software Update.app/Contents/Resources/softwareupdated', 118 | '/System/Library/CoreServices/cloudpaird', 119 | '/System/Library/PrivateFrameworks/MediaRemote.framework/Support/mediaremoted', 120 | '/usr/libexec/PerfPowerServices', 121 | '/System/Library/PrivateFrameworks/SkyLight.framework/Versions/A/Resources/WindowServer', 122 | '/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/Metadata.framework/Versions/A/Support/mds', 123 | '/System/Library/Frameworks/Contacts.framework/Support/contactsd', 124 | '/System/Applications/FindMy.app/Contents/PlugIns/FindMyWidgetIntentsPeople.appex/Contents/MacOS/FindMyWidgetIntentsPeople', 125 | '/System/Library/ExtensionKit/Extensions/UsersGroups.appex/Contents/MacOS/UsersGroups', 126 | '/System/Library/PrivateFrameworks/IntelligencePlatformCore.framework/Versions/A/knowledgeconstructiond', 127 | '/usr/sbin/bluetoothd', 128 | '/System/Applications/FindMy.app/Contents/PlugIns/FindMyWidgetItems.appex/Contents/MacOS/FindMyWidgetItems', 129 | '/System/Applications/FindMy.app/Contents/PlugIns/FindMyWidgetPeople.appex/Contents/MacOS/FindMyWidgetPeople', 130 | '/System/Applications/Notes.app/Contents/PlugIns/com.apple.Notes.IntentsExtension.appex/Contents/MacOS/com.apple.Notes.IntentsExtension', 131 | '/System/Applications/Photos.app/Contents/PlugIns/PhotosReliveWidget.appex/Contents/MacOS/PhotosReliveWidget', 132 | '/System/Applications/Reminders.app/Contents/PlugIns/RemindersIntentsExtension.appex/Contents/MacOS/RemindersIntentsExtension', 133 | '/System/Library/CoreServices/mapspushd', 134 | '/System/Library/ExtensionKit/Extensions/Appearance.appex/Contents/MacOS/Appearance', 135 | '/System/Library/ExtensionKit/Extensions/CDs & DVDs Settings Extension.appex/Contents/MacOS/CDs & DVDs Settings Extension', 136 | '/System/Library/ExtensionKit/Extensions/ClassKitSettings.appex/Contents/MacOS/ClassKitSettings', 137 | '/System/Library/ExtensionKit/Extensions/ClassroomSettings.appex/Contents/MacOS/ClassroomSettings', 138 | '/System/Library/ExtensionKit/Extensions/ControlCenterSettings.appex/Contents/MacOS/ControlCenterSettings', 139 | '/System/Library/ExtensionKit/Extensions/FamilySettings.appex/Contents/MacOS/FamilySettings', 140 | '/System/Library/ExtensionKit/Extensions/FollowUpSettingsExtension.appex/Contents/MacOS/FollowUpSettingsExtension', 141 | '/System/Library/ExtensionKit/Extensions/GameControllerMacSettings.appex/Contents/MacOS/GameControllerMacSettings', 142 | '/System/Library/ExtensionKit/Extensions/HeadphoneSettingsExtension.appex/Contents/MacOS/HeadphoneSettingsExtension', 143 | '/System/Library/ExtensionKit/Extensions/MouseExtension.appex/Contents/MacOS/MouseExtension', 144 | '/System/Library/ExtensionKit/Extensions/PowerPreferences.appex/Contents/MacOS/PowerPreferences', 145 | '/System/Library/ExtensionKit/Extensions/Touch ID & Password.appex/Contents/MacOS/Touch ID & Password', 146 | '/System/Library/ExtensionKit/Extensions/AppleIDSettings.appex/Contents/MacOS/AppleIDSettings', 147 | '/System/Library/ExtensionKit/Extensions/WalletSettingsExtension.appex/Contents/MacOS/WalletSettingsExtension', 148 | '/System/Library/ExtensionKit/Extensions/TrackpadExtension.appex/Contents/MacOS/TrackpadExtension', 149 | '/System/Library/ExtensionKit/Extensions/VPN.appex/Contents/MacOS/VPN', 150 | '/System/Library/ExtensionKit/Extensions/LoginItems.appex/Contents/MacOS/LoginItems', 151 | '/System/iOSSupport/System/Library/PrivateFrameworks/AvatarUI.framework/PlugIns/AvatarPickerMemojiPicker.appex/Contents/MacOS/AvatarPickerMemojiPicker', 152 | '/System/Library/PrivateFrameworks/EFILogin.framework/Versions/A/Resources/efilogin-helper', 153 | '/System/Library/PrivateFrameworks/HomeKitDaemon.framework/Support/homed', 154 | '/usr/libexec/fmfd', 155 | '/System/Library/PrivateFrameworks/ScreenTimeCore.framework/Versions/A/ScreenTimeAgent', 156 | '/System/Library/CoreServices/Setup Assistant.app/Contents/MacOS/Setup Assistant', 157 | '/System/Library/CoreServices/Setup Assistant.app/Contents/Resources/mbuseragent', 158 | '/System/Applications/System Settings.app/Contents/PlugIns/GeneralSettings.appex/Contents/MacOS/GeneralSettings', 159 | '/System/Applications/Notes.app/Contents/PlugIns/com.apple.Notes.WidgetExtension.appex/Contents/MacOS/com.apple.Notes.WidgetExtension', 160 | '/System/Library/ExtensionKit/Extensions/WiFiSettings.appex/Contents/MacOS/WiFiSettings', 161 | '/System/Library/ExtensionKit/Extensions/Network.appex/Contents/MacOS/Network', 162 | '/usr/sbin/spindump', 163 | '/System/Library/CoreServices/Install in Progress.app/Contents/MacOS/Install in Progress', 164 | ) 165 | 166 | TccAuthValue = { 167 | 0: "Denied", 168 | 1: "Unknown", 169 | 2: "Allowed", 170 | 3: "Limited" 171 | } 172 | 173 | TccAuthReason = { 174 | 0: "None", 175 | 1: "Error", 176 | 2: "User Consent", 177 | 3: "User Set", 178 | 4: "System Set", 179 | 5: "Service Policy", 180 | 6: "MDM Policy", 181 | 7: "Override Policy", 182 | 8: "Missing Usage String", 183 | 9: "Prompt Timeout", 184 | 10: "Preflight Unknown", 185 | 11: "Entitled", 186 | 12: "App Type Policy" 187 | } 188 | 189 | 190 | class ProgExecEvent: 191 | def __init__(self, ts, app_name, app_path, other_info=''): 192 | self.ts = ts 193 | self.app_name = app_name 194 | self.app_path = app_path 195 | self.other_info = other_info 196 | 197 | 198 | class TccAuthreqEvent: 199 | def __init__(self, timeutc="", msg_id="", service="", attribution="", auth_value=0, auth_reason=0, auth_version=0) -> None: 200 | self.timeutc = timeutc 201 | self.msg_id = msg_id 202 | self.service = service 203 | self.attribution = attribution 204 | self.attribution_dict: dict[str, dict] = dict() 205 | self.auth_value = auth_value 206 | self.auth_reason = auth_reason 207 | self.auth_version = auth_version 208 | 209 | def get_auth_value(self) -> str: 210 | return TccAuthValue[self.auth_value] 211 | 212 | def get_auth_reason(self) -> str: 213 | return TccAuthReason[self.auth_reason] 214 | 215 | 216 | def extract_program_exec_spotlightshortcuts(basic_info: BasicInfo, timeline_events: list) -> bool: 217 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.MACAPT_DB): 218 | return False 219 | 220 | run_query = basic_info.mac_apt_dbs.run_query 221 | start_ts, end_ts = basic_info.get_between_dates_utc() 222 | sql = f'SELECT * FROM SpotlightShortcuts WHERE LastUsed BETWEEN "{start_ts}" AND "{end_ts}" ORDER BY LastUsed;' 223 | 224 | for row in run_query(MacAptDBType.MACAPT_DB, sql): 225 | ts = row['LastUsed'] 226 | user_typed = row['UserTyped'] 227 | display_name = row['DisplayName'] 228 | app_path = row['URL'] 229 | 230 | event = [ts, PLUGIN_ACTIVITY_TYPE, f"{display_name} ({app_path}) , Typed in: \"{user_typed}\"", PLUGIN_NAME] 231 | timeline_events.append(event) 232 | 233 | return True 234 | 235 | 236 | # Extract program execution logs with "LAUNCHING:0x" or "LAUNCH: 0x" 237 | # This function will work for macOS 10.15+ 238 | def extract_program_exec_logs_launch(basic_info: BasicInfo, timeline_events: list) -> bool: 239 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 240 | return False 241 | 242 | run_query = basic_info.mac_apt_dbs.run_query 243 | start_ts, end_ts = basic_info.get_between_dates_utc() 244 | sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 245 | (SenderName == "LaunchServices" AND (Message LIKE "LAUNCHING:0x%" OR Message LIKE "LAUNCH: 0x%")) \ 246 | ORDER BY TimeUtc;' 247 | sql_null = 'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{}" AND "{}" AND \ 248 | ProcessName = "lsd" AND Message LIKE "Non-fatal error enumerating %" \ 249 | ORDER BY TimeUtc DESC LIMIT 1;' 250 | 251 | # macOS 10.15.7 : ^LAUNCHING:0x.+ (.+) foreground=(\d) bringForward=(\d) .+ 252 | # macOS 11+ : ^LAUNCH: 0x.+ (.+) starting stopped process. 253 | # macOS 11+ (Info) : ^LAUNCH: 0x.+ (.+) launched with launchInStoppedState=true, and not starting the application. 254 | # macOS 11+ (Info) : LAUNCH: 0x0-0xa00a0 com.ridiculousfish.HexFiend launched with launchInQuarantine == true, so not starting the application. 255 | regex = r'^(LAUNCHING:|LAUNCH: )0x.+-0x.+ (.+) (foreground=\d bringForward=\d|starting stopped process|launched with )' 256 | 257 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql): 258 | result = re.match(regex, row['Message']) 259 | if result: 260 | if result.group(2) not in ignore_processes: 261 | app_name = result.group(2) 262 | parent_app = row['ProcessImagePath'] 263 | else: 264 | continue 265 | 266 | # If the application bundle ID is "(null)" 267 | if app_name == '(null)': 268 | regex_null = r'^Non-fatal error enumerating .+ file://(.+)/Contents/, .+' 269 | delta_ts = (datetime.datetime.strptime(row['TimeUtc'], '%Y-%m-%d %H:%M:%S.%f') - datetime.timedelta(microseconds=100000)).strftime('%Y-%m-%d %H:%M:%S.%f') 270 | for row_null in run_query(MacAptDBType.UNIFIED_LOGS, sql_null.format(delta_ts, row['TimeUtc'])): 271 | result_null = re.match(regex_null, row_null['Message']) 272 | if result_null: 273 | app_name = result_null.group(1) 274 | 275 | msg = f"{app_name} (Launched from {parent_app})" 276 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 277 | timeline_events.append(event) 278 | 279 | return True 280 | 281 | 282 | # Extract unsigned program execution logs with "temporarySigning" message (checked by Gatekeeper) 283 | # This function will work up to macOS 12 284 | # However, it is only for Intel Macs, and AppleSilicon Macs require at least an adhoc signature to the program. 285 | # If an unsigned program is run on macOS 13/14, it will not be logged at all in Unified Logs. 286 | def extract_program_exec_logs_tempsign(basic_info: BasicInfo, timeline_events: list) -> bool: 287 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 288 | return False 289 | 290 | run_query = basic_info.mac_apt_dbs.run_query 291 | start_ts, end_ts = basic_info.get_between_dates_utc() 292 | sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 293 | (Category == "gk" AND Message LIKE "temporarySigning %") \ 294 | ORDER BY TimeUtc;' 295 | regex = r'^temporarySigning .+ path=(.+)' 296 | 297 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql): 298 | result = re.match(regex, row['Message']) 299 | if result: 300 | if result.group(1) not in ignore_processes: 301 | msg = result.group(1) 302 | else: 303 | continue 304 | 305 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 306 | timeline_events.append(event) 307 | 308 | return True 309 | 310 | 311 | # Extract adhoc signed program execution logs 312 | # This function will work for macOS 10.15+ 313 | def extract_program_exec_logs_adhoc(basic_info: BasicInfo, timeline_events: list) -> bool: 314 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 315 | return False 316 | 317 | run_query = basic_info.mac_apt_dbs.run_query 318 | start_ts, end_ts = basic_info.get_between_dates_utc() 319 | # sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 320 | # (ProcessName = "kernel" AND Message LIKE "AMFI: % is %") OR (ProcessName = "amfid" and Message LIKE "% signature %") \ 321 | # ORDER BY TimeUtc;' 322 | sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 323 | (ProcessName = "kernel" AND Message LIKE "AMFI: % is %") OR (ProcessName = "amfid" and Message LIKE "% not valid: %") \ 324 | ORDER BY TimeUtc;' 325 | regex_kernel = r'^AMFI: \'(.+)\' is (.+)' 326 | # regex_amfid = r'^(/.+) (signature .+): .+' 327 | regex_amfid = r'^(/.+) not valid: .+' 328 | prog_exec_events: list[ProgExecEvent] = [] 329 | 330 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql): 331 | row_msg = row['Message'].strip() 332 | log.debug(f"REGEX: {regex_kernel} , ROW: {row_msg}") 333 | result = re.match(regex_kernel, row_msg) 334 | if result: 335 | ts = row['TimeUtc'] 336 | app_name = result.group(1) 337 | app_path = app_name 338 | if app_path.startswith("/System/Volumes/Preboot/Cryptexes/"): 339 | continue 340 | other_info = result.group(2) 341 | prog_exec_events.append(ProgExecEvent(ts, app_name, app_path, other_info)) 342 | continue 343 | 344 | log.debug(f"REGEX: {regex_amfid} , ROW: {row_msg}") 345 | result = re.match(regex_amfid, row_msg) 346 | if result: 347 | ts = row['TimeUtc'] 348 | app_name = result.group(1) 349 | app_path = app_name 350 | if app_path.startswith("/System/Volumes/Preboot/Cryptexes/"): 351 | continue 352 | # other_info = result.group(2) 353 | other_info = "The file does not have a valid signature." 354 | found_pair = False 355 | for event in prog_exec_events: 356 | if event.app_path == app_path and get_timedelta(event.ts, ts) <= 0.1: 357 | # event.other_info += ' ' + other_info + '.' 358 | event.other_info += ' ' + other_info 359 | found_pair = True 360 | break 361 | 362 | if not found_pair: 363 | prog_exec_events.append(ProgExecEvent(ts, app_name, app_path, other_info)) 364 | 365 | for event in prog_exec_events: 366 | msg = f"{event.app_path} ({event.other_info})" 367 | event = [event.ts, PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 368 | timeline_events.append(event) 369 | 370 | return True 371 | 372 | 373 | # Extract program execution logs with "Resolved pid" 374 | # This function will work correctly for macOS 10.15 only 375 | def extract_program_exec_logs_resolved_pid(basic_info: BasicInfo, timeline_events: list) -> bool: 376 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 377 | return False 378 | 379 | run_query = basic_info.mac_apt_dbs.run_query 380 | start_ts, end_ts = basic_info.get_between_dates_utc() 381 | sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 382 | (Category == "process" AND Message LIKE "Resolved pid %" AND Message LIKE "%[executable<%") \ 383 | ORDER BY TimeUtc;' 384 | regex_executable = r'^Resolved pid (\d+) to \[executable<(.+)\(\d+\)>:\d+\]' 385 | 386 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql): 387 | result = re.match(regex_executable, row['Message']) 388 | if result and result.group(2) not in ignore_processes: 389 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, f"{result.group(2)}, PID={result.group(1)}", PLUGIN_NAME] 390 | timeline_events.append(event) 391 | 392 | return True 393 | 394 | 395 | # Extract security policy would not allow logs 396 | # This function will work for macOS 10.15+ 397 | def extract_program_exec_logs_sec_pol_not_allow(basic_info: BasicInfo, timeline_events: list) -> bool: 398 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 399 | return False 400 | 401 | run_query = basic_info.mac_apt_dbs.run_query 402 | start_ts, end_ts = basic_info.get_between_dates_utc() 403 | sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 404 | ProcessName = "kernel" AND SenderName = "AppleSystemPolicy" AND \ 405 | Message LIKE "Security policy would not allow process:%" \ 406 | ORDER BY TimeUtc;' 407 | regex_sec_pol_not_allow = r'.*Security policy would not allow process: \d+, (.+)' 408 | 409 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql): 410 | result = re.match(regex_sec_pol_not_allow, row['Message']) 411 | if result: 412 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, f"{result.group(1)} would not allow to execute", PLUGIN_NAME] 413 | timeline_events.append(event) 414 | 415 | return True 416 | 417 | 418 | # Extract sudo logs 419 | # This function will work for macOS 10.15+ 420 | def extract_program_exec_logs_sudo(basic_info: BasicInfo, timeline_events: list) -> bool: 421 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 422 | return False 423 | 424 | run_query = basic_info.mac_apt_dbs.run_query 425 | start_ts, end_ts = basic_info.get_between_dates_utc() 426 | sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 427 | (ProcessName == "sudo" AND Message LIKE "%COMMAND=%") \ 428 | ORDER BY TimeUtc;' 429 | regex_sudo_succeeded = r'^(?P.+) : TTY=(?P.+) ; PWD=(?P.+) ; USER=(?P.+) ; COMMAND=(?P.+)' 430 | regex_sudo_failed = r'^(?P.+) : (?P\d+) incorrect password attempts ; TTY=(?P.+) ; PWD=(?P.+) ; USER=(?P.+) ; COMMAND=(?P.+)' 431 | 432 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql): 433 | if result := re.match(regex_sudo_succeeded, row['Message']): 434 | msg = f"{result['exec_user']} executed {result['command']} as {result['user']} on {result['pwd']} ({result['tty']})" 435 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 436 | timeline_events.append(event) 437 | 438 | elif result := re.match(regex_sudo_failed, row['Message']): 439 | msg = f"{result['exec_user']} failed to execute {result['command']} as {result['user']} on {result['pwd']} ({result['tty']})" 440 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 441 | timeline_events.append(event) 442 | 443 | return True 444 | 445 | 446 | # Extract tccd's AUTHREQ_* logs 447 | # This function is confirmed to work correctly for macOS 13+ 448 | def extract_program_exec_logs_tccd(basic_info: BasicInfo, timeline_events: list) -> bool: 449 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 450 | return False 451 | 452 | run_query = basic_info.mac_apt_dbs.run_query 453 | start_ts, end_ts = basic_info.get_between_dates_utc() 454 | sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 455 | (ProcessName == "tccd" AND \ 456 | (Message LIKE "AUTHREQ_CTX: %" OR \ 457 | Message LIKE "AUTHREQ_ATTRIBUTION: %" OR \ 458 | Message LIKE "AUTHREQ_RESULT: %" OR \ 459 | Message LIKE "AUTHREQ_PROMPTING: %"\ 460 | )\ 461 | ) \ 462 | ORDER BY TimeUtc;' 463 | regex_ctx = r'^AUTHREQ_CTX: msgID=(?P[\d\.]+), function=.+, service=(?P.+?), .+' 464 | regex_attrib = r'^AUTHREQ_ATTRIBUTION: msgID=(?P[\d\.]+), attribution={(?P.+)},' 465 | regex_result = r'^AUTHREQ_RESULT: msgID=(?P[\d\.]+), authValue=(?P\d+), authReason=(?P\d+), authVersion=(?P\d+), error=.+' 466 | 467 | tcc_authreq_events: dict[str, TccAuthreqEvent] = dict() 468 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql): 469 | if result := re.match(regex_ctx, row['Message']): 470 | if result['msg_id'] in tcc_authreq_events.keys(): 471 | tcc_authreq_events[result['msg_id']].msg_id = result['msg_id'] 472 | tcc_authreq_events[result['msg_id']].service = result['service'] 473 | else: 474 | tcc_authreq_events[result['msg_id']] = TccAuthreqEvent(timeutc=row['TimeUtc'], msg_id=result['msg_id'], service=result['service']) 475 | 476 | elif result := re.match(regex_attrib, row['Message']): 477 | if result['msg_id'] in tcc_authreq_events.keys(): 478 | tcc_authreq_events[result['msg_id']].attribution = result['attribution'] 479 | else: 480 | tcc_authreq_events[result['msg_id']] = TccAuthreqEvent(timeutc=row['TimeUtc'], msg_id=result['msg_id'], attribution=result['attribution']) 481 | 482 | elif result := re.match(regex_result, row['Message']): 483 | if result['msg_id'] in tcc_authreq_events.keys(): 484 | tcc_authreq_events[result['msg_id']].auth_value = int(result['auth_value']) 485 | tcc_authreq_events[result['msg_id']].auth_reason = int(result['auth_reason']) 486 | tcc_authreq_events[result['msg_id']].auth_version = int(result['auth_version']) 487 | else: 488 | tcc_authreq_events[result['msg_id']] = TccAuthreqEvent(timeutc=row['TimeUtc'], msg_id=result['msg_id'], 489 | auth_value=int(result['auth_value']), 490 | auth_reason=int(result['auth_reason']), 491 | auth_version=int(result['auth_version'])) 492 | 493 | ignore_events = list() 494 | for msg_id, event in tcc_authreq_events.items(): 495 | for attr in event.attribution.split("}, "): 496 | attr_items = attr.split("={") 497 | if len(attr_items) == 2: 498 | attr_name = attr_items[0] 499 | attr_items[1] = attr_items[1][len("TCCDProcess: "):] 500 | element_dict = dict() 501 | for elements in attr_items[1].split(", "): 502 | element_name, element_value = elements.split("=") 503 | element_dict[element_name] = element_value 504 | if attr_name in ("responsible", "accessing", "requesting") and element_name == "binary_path": 505 | if element_value in ignore_tccd_processes or \ 506 | element_value.startswith('/Library/Apple/System/Library/CoreServices/XProtect.app/Contents/MacOS/XProtectRemediator'): 507 | # if element_value in ignore_tccd_processes: 508 | ignore_events.append(msg_id) 509 | 510 | event.attribution_dict[attr_name] = element_dict 511 | 512 | for msg_id, tcc_event in tcc_authreq_events.items(): 513 | if msg_id not in ignore_events: 514 | msg = "TCC authreq: " 515 | if tcc_event.attribution_dict.get('accessing'): 516 | if msg == "TCC authreq: ": 517 | msg += f"service={tcc_event.service}, " 518 | msg += f"accessing={tcc_event.attribution_dict['accessing']['binary_path']} ({tcc_event.attribution_dict['accessing']['identifier']}) " 519 | msg += f"pid={tcc_event.attribution_dict['accessing']['pid']}, auid={tcc_event.attribution_dict['accessing']['auid']}, euid={tcc_event.attribution_dict['accessing']['euid']}, " 520 | 521 | if tcc_event.attribution_dict.get('responsible'): 522 | if msg == "TCC authreq: ": 523 | msg += f"service={tcc_event.service}, " 524 | msg += f"responsible={tcc_event.attribution_dict['responsible']['binary_path']} ({tcc_event.attribution_dict['responsible']['identifier']}) " 525 | msg += f"pid={tcc_event.attribution_dict['responsible']['pid']}, auid={tcc_event.attribution_dict['responsible']['auid']}, euid={tcc_event.attribution_dict['responsible']['euid']}, " 526 | 527 | if tcc_event.attribution_dict.get('requesting'): 528 | if msg == "TCC authreq: ": 529 | msg += f"service={tcc_event.service}, " 530 | msg += f"requesting={tcc_event.attribution_dict['requesting']['binary_path']} ({tcc_event.attribution_dict['requesting']['identifier']}) " 531 | msg += f"pid={tcc_event.attribution_dict['requesting']['pid']}, auid={tcc_event.attribution_dict['requesting']['auid']}, euid={tcc_event.attribution_dict['requesting']['euid']}, " 532 | 533 | if msg != "TCC authreq: ": 534 | msg += f"Result: authValue={tcc_event.get_auth_value()}({tcc_event.auth_value}), authReason={tcc_event.get_auth_reason()}({tcc_event.auth_reason}), authVersion={tcc_event.auth_version}" 535 | event = [tcc_event.timeutc, PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 536 | timeline_events.append(event) 537 | 538 | return True 539 | 540 | 541 | # Extract TCC violation logs 542 | # This function is confirmed to work correctly for macOS 13+ 543 | def extract_program_exec_logs_sandbox_violation(basic_info: BasicInfo, timeline_events: list) -> bool: 544 | if not basic_info.mac_apt_dbs.has_dbs(MacAptDBType.UNIFIED_LOGS): 545 | return False 546 | 547 | run_query = basic_info.mac_apt_dbs.run_query 548 | start_ts, end_ts = basic_info.get_between_dates_utc() 549 | sql = f'SELECT * FROM UnifiedLogs WHERE TimeUtc BETWEEN "{start_ts}" AND "{end_ts}" AND \ 550 | (ProcessName == "sandboxd" AND Subsystem == "com.apple.sandbox.reporting" AND Category == "violation") \ 551 | ORDER BY TimeUtc;' 552 | regex_metadata = r'^MetaData: (?P.+)' 553 | 554 | for row in run_query(MacAptDBType.UNIFIED_LOGS, sql): 555 | for msg_line in row['Message'].splitlines(): 556 | if result := re.match(regex_metadata, msg_line): 557 | data = json.loads(result['metadata']) 558 | msg = f"Sandbox violation: summary={data['summary']}, process={data['process-path']}, responsible-process={data['responsible-process-path']}" 559 | event = [row['TimeUtc'], PLUGIN_ACTIVITY_TYPE, msg, PLUGIN_NAME] 560 | timeline_events.append(event) 561 | break 562 | 563 | return True 564 | 565 | 566 | def run(basic_info: BasicInfo) -> bool: 567 | global log 568 | log = logging.getLogger(basic_info.output_params.logger_root + '.PLUGINS.' + PLUGIN_NAME) 569 | timeline_events = [] 570 | extract_program_exec_spotlightshortcuts(basic_info, timeline_events) 571 | extract_program_exec_logs_launch(basic_info, timeline_events) 572 | extract_program_exec_logs_tempsign(basic_info, timeline_events) 573 | extract_program_exec_logs_adhoc(basic_info, timeline_events) 574 | extract_program_exec_logs_resolved_pid(basic_info, timeline_events) 575 | extract_program_exec_logs_sec_pol_not_allow(basic_info, timeline_events) 576 | extract_program_exec_logs_sudo(basic_info, timeline_events) 577 | extract_program_exec_logs_tccd(basic_info, timeline_events) 578 | extract_program_exec_logs_sandbox_violation(basic_info, timeline_events) 579 | 580 | log.info(f"Detected {len(timeline_events)} events.") 581 | if len(timeline_events) > 0: 582 | basic_info.data_writer.write_data_rows(timeline_events) 583 | return True 584 | 585 | return False 586 | 587 | 588 | if __name__ == '__main__': 589 | print('This file is part of forensic timeline generator "ma2tl". So, it cannot run separately.') 590 | --------------------------------------------------------------------------------