├── FSEParser_V4.1.py ├── LICENSE ├── README.md └── report_queries.json /FSEParser_V4.1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # FSEvents Parser Python Script 4 | # ------------------------------------------------------ 5 | # Parse FSEvent records from allocated fsevent files and carved gzip files. 6 | # Outputs parsed information to a tab delimited txt file and SQLite database. 7 | # Errors and exceptions are recorded in the exceptions logfile. 8 | 9 | # Copyright 2024 10 | # Nicole Ibrahim 11 | # 12 | # Nicole Ibrahim licenses this file to you under the Apache License, Version 13 | # 2.0 (the "License"); you may not use this file except in compliance with the 14 | # License. You may obtain a copy of the License at: 15 | # 16 | # http://www.apache.org/licenses/LICENSE-2.0 17 | # 18 | # Unless required by applicable law or agreed to in writing, software 19 | # distributed under the License is distributed on an "AS IS" BASIS, 20 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 21 | # implied. See the License for the specific language governing 22 | # permissions and limitations under the License. 23 | 24 | import sys 25 | import os 26 | import struct 27 | import binascii 28 | import gzip 29 | import re 30 | import datetime 31 | import sqlite3 32 | import json 33 | import io 34 | from time import (gmtime, strftime) 35 | from optparse import OptionParser 36 | import contextlib 37 | 38 | try: 39 | from dfvfs.analyzer import analyzer 40 | from dfvfs.lib import definitions 41 | from dfvfs.path import factory as path_spec_factory 42 | from dfvfs.volume import tsk_volume_system 43 | from dfvfs.resolver import resolver 44 | from dfvfs.lib import raw 45 | from dfvfs.helpers import source_scanner 46 | DFVFS_IMPORT = True 47 | IMPORT_ERROR = None 48 | except ImportError as exp: 49 | DFVFS_IMPORT = False 50 | IMPORT_ERROR =("\n%s\n\ 51 | You have specified the source type as image but DFVFS \n\ 52 | is not installed and is required for image support. \n\ 53 | To install DFVFS please refer to \n\ 54 | http://www.hecfblog.com/2015/12/how-to-install-dfvfs-on-windows-without.html" % (exp)) 55 | 56 | VERSION = '4.1' 57 | 58 | EVENTMASK = { 59 | 0x00000000: 'None;', 60 | 0x00000001: 'FolderEvent;', 61 | 0x00000002: 'Mount;', 62 | 0x00000004: 'Unmount;', 63 | 0x00000020: 'EndOfTransaction;', 64 | 0x00000800: 'LastHardLinkRemoved;', 65 | 0x00001000: 'HardLink;', 66 | 0x00004000: 'SymbolicLink;', 67 | 0x00008000: 'FileEvent;', 68 | 0x00010000: 'PermissionChange;', 69 | 0x00020000: 'ExtendedAttrModified;', 70 | 0x00040000: 'ExtendedAttrRemoved;', 71 | 0x00100000: 'DocumentRevisioning;', 72 | 0x00400000: 'ItemCloned;', # macOS HighSierra 73 | 0x01000000: 'Created;', 74 | 0x02000000: 'Removed;', 75 | 0x04000000: 'InodeMetaMod;', 76 | 0x08000000: 'Renamed;', 77 | 0x10000000: 'Modified;', 78 | 0x20000000: 'Exchange;', 79 | 0x40000000: 'FinderInfoMod;', 80 | 0x80000000: 'FolderCreated;', 81 | 0x00000008: 'NOT_USED-0x00000008;', 82 | 0x00000010: 'NOT_USED-0x00000010;', 83 | 0x00000040: 'NOT_USED-0x00000040;', 84 | 0x00000080: 'NOT_USED-0x00000080;', 85 | 0x00000100: 'NOT_USED-0x00000100;', 86 | 0x00000200: 'NOT_USED-0x00000200;', 87 | 0x00000400: 'NOT_USED-0x00000400;', 88 | 0x00002000: 'NOT_USED-0x00002000;', 89 | 0x00080000: 'NOT_USED-0x00080000;', 90 | 0x00200000: 'NOT_USED-0x00200000;', 91 | 0x00800000: 'NOT_USED-0x00800000;' 92 | } 93 | 94 | print('\n==========================================================================') 95 | print(('FSEParser v {} -- provided by G-C Partners, LLC'.format(VERSION))) 96 | print('==========================================================================') 97 | 98 | 99 | def get_options(): 100 | """ 101 | Get needed options for processing 102 | """ 103 | usage = "usage: %prog -s SOURCE -o OUTDIR -t SOURCETYPE [folder|image] [-c CASENAME -q REPORT_QUERIES]" 104 | options = OptionParser(usage=usage) 105 | options.add_option("-s", 106 | action="store", 107 | type="string", 108 | dest="source", 109 | default=False, 110 | help="REQUIRED. The source directory or image containing fsevent files to be parsed") 111 | options.add_option("-o", 112 | action="store", 113 | type="string", 114 | dest="outdir", 115 | default=False, 116 | help="REQUIRED. The destination directory used to store parsed reports") 117 | options.add_option("-t", 118 | action="store", 119 | type="string", 120 | dest="sourcetype", 121 | default=False, 122 | help="REQUIRED. The source type to be parsed. Available options are 'folder' or 'image'") 123 | options.add_option("-c", 124 | action="store", 125 | type="string", 126 | dest="casename", 127 | default=False, 128 | help="OPTIONAL. The name of the current session, \ 129 | used for naming standards. Defaults to 'FSE_Reports'") 130 | options.add_option("-q", 131 | action="store", 132 | type="string", 133 | dest="report_queries", 134 | default=False, 135 | help="OPTIONAL. The location of the report_queries.json file \ 136 | containing custom report queries to generate targeted reports." 137 | ) 138 | 139 | # Return options to caller # 140 | return options 141 | 142 | 143 | def parse_options(): 144 | """ 145 | Capture and return command line arguments. 146 | """ 147 | # Get options 148 | options = get_options() 149 | (opts, args) = options.parse_args() 150 | 151 | # The meta will store all information about the arguments passed # 152 | meta = { 153 | 'casename': opts.casename, 154 | 'reportqueries': opts.report_queries, 155 | 'sourcetype': opts.sourcetype, 156 | 'source': opts.source, 157 | 'outdir': opts.outdir 158 | } 159 | 160 | # Print help if no options are provided 161 | if len(sys.argv[1:]) == 0: 162 | options.print_help() 163 | sys.exit(1) 164 | # Test required arguments 165 | if meta['source'] is False or meta['outdir'] is False or meta['sourcetype'] is False: 166 | options.error('Unable to proceed. The following parameters ' 167 | 'are required:\n-s SOURCE\n-o OUTDIR\n-t SOURCETYPE') 168 | 169 | if not os.path.exists(meta['source']): 170 | options.error("Unable to proceed. \n\n%s does not exist.\n" % meta['source']) 171 | 172 | if not os.path.exists(meta['outdir']): 173 | options.error("Unable to proceed. \n\n%s does not exist.\n" % meta['outdir']) 174 | 175 | if meta['reportqueries'] and not os.path.exists(meta['reportqueries']): 176 | options.error("Unable to proceed. \n\n%s does not exist.\n" % meta['reportqueries']) 177 | 178 | if meta['sourcetype'].lower() != 'folder' and meta['sourcetype'].lower() != 'image': 179 | options.error( 180 | 'Unable to proceed. \n\nIncorrect source type provided: "%s". The following are valid options:\ 181 | \n -t folder\n -t image\n' % (meta['sourcetype'])) 182 | 183 | if meta['sourcetype'] == 'image' and DFVFS_IMPORT is False: 184 | options.error(IMPORT_ERROR) 185 | 186 | if meta['reportqueries'] ==False: 187 | print('[Info]: Report queries file not specified using the -q option. Custom reports will not be generated.') 188 | 189 | if meta['casename'] is False: 190 | print('[Info]: No casename specified using -c. Defaulting to "FSE_Reports".') 191 | meta['casename'] = 'FSE_Reports' 192 | 193 | # Return meta to caller # 194 | return meta 195 | 196 | 197 | def main(): 198 | """ 199 | Call the main processes. 200 | """ 201 | # Process fsevents 202 | FSEventHandler() 203 | 204 | # Commit transaction 205 | SQL_CON.commit() 206 | 207 | # Close database connection 208 | SQL_CON.close() 209 | 210 | 211 | def enumerate_flags(flag, f_map): 212 | """ 213 | Iterate through record flag mappings and enumerate. 214 | """ 215 | # Reset string based flags to null 216 | f_type = '' 217 | f_flag = '' 218 | # Iterate through flags 219 | for i in f_map: 220 | if i & flag: 221 | if f_map[i] == 'FolderEvent;' or \ 222 | f_map[i] == 'FileEvent;' or \ 223 | f_map[i] == 'SymbolicLink;' or \ 224 | f_map[i] == 'HardLink;': 225 | f_type = ''.join([f_type, f_map[i]]) 226 | else: 227 | f_flag = ''.join([f_flag, f_map[i]]) 228 | return f_type, f_flag 229 | 230 | 231 | def progress(count, total): 232 | """ 233 | Handles the progress bar in the console. 234 | """ 235 | bar_len = 45 236 | filled_len = int(round(bar_len * count / float(total))) 237 | 238 | percents = round(100 * count / float(total), 1) 239 | p_bar = '=' * filled_len + '.' * (bar_len - filled_len) 240 | try: 241 | sys.stdout.write(' File {} of {} [{}] {}{}\r'.format(count, total, p_bar, percents, '%')) 242 | except: 243 | pass 244 | sys.stdout.flush() 245 | 246 | 247 | class FSEventHandler(): 248 | """ 249 | FSEventHandler iterates through and parses fsevents. 250 | """ 251 | 252 | def __init__(self): 253 | """ 254 | """ 255 | self.meta = parse_options() 256 | if self.meta['reportqueries']: 257 | # Check json file 258 | try: 259 | # Basic json syntax 260 | self.r_queries = json.load(open(self.meta['reportqueries'])) 261 | # Check to see if required keys are present 262 | for i in self.r_queries['process_list']: 263 | i['report_name'] 264 | i['query'] 265 | except Exception as exp: 266 | print(('An error occurred while reading the json file. \n{}'.format(str(exp)))) 267 | sys.exit(0) 268 | else: 269 | # if report queries option was not specified 270 | self.r_queries = False 271 | 272 | self.path = self.meta['source'] 273 | 274 | create_sqlite_db(self) 275 | 276 | self.files = [] 277 | self.pages = [] 278 | self.src_fullpath = '' 279 | self.dls_version = 0 280 | 281 | # Initialize statistic counters 282 | self.all_records_count = 0 283 | self.all_files_count = 0 284 | self.parsed_file_count = 0 285 | self.error_file_count = 0 286 | 287 | # Try to open the output files 288 | try: 289 | # Try to open ouput files 290 | self.l_all_fsevents = open( 291 | os.path.join(self.meta['outdir'], self.meta['casename'], 'All_FSEVENTS.tsv'), 292 | 'wb' 293 | ) 294 | # Process report queries output files 295 | # if option was specified. 296 | if self.r_queries: 297 | # Try to open custom report query output files 298 | for i in self.r_queries['process_list']: 299 | r_file = os.path.join(self.meta['outdir'], self.meta['casename'], i['report_name'] + '.tsv') 300 | if os.path.exists(r_file): 301 | os.remove(r_file) 302 | setattr(self, 'l_' + i['report_name'], open(r_file, 'wb')) 303 | 304 | # Output log file for exceptions 305 | l_file = os.path.join(self.meta['outdir'], self.meta['casename'], 'EXCEPTIONS_LOG.txt') 306 | self.logfile = open(l_file, 'w') 307 | except Exception as exp: 308 | # Print error to command prompt if unable to open files 309 | if 'Permission denied' in str(exp): 310 | print(('{}\nEnsure that you have permissions to write to file ' 311 | '\nand output file is not in use by another application.\n'.format(str(exp)))) 312 | else: 313 | print(exp) 314 | sys.exit(0) 315 | 316 | # Begin FSEvent processing 317 | 318 | print(('\n[STARTED] {} UTC Parsing files.'.format(strftime("%m/%d/%Y %H:%M:%S", gmtime())))) 319 | 320 | if self.meta['sourcetype'] == 'image': 321 | self._get_fsevent_image_files() 322 | elif self.meta['sourcetype'] == 'folder': 323 | self._get_fsevent_files() 324 | print(('\n All Files Attempted: {}\n All Parsed Files: {}\n Files ' 325 | 'with Errors: {}\n All Records Parsed: {}'.format( 326 | self.all_files_count, 327 | self.parsed_file_count, 328 | self.error_file_count, 329 | self.all_records_count))) 330 | 331 | print(('[FINISHED] {} UTC Parsing files.\n'.format(strftime("%m/%d/%Y %H:%M:%S", gmtime())))) 332 | 333 | print(('[STARTED] {} UTC Sorting fsevents table in Database.'.format(strftime("%m/%d/%Y %H:%M:%S", gmtime())))) 334 | 335 | row_count = reorder_sqlite_db(self) 336 | if row_count != 0: 337 | print(('[FINISHED] {} UTC Sorting fsevents table in Database.\n'.format(strftime("%m/%d/%Y %H:%M:%S", gmtime())))) 338 | 339 | print(('[STARTED] {} UTC Exporting fsevents table from Database.'.format( 340 | strftime("%m/%d/%Y %H:%M:%S", gmtime())))) 341 | 342 | self.export_fsevent_report(self.l_all_fsevents, row_count) 343 | 344 | print(('[FINISHED] {} UTC Exporting fsevents table from Database.\n'.format( 345 | strftime("%m/%d/%Y %H:%M:%S", gmtime())))) 346 | 347 | if self.r_queries: 348 | print(('[STARTED] {} UTC Exporting views from database ' 349 | 'to TSV files.'.format(strftime("%m/%d/%Y %H:%M:%S", gmtime())))) 350 | for i in self.r_queries['process_list']: 351 | Output.print_columns(getattr(self, 'l_' + i['report_name'])) 352 | # Export report views to output files 353 | self.export_sqlite_views() 354 | print(('[FINISHED] {} UTC Exporting views from database ' 355 | 'to TSV files.\n'.format(strftime("%m/%d/%Y %H:%M:%S", gmtime())))) 356 | 357 | print((" Exception log and Reports exported to:\n '{}'\n".format(os.path.join(self.meta['outdir'], self.meta['casename'])))) 358 | 359 | # Close output files 360 | self.l_all_fsevents.close() 361 | self.logfile.close() 362 | else: 363 | print(('[FINISHED] {} UTC No records were parsed.\n'.format(strftime("%m/%d/%Y %H:%M:%S", gmtime())))) 364 | print('Nothing to export.\n') 365 | 366 | 367 | @contextlib.contextmanager 368 | def skip_gzip_check(self): 369 | """ 370 | Context manager that replaces gzip.GzipFile._read_eof with a no-op. 371 | This is useful when decompressing partial files, something that won't 372 | work if GzipFile does it's checksum comparison. 373 | stackoverflow.com/questions/1732709/unzipping-part-of-a-gz-file-using-python/18602286 374 | """ 375 | _read_eof = gzip._GzipReader._read_eof 376 | gzip.GzipFile._read_eof = lambda *args, **kwargs: None 377 | yield 378 | gzip.GzipFile._read_eof = _read_eof 379 | 380 | 381 | def _get_fsevent_files(self): 382 | """ 383 | get_fsevent_files will iterate through each file in the fsevents dir provided, 384 | and attempt to decompress the gzip. If it is unable to decompress, 385 | it will write an entry in the logfile. If successful, the script will 386 | check for a DLS header signature in the decompress gzip. If found, the contents of 387 | the gzip will be placed into a buffer and passed to the next phase of processing. 388 | """ 389 | # Print the header columns to the output files 390 | Output.print_columns(self.l_all_fsevents) 391 | 392 | # Total number of files in events dir # 393 | t_files = len(os.listdir(self.path)) 394 | for filename in os.listdir(self.path): 395 | if filename == 'fseventsd-uuid': 396 | t_files -= 1 397 | self.time_range_src_mod = [] 398 | prev_mod_date = "Unknown" 399 | prev_last_wd = 0 400 | c_last_wd = 0 401 | 402 | # Uses file mod dates to generate time ranges by default unless 403 | # files are carved or mod dates lost due to exporting 404 | self.use_file_mod_dates = True 405 | 406 | # Run simple test to see if file mod dates 407 | # should be used to generate time ranges 408 | # In some instances fsevent files may not have 409 | # their original mod times preserved on export 410 | # This code will flag true when the same date and hour 411 | # exists for the first file and the last file 412 | # in the provided source fsevents folder 413 | first = os.path.join(self.path, os.listdir(self.path)[0]) 414 | last = os.path.join(self.path, os.listdir(self.path)[len(os.listdir(self.path)) - 1]) 415 | first = os.path.getmtime(first) 416 | last = os.path.getmtime(last) 417 | first = str(datetime.datetime.utcfromtimestamp(first))[:14] 418 | last = str(datetime.datetime.utcfromtimestamp(last))[:14] 419 | 420 | if first == last: 421 | self.use_file_mod_dates = False 422 | 423 | # Iterate through each file in supplied fsevents dir 424 | for filename in os.listdir(self.path): 425 | if filename == 'fseventsd-uuid': 426 | continue 427 | # Variables 428 | self.all_files_count += 1 429 | 430 | # Call the progress bar which shows parsing stats 431 | progress(self.all_files_count, t_files) 432 | 433 | buf = "" 434 | 435 | # Full path to source fsevent file 436 | self.src_fullpath = os.path.join(self.path, filename) 437 | # Name of source fsevent file 438 | self.src_filename = filename 439 | # UTC mod date of source fsevent file 440 | self.m_time = os.path.getmtime(self.src_fullpath) 441 | self.m_time = str(datetime.datetime.utcfromtimestamp((self.m_time))) + " [UTC]" 442 | 443 | # Regex to match against source fsevent log filename 444 | regexp = re.compile(r'^.*[\][0-9a-fA-F]{16}$') 445 | 446 | # Test to see if fsevent file name matches naming standard 447 | # if not, assume this is a carved gzip 448 | if len(self.src_filename) == 16 and regexp.search(filename) is not None: 449 | c_last_wd = int(self.src_filename, 16) 450 | self.time_range_src_mod = prev_last_wd, c_last_wd, prev_mod_date, self.m_time 451 | self.is_carved_gzip = False 452 | else: 453 | self.is_carved_gzip = True 454 | 455 | # Attempt to decompress the fsevent archive 456 | try: 457 | with self.skip_gzip_check(): 458 | self.files = gzip.GzipFile(self.src_fullpath, "rb") 459 | buf = self.files.read() 460 | 461 | except Exception as exp: 462 | # When permission denied is encountered 463 | if "Permission denied" in str(exp) and not os.path.isdir(self.src_fullpath): 464 | print(('\nEnsure that you have permissions to read ' 465 | 'from {}\n{}\n'.format(self.path, str(exp)))) 466 | sys.exit(0) 467 | # Otherwise write error to log file 468 | else: 469 | self.logfile.write( 470 | "%s\tError: Error while decompressing FSEvents file.%s\n" % ( 471 | self.src_filename, 472 | str(exp) 473 | ) 474 | ) 475 | self.error_file_count += 1 476 | continue 477 | 478 | # If decompress is success, check for DLS headers in the current file 479 | dls_chk = FSEventHandler.dls_header_search(self, buf, self.src_fullpath) 480 | 481 | # If check for DLS returns false, write information to logfile 482 | if dls_chk is False: 483 | self.logfile.write('%s\tInfo: DLS Header Check Failed. Unable to find a ' 484 | 'DLS header. Unable to parse File.\n' % (self.src_filename)) 485 | # Continue to the next file in the fsevents directory 486 | self.error_file_count += 1 487 | continue 488 | 489 | self.parsed_file_count += 1 490 | 491 | # Accounts for fsevent files that get flushed to disk 492 | # at the same time. Usually the result of a shutdown 493 | # or unmount 494 | if not self.is_carved_gzip and self.use_file_mod_dates: 495 | prev_mod_date = self.m_time 496 | prev_last_wd = int(self.src_filename, 16) 497 | 498 | # If DLSs were found, pass the decompressed file to be parsed 499 | FSEventHandler.parse(self, buf) 500 | 501 | 502 | def _get_fsevent_image_files(self): 503 | """ 504 | get_fsevent_files will iterate through each file in the fsevents dir 505 | and attempt to decompress the gzip. If it is unable to decompress, 506 | it will write an entry in the logfile. If successful, the script will 507 | check for a DLS header signature in the decompress gzip. If found, the contents of 508 | the gzip will be placed into a buffer and passed to the next phase of processing. 509 | """ 510 | # Print the header columns to the output file 511 | Output.print_columns(self.l_all_fsevents) 512 | 513 | scan_path_spec = None 514 | scanner = source_scanner.SourceScanner() 515 | scan_context = source_scanner.SourceScannerContext() 516 | scan_context.OpenSourcePath(self.meta['source']) 517 | 518 | scanner.Scan( 519 | scan_context, 520 | scan_path_spec=scan_path_spec 521 | ) 522 | 523 | for file_system_path_spec, file_system_scan_node in list(scan_context._file_system_scan_nodes.items()): 524 | t_files = 0 525 | self.all_files_count = 0 526 | self.error_file_count = 0 527 | self.all_records_count = 0 528 | self.parsed_file_count = 0 529 | 530 | try: 531 | location = file_system_path_spec.parent.location 532 | except: 533 | location = file_system_path_spec.location 534 | 535 | print(" Processing Volume {}.\n".format(location)) 536 | fsevent_locs = ["/.fseventsd","/System/Volumes/Data/.fseventsd"] 537 | 538 | for f_loc in fsevent_locs: 539 | fs_event_path_spec = path_spec_factory.Factory.NewPathSpec( 540 | file_system_path_spec.type_indicator, 541 | parent=file_system_path_spec.parent, 542 | location=f_loc 543 | ) 544 | 545 | file_entry = resolver.Resolver.OpenFileEntry( 546 | fs_event_path_spec 547 | ) 548 | 549 | if file_entry != None: 550 | 551 | t_files = file_entry.number_of_sub_file_entries 552 | for sub_file_entry in file_entry.sub_file_entries: 553 | if sub_file_entry.name == 'fseventsd-uuid': 554 | t_files -= 1 555 | 556 | self.time_range_src_mod = [] 557 | prev_mod_date = "Unknown" 558 | prev_last_wd = 0 559 | c_last_wd = 0 560 | counter = 0 561 | 562 | # Uses file mod dates to generate time ranges by default unless 563 | # files are carved or mod dates lost due to exporting 564 | self.use_file_mod_dates = True 565 | 566 | # Iterate through each file in supplied fsevents dir 567 | for sub_file_entry in file_entry.sub_file_entries: 568 | if sub_file_entry.name == 'fseventsd-uuid': 569 | continue 570 | # Variables 571 | counter += 1 572 | self.all_files_count += 1 573 | 574 | # Call the progress bar which shows parsing stats 575 | progress(counter, t_files) 576 | 577 | buf = "" 578 | 579 | # Name of source fsevent file 580 | self.src_filename = sub_file_entry.name 581 | self.src_fullpath = self.meta['source'] + ": " + location + sub_file_entry.path_spec.location 582 | 583 | stat_object = sub_file_entry.GetStat() 584 | 585 | # UTC mod date of source fsevent file 586 | self.m_time = datetime.datetime.fromtimestamp( 587 | stat_object.mtime).strftime( 588 | '%Y-%m-%d %H:%M:%S') + " [UTC]" 589 | 590 | # Regex to match against source fsevent log filename 591 | regexp = re.compile(r'^.*[\][0-9a-fA-F]{16}$') 592 | 593 | # Test to see if fsevent file name matches naming standard 594 | # if not, assume this is a carved gzip 595 | if len(self.src_filename) == 16 and regexp.search(self.src_filename) is not None: 596 | c_last_wd = int(self.src_filename, 16) 597 | self.time_range_src_mod = prev_last_wd, c_last_wd, prev_mod_date, self.m_time 598 | self.is_carved_gzip = False 599 | else: 600 | self.is_carved_gzip = True 601 | file_object = sub_file_entry.GetFileObject() 602 | 603 | compressedFile = io.StringIO() 604 | compressedFile.write(file_object.read()) 605 | compressedFile.seek(0) 606 | # Attempt to decompress the fsevent archive 607 | try: 608 | with self.skip_gzip_check(): 609 | self.files = gzip.GzipFile(fileobj=compressedFile, mode='rb') 610 | buf = self.files.read() 611 | 612 | except Exception as exp: 613 | self.logfile.write( 614 | "%s\tError: Error while decompressing FSEvents file.%s\n" % ( 615 | self.src_filename, 616 | str(exp) 617 | ) 618 | ) 619 | self.error_file_count += 1 620 | continue 621 | 622 | # If decompress is success, check for DLS headers in the current file 623 | dls_chk = FSEventHandler.dls_header_search(self, buf, self.src_filename) 624 | 625 | # If check for DLS returns false, write information to logfile 626 | if dls_chk is False: 627 | self.logfile.write('%s\tInfo: DLS Header Check Failed. Unable to find a ' 628 | 'DLS header. Unable to parse File.\n' % (self.src_filename)) 629 | # Continue to the next file in the fsevents directory 630 | self.error_file_count += 1 631 | continue 632 | 633 | self.parsed_file_count += 1 634 | 635 | # Accounts for fsevent files that get flushed to disk 636 | # at the same time. Usually the result of a shutdown 637 | # or unmount 638 | if not self.is_carved_gzip and self.use_file_mod_dates: 639 | prev_mod_date = self.m_time 640 | prev_last_wd = int(self.src_filename, 16) 641 | 642 | # If DLSs were found, pass the decompressed file to be parsed 643 | FSEventHandler.parse(self, buf) 644 | 645 | else: 646 | print('Unable to process volume or no fsevent files found') 647 | continue 648 | 649 | print(('\n\n All Files Attempted: {}\n All Parsed Files: {}\n Files ' 650 | 'with Errors: {}\n All Records Parsed: {}'.format( 651 | self.all_files_count, 652 | self.parsed_file_count, 653 | self.error_file_count, 654 | self.all_records_count))) 655 | 656 | 657 | def dls_header_search(self, buf, f_name): 658 | """ 659 | Search within the unzipped file 660 | for all occurrences of the DLS magic header. 661 | There can be more than one DLS header in an fsevents file. 662 | The start and end offsets are stored and used for parsing 663 | the records contained within each DLS page. 664 | """ 665 | self.file_size = len(buf) 666 | self.my_dls = [] 667 | 668 | raw_file = buf 669 | dls_count = 0 670 | start_offset = 0 671 | end_offset = 0 672 | 673 | while end_offset != self.file_size: 674 | try: 675 | start_offset = end_offset 676 | page_len = struct.unpack(" start_offset and self.valid_record_check: 995 | # Grab the first char 996 | char = page_buf[start_offset:end_offset].hex() 997 | 998 | if char != '00': 999 | # Replace non-printable char with nothing 1000 | if str(char).lower() == '0d' or str(char).lower() == '0a': 1001 | self.logfile.write('%s\tInfo: Non-printable char %s in record fullpath at ' 1002 | 'page offset %d. Parser removed char for reporting ' 1003 | 'purposes.\n' % \ 1004 | (self.src_filename, char, page_start_off + start_offset)) 1005 | char = '' 1006 | # Append the current char to the full path for current record 1007 | fullpath = fullpath + char 1008 | # Increment the offsets by one 1009 | start_offset += 1 1010 | end_offset += 1 1011 | # Continue the while loop 1012 | continue 1013 | elif char == '00': 1014 | # When 00 is found, then it is the end of fullpath 1015 | # Increment the offsets by bin_len, this will be the start of next full path 1016 | start_offset += bin_len 1017 | end_offset += bin_len 1018 | 1019 | # Decode fullpath that was stored as hex 1020 | 1021 | fullpath = bytes.fromhex(fullpath) 1022 | fullpath = fullpath.replace(b'\t', b'') 1023 | fullpath = str(fullpath,'utf-8') 1024 | # Store the record length 1025 | record_len = len(fullpath) + bin_len 1026 | 1027 | # Account for records that do not have a fullpath 1028 | if record_len == bin_len: 1029 | # Assign NULL as the path 1030 | fullpath = "NULL" 1031 | 1032 | # Assign raw record offsets # 1033 | r_start = start_offset - rbin_len 1034 | r_end = start_offset 1035 | 1036 | # Strip raw record from page buffer # 1037 | raw_record = page_buf[r_start:r_end] 1038 | 1039 | # Strip mask from buffer and encode as hex # 1040 | mask_hex = "0x" + raw_record[8:12].hex() 1041 | 1042 | # Account for carved files when record end offset 1043 | # occurs after the length of the buffer 1044 | if r_end > len_buf: 1045 | continue 1046 | 1047 | fs_node_id = "" 1048 | fs_uid = "" 1049 | 1050 | # Set fs_node_id to empty for DLS version 1 1051 | # Prior to HighSierra 1052 | if self.dls_version == 1: 1053 | pass 1054 | # Assign file system node id if DLS version is 2 1055 | # Introduced with HighSierra 1056 | if self.dls_version == 2: 1057 | fs_node_id = struct.unpack(" i[0] and wd < i[1]: 1183 | # When the previous date is the same as current 1184 | if i[2] == i[3]: 1185 | return i[2] 1186 | # Otherwise return the date range 1187 | else: 1188 | return i[2] + " - " + i[3] 1189 | # When event id matches previous wd in list 1190 | # assign previous date 1191 | elif wd == i[0]: 1192 | return str(i[2]) 1193 | # When event id matches current wd in list 1194 | # assign current date 1195 | elif wd == i[1]: 1196 | return str(i[3]) 1197 | # When the event id is greater than the last in list 1198 | # assign return source mod date 1199 | elif count == t_range_count and wd >= i[1] and self.use_file_mod_dates: 1200 | return c_mod_date 1201 | else: 1202 | count = count + 1 1203 | continue 1204 | else: 1205 | return "Unknown" 1206 | 1207 | 1208 | def export_fsevent_report(self, outfile, row_count): 1209 | """ 1210 | Export rows from fsevents table in DB to tab delimited report. 1211 | """ 1212 | counter = 0 1213 | 1214 | query = 'SELECT \ 1215 | id_hex, \ 1216 | node_id, \ 1217 | fs_uid, \ 1218 | fullpath, \ 1219 | type, \ 1220 | flags, \ 1221 | approx_dates_plus_minus_one_day, \ 1222 | source, \ 1223 | source_modified_time \ 1224 | FROM fsevents_sorted_by_event_id' 1225 | 1226 | SQL_TRAN.execute(query) 1227 | 1228 | while row_count > counter: 1229 | row = SQL_TRAN.fetchone() 1230 | values = [] 1231 | for cell in row: 1232 | if type(cell) is str or type(cell) is str: 1233 | try: 1234 | values.append(cell) 1235 | except: 1236 | print(row_count) 1237 | print(type(cell)) 1238 | print(cell) 1239 | print(row) 1240 | values.append("ERROR_IN_VALUE") 1241 | else: 1242 | try: 1243 | values.append(str(cell)) 1244 | except: 1245 | print(row_count) 1246 | print(type(cell)) 1247 | print(cell) 1248 | print(row) 1249 | values.append("ERROR_IN_VALUE") 1250 | m_row = '\t'.join(values) 1251 | m_row = m_row + '\n' 1252 | outfile.write(m_row.encode("utf-8")) 1253 | counter = counter + 1 1254 | 1255 | 1256 | def export_sqlite_views(self): 1257 | """ 1258 | Exports sqlite views from database if -q is set. 1259 | """ 1260 | # Gather the names of report views in the db 1261 | SQL_TRAN.execute("SELECT name FROM sqlite_master WHERE type='view'") 1262 | view_names = SQL_TRAN.fetchall() 1263 | 1264 | # Export report views to tsv files 1265 | for i in view_names: 1266 | 1267 | query = "SELECT * FROM %s" % (i[0]) 1268 | SQL_TRAN.execute(query) 1269 | row = ' ' 1270 | # Get outfile to write to 1271 | outfile = getattr(self, "l_" + i[0]) 1272 | row = SQL_TRAN.fetchone() 1273 | if row is None: 1274 | print((" No records found in view {}. Nothing to export".format(i[0]))) 1275 | outfile.close() 1276 | os.remove(outfile.name) 1277 | else: 1278 | print((" Exporting view {} from database".format(i[0]))) 1279 | # For each row join using tab and output to file 1280 | while row is not None: 1281 | values = [] 1282 | try: 1283 | for cell in row: 1284 | if type(cell) is str or type(cell) is str: 1285 | values.append(cell) 1286 | else: 1287 | values.append(str(cell)) 1288 | except: 1289 | values.append("ERROR_IN_VALUE") 1290 | print("ERROR: ", row) 1291 | m_row = '\t'.join(values) 1292 | m_row = m_row + '\n' 1293 | outfile.write(m_row.encode("utf-8")) 1294 | row = SQL_TRAN.fetchone() 1295 | 1296 | 1297 | class FsEventFileHeader(): 1298 | """ 1299 | FSEvent file header structure. 1300 | Each page within the decompressed begins with DLS1 or DLS2 1301 | It is stored using a byte order of little-endian. 1302 | """ 1303 | 1304 | def __init__(self, buf, filename): 1305 | """ 1306 | """ 1307 | # Name and path of current source fsevent file 1308 | self.src_fullpath = filename 1309 | # Page header 'DLS1' or 'DLS2' 1310 | # Was written to disk using little-endian 1311 | # Byte stream contains either "1SLD" or "2SLD", reversing order 1312 | self.signature = buf[4] + buf[3] + buf[2] + buf[1] 1313 | # Unknown raw values in DLS header 1314 | # self.unknown_raw = buf[4:8] 1315 | # Unknown hex version 1316 | # self.unknown_hex = buf[4:8].encode("hex") 1317 | # Unknown integer version 1318 | # self.unknown_int = struct.unpack("Q",self.wd) 1338 | 1339 | self.wd_hex = wd_buf.hex() 1340 | # Enumerate mask flags, string version 1341 | self.mask = enumerate_flags( 1342 | struct.unpack(">I", buf[8:12])[0], 1343 | EVENTMASK 1344 | ) 1345 | 1346 | 1347 | class Output(dict): 1348 | """ 1349 | Output class handles outputting parsed 1350 | fsevent records to report files. 1351 | """ 1352 | COLUMNS = [ 1353 | 'id', 1354 | 'id_hex', 1355 | 'fullpath', 1356 | 'filename', 1357 | 'type', 1358 | 'flags', 1359 | 'approx_dates_plus_minus_one_day', 1360 | 'mask', 1361 | 'node_id', 1362 | 'fs_uid', 1363 | 'dls_version', 1364 | 'record_end_offset', 1365 | 'source', 1366 | 'source_modified_time' 1367 | ] 1368 | R_COLUMNS = [ 1369 | 'id', 1370 | 'node_id', 1371 | 'fs_uid', 1372 | 'fullpath', 1373 | 'type', 1374 | 'flags', 1375 | 'approx_dates_plus_minus_one_day', 1376 | 'source', 1377 | 'source_modified_time' 1378 | ] 1379 | 1380 | 1381 | def __init__(self, attribs): 1382 | """ 1383 | Update column values. 1384 | """ 1385 | self.update(attribs) 1386 | 1387 | 1388 | @staticmethod 1389 | def print_columns(outfile): 1390 | """ 1391 | Output column header to report files. 1392 | """ 1393 | values = [] 1394 | for key in Output.R_COLUMNS: 1395 | #values.append(str(key)) 1396 | values.append(key) 1397 | row = '\t'.join(values) 1398 | row = "{}\n".format(row) 1399 | row = row.encode("utf-8") 1400 | 1401 | outfile.write(row) 1402 | 1403 | 1404 | def append_row(self): 1405 | """ 1406 | Output parsed fsevents row to database. 1407 | """ 1408 | values = [] 1409 | vals_to_insert = '' 1410 | 1411 | for key in Output.COLUMNS: 1412 | values.append(str(self[key])) 1413 | 1414 | # Replace any Quotes in parsed record with double quotes 1415 | for i in values: 1416 | vals_to_insert += i.replace('"', '""') + '","' 1417 | 1418 | vals_to_insert = '"' + vals_to_insert[:-3] + '"' 1419 | insert_sqlite_db(vals_to_insert) 1420 | 1421 | 1422 | def create_sqlite_db(self): 1423 | """ 1424 | Creates our output database for parsed records 1425 | and connects to it. 1426 | """ 1427 | db_filename = os.path.join(self.meta['outdir'], self.meta['casename'], 'FSEvents.sqlite') 1428 | table_schema = "CREATE TABLE [fsevents](\ 1429 | [id] [INT] NULL, \ 1430 | [id_hex] [TEXT] NULL, \ 1431 | [fullpath] [TEXT] NULL, \ 1432 | [filename] [TEXT] NULL, \ 1433 | [type] [TEXT] NULL, \ 1434 | [flags] [TEXT] NULL, \ 1435 | [approx_dates_plus_minus_one_day] [TEXT] NULL, \ 1436 | [mask] [TEXT] NULL, \ 1437 | [node_id] [TEXT] NULL, \ 1438 | [fs_uid] [TEXT] NULL, \ 1439 | [dls_version] [TEXT] NULL, \ 1440 | [record_end_offset] [TEXT] NULL, \ 1441 | [source] [TEXT] NULL, \ 1442 | [source_modified_time] [TEXT] NULL)" 1443 | if not os.path.isdir(os.path.join(self.meta['outdir'], self.meta['casename'])): 1444 | os.makedirs(os.path.join(self.meta['outdir'], self.meta['casename'])) 1445 | 1446 | # If database already exists delete it 1447 | try: 1448 | if os.path.isfile(db_filename): 1449 | os.remove(db_filename) 1450 | # Create database file if it doesn't exist 1451 | db_is_new = not os.path.exists(db_filename) 1452 | except: 1453 | print(("\nThe following output file is currently in use by " 1454 | "another program.\n -{}\nPlease ensure that the file is closed." 1455 | " Then rerun the parser.".format(db_filename))) 1456 | sys.exit(0) 1457 | 1458 | # Setup global 1459 | global SQL_CON 1460 | 1461 | SQL_CON = sqlite3.connect(os.path.join("", db_filename)) 1462 | 1463 | if db_is_new: 1464 | # Create table if it's a new database 1465 | SQL_CON.execute(table_schema) 1466 | if self.r_queries: 1467 | # Run queries in report queries list 1468 | # to add report database views 1469 | for i in self.r_queries['process_list']: 1470 | # Try to execute the query 1471 | cols = 'id, \ 1472 | node_id, \ 1473 | fs_uid, \ 1474 | fullpath, \ 1475 | type, \ 1476 | flags, \ 1477 | approx_dates_plus_minus_one_day, \ 1478 | source, \ 1479 | source_modified_time' 1480 | 1481 | query = i['query'].split("*") 1482 | query = query[0] + cols + query[1] 1483 | 1484 | try: 1485 | SQL_CON.execute(query) 1486 | except Exception as exp: 1487 | print(("SQLite error when executing query in json file. {}".format(str(exp)))) 1488 | sys.exit(0) 1489 | 1490 | # Setup global 1491 | global SQL_TRAN 1492 | 1493 | # Setup transaction cursor and return it 1494 | SQL_TRAN = SQL_CON.cursor() 1495 | 1496 | 1497 | def insert_sqlite_db(vals_to_insert): 1498 | """ 1499 | Insert parsed fsevent record values into database. 1500 | """ 1501 | insert_statement = "\ 1502 | insert into fsevents (\ 1503 | [id], \ 1504 | [id_hex], \ 1505 | [fullpath], \ 1506 | [filename], \ 1507 | [type], \ 1508 | [flags], \ 1509 | [approx_dates_plus_minus_one_day], \ 1510 | [mask], \ 1511 | [node_id], \ 1512 | [fs_uid], \ 1513 | [dls_version], \ 1514 | [record_end_offset], \ 1515 | [source], \ 1516 | [source_modified_time]\ 1517 | ) values (" + vals_to_insert + ")" 1518 | 1519 | try: 1520 | SQL_TRAN.execute(insert_statement) 1521 | except Exception as exp: 1522 | print(("insert failed!: {}".format(exp))) 1523 | 1524 | 1525 | def reorder_sqlite_db(self): 1526 | """ 1527 | Order database table rows by id. 1528 | Returns 1529 | count: The number of rows in the table 1530 | """ 1531 | query = "CREATE TABLE [fsevents_sorted_by_event_id](\ 1532 | [id] [INT] NULL, \ 1533 | [id_hex] [TEXT] NULL, \ 1534 | [fullpath] [TEXT] NULL, \ 1535 | [filename] [TEXT] NULL, \ 1536 | [type] [TEXT] NULL, \ 1537 | [flags] [TEXT] NULL, \ 1538 | [approx_dates_plus_minus_one_day] [TEXT] NULL, \ 1539 | [mask] [TEXT] NULL, \ 1540 | [node_id] [TEXT] NULL, \ 1541 | [fs_uid] [TEXT] NULL, \ 1542 | [dls_version] [TEXT] NULL, \ 1543 | [record_end_offset] [TEXT] NULL, \ 1544 | [source] [TEXT] NULL, \ 1545 | [source_modified_time] [TEXT] NULL)" 1546 | 1547 | SQL_TRAN.execute(query) 1548 | 1549 | query = "INSERT INTO fsevents_sorted_by_event_id ( \ 1550 | id, \ 1551 | id_hex, \ 1552 | fullpath, \ 1553 | filename, \ 1554 | type, \ 1555 | flags, \ 1556 | approx_dates_plus_minus_one_day, \ 1557 | mask, \ 1558 | node_id, \ 1559 | fs_uid, \ 1560 | dls_version, \ 1561 | record_end_offset, \ 1562 | source, \ 1563 | source_modified_time) \ 1564 | SELECT id,\ 1565 | id_hex,\ 1566 | fullpath,\ 1567 | filename,\ 1568 | type,flags,\ 1569 | approx_dates_plus_minus_one_day,\ 1570 | mask,\ 1571 | node_id,\ 1572 | fs_uid, \ 1573 | dls_version, \ 1574 | record_end_offset,\ 1575 | source,\ 1576 | source_modified_time \ 1577 | FROM fsevents ORDER BY id ASC;" 1578 | 1579 | SQL_TRAN.execute(query) 1580 | 1581 | count = SQL_TRAN.lastrowid 1582 | 1583 | query = "DROP TABLE fsevents;" 1584 | 1585 | SQL_TRAN.execute(query) 1586 | 1587 | return count 1588 | 1589 | 1590 | if __name__ == '__main__': 1591 | """ 1592 | Init 1593 | """ 1594 | main() 1595 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Overview 2 | --------------------- 3 | 4 | FSEvents files are written to disk by macOS APIs and contain historical records of file system activity that occurred for a particular volume. 5 | They can be found on devices running macOS and devices that were plugged in to a device running macOS. They are GZIP format, so you can also try carving for GZIPs to find FSEvents files that may be unallocated. 6 | 7 | FSEventsParser can be used to parse FSEvents files from the '/.fseventsd/' on a live system or FSEvents files extracted from an image. 8 | 9 | Carved GZIP files from a macOS volume or a device that was plugged into a macOS system can also be parsed. 10 | 11 | The parser outputs parsed information to tab delimited txt files and an SQLite database. Errors and exceptions are recorded in the exceptions logfile. 12 | 13 | The report_queries.json file can be used to generate custom reports based off of SQLite queries. Use -q to specify the file's location when running the parser. 14 | You can download predefined SQLite queries from https://github.com/dlcowen/FSEventsParser/blob/master/report_queries.json. 15 | Create your own targeted reports by editing the 'report_queries.json' file or just get default targeted reports including: 16 | - UserProfileActivity 17 | - UsersPictureTypeFiles 18 | - UsersDocumentTypeFiles 19 | - DownloadsActivity 20 | - TrashActivity 21 | - BrowserActivity 22 | - MountActivity 23 | - EmailAttachments 24 | - CloudStorageDropBoxActivity 25 | - CloudStorageBoxActivity 26 | - DSStoreActivity 27 | - SavedApplicationState 28 | - RootShellActivity 29 | - GuestAccountActivity 30 | - SudoUsageActivity 31 | - BashActivity 32 | - FailedPasswordActivity 33 | - iCloudSyncronizationActivity 34 | - SharedFileLists 35 | 36 | Requires 37 | --------------------- 38 | When the source type is an image DFVFS is required to run the script. Refer to https://github.com/log2timeline/dfvfs/wiki/Building. 39 | Alternately, you can run the compiled version of FSEParser to avoid having to install any other dependancies. The latest compiled version can be downloaded here: 40 | 41 | https://github.com/dlcowen/FSEventsParser/releases 42 | 43 | Usage 44 | --------------------- 45 | ========================================================================== 46 | FSEParser v 4.0 -- provided by G-C Partners, LLC 47 | ========================================================================== 48 | Usage: FSEParser_V4 -s SOURCE -o OUTDIR -t SOURCETYPE [folder|image] [-c CASENAME -q REPORT_QUERIES] 49 | 50 | Options: 51 | -h, --help show this help message and exit 52 | -s SOURCE REQUIRED. The source directory or image containing 53 | fsevent files to be parsed 54 | -o OUTDIR REQUIRED. The destination directory used to store parsed 55 | reports 56 | -t SOURCETYPE REQUIRED. The source type to be parsed. Available options 57 | are 'folder' or 'image' 58 | -c CASENAME OPTIONAL. The name of the current session, 59 | used for naming standards. Defaults to 'FSE_Reports' 60 | -q REPORT_QUERIES OPTIONAL. The location of the report_queries.json file 61 | containing custom report queries to generate targeted 62 | reports. 63 | 64 | 65 | Examples 66 | --------------------- 67 | A live system. 68 | > sudo ./FSEParser_V4 -s /.fseventsd -t folder -o /some_folder -c test_case -q report_queries.json 69 | 70 | Exported fsevent files 71 | > FSEParser_V4.exe -s E:\My_Exports\.fseventsd -t folder -o E:\My_Out_Folder -q report_queries.json 72 | 73 | An image file. 74 | > FSEParser_V4.exe -s E:\001-My_Source_Image.E01 -t image -o E:\My_Out_Folder -c Test_Case 75 | 76 | An attached external device or mounted volume/image. 77 | > FSEParser_V4.exe -s G:\\.fseventsd -t folder -o E:\My_Out_Folder -q report_queries.json 78 | 79 | > sudo ./FSEParser_V4 -s /Volumes/USBDISK/.fseventsd -t folder -o /some_folder -c test_case -q report_queries.json 80 | 81 | Notes 82 | ---------------------- 83 | - Parsed records can be in excess of 1 million records. 84 | - The script does not recursively search subdirectories in the source_dir provided. All FSEvents files including carved gzip if any must be placed in the same directory. 85 | - Currently the script does not perform deduplication. Duplicate records may occur when carved gzips are also parsed. 86 | 87 | 88 | Ouput Column Reference 89 | ----------------------- 90 | 91 | event_id: The fsevent record ID in hex and decimal format. The record ID is assigned in chronological order. 92 | 93 | node_id: Introduced in HighSierra. The file system node ID (stored in the catalog file for HFS+) of the record fullpath at the time the event was recorded. This value is empty for MacOS versions prior to High Sierra. 94 | 95 | fullpath: The record fullpath. 96 | 97 | type: The file type of the record fullpath/the event type: 98 | - FileEvent 99 | - FolderEvent 100 | - HardLink 101 | - SymbolicLink 102 | 103 | flags: The changes that occurred to the record fullpath: 104 | - Created 105 | - Modified 106 | - Renamed 107 | - Removed 108 | - InodeMetaMod 109 | - ExtendedAttrModified 110 | - FolderCreated 111 | - PermissionChange 112 | - ExtendedAttrRemoved 113 | - FinderInfoMod 114 | - DocumentRevisioning 115 | - Exchange 116 | - ItemCloned 117 | - LastHardLinkRemoved 118 | - Mount 119 | - Unmount 120 | - EndOfTransaction 121 | 122 | approx_dates_plus_minus_one_day: Approximate dates (no times) that the event occurred. The date ranges were pulled using the name of Log files that have the Created flag within an FSEvents file. This value may or may not be off by one day due to timezone variances. 123 | 124 | source: The fullpath of the FSEvents file that the record was parsed from. 125 | 126 | source_modified_time: The FSEvents source file modified date in UTC. 127 | -------------------------------------------------------------------------------- /report_queries.json: -------------------------------------------------------------------------------- 1 | { 2 | "process_list": [ 3 | { 4 | "report_name":"UserProfileActivity", 5 | "description":"All file activity that occurs within the user folders Desktop, Documents, Downloads, Pictures, Videos, and Music.", 6 | "query":"CREATE VIEW UserProfileActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%Users/%/Documents/%' OR fullpath LIKE '%Users/%/Desktop/%' OR fullpath LIKE '%Users/%/Downloads/%' OR fullpath LIKE '%Users/%/Pictures/%' OR fullpath LIKE '%Users/%/Videos/%' OR fullpath LIKE '%Users/%/Music/%';" 7 | }, 8 | { 9 | "report_name":"UsersPictureTypeFiles", 10 | "description":"Activity within the folder '/Users/' that has a file extension of a known image type file.", 11 | "query":"CREATE VIEW UsersPictureTypeFiles AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%Users/%' AND (fullpath LIKE '%.tif' OR fullpath LIKE '%.tiff' OR fullpath LIKE '%.gif' OR fullpath LIKE '%.jpeg' OR fullpath LIKE '%.jpg' OR fullpath LIKE '%.kdc' OR fullpath LIKE '%.xbm' OR fullpath LIKE '%.jif' OR fullpath LIKE '%.jfif' OR fullpath LIKE '%.bmp' OR fullpath LIKE '%.pcd' OR fullpath LIKE '%.png');" 12 | }, 13 | { 14 | "report_name":"UsersDocumentTypeFiles", 15 | "description":"Events related to Microsoft Office documents, PDFs, .Pages, .keynote, and .numbers files.", 16 | "query":"CREATE VIEW UsersDocumentTypeFiles AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%Users/%' AND (fullpath LIKE '%.pages' OR fullpath LIKE '%.numbers' OR fullpath LIKE '%.keynote' OR fullpath LIKE '%.xls' OR fullpath LIKE '%.xlsx' OR fullpath LIKE '%.ppt' OR fullpath LIKE '%.pptx' OR fullpath LIKE '%.doc' OR fullpath LIKE '%.docx' OR fullpath LIKE '%.pdf');" 17 | }, 18 | { 19 | "report_name":"DownloadsActivity", 20 | "description":"File activity that takes place with the User’s Downloads activity.", 21 | "query":"CREATE VIEW DownloadsActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%Users/%/Downloads/%' AND fullpath NOT LIKE '%com.apple.nsurlsessiond/Download%';" 22 | }, 23 | { 24 | "report_name":"TrashActivity", 25 | "description":"File activity that occurs within the '/Users//.Trash' folder. For example, when the user sends files to the Trash or empties the Trash.", 26 | "query":"CREATE VIEW TrashActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%/.Trash%';" 27 | }, 28 | { 29 | "report_name":"BrowserActivity", 30 | "description":"File activity related to web browser usage such as Safari or Chrome. This can reveal the full URL that was visited in a web browser or the domain that was being visited.", 31 | "query":"CREATE VIEW BrowserActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE (fullpath LIKE '%Users/%/Library%' AND (fullpath LIKE '%www.%' OR fullpath LIKE '%http%')) OR fullpath LIKE '%Users/%/Library/%www.%' OR fullpath LIKE '%Users/%/Library/%http%' OR fullpath LIKE '%Users/%/Library/Caches/Metadata/Safari/History/%' OR fullpath LIKE '%Users/%/Library/Application Support/Google/Chrome/Default/Local Storage/%';" 32 | }, 33 | { 34 | "report_name":"MountActivity", 35 | "description":"This will include activity related to mounting and unmounting of DMGs, external devices, network shares, and other mounted volumes.", 36 | "query":"CREATE VIEW MountActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE (flags LIKE '%ount%' OR fullpath like 'Volumes/%' OR fullpath like '/Volumes/%') and fullpath NOT LIKE '/Volumes/Preboot/%' and fullpath NOT LIKE '%sparsebundle/%';" 37 | }, 38 | { 39 | "report_name":"EmailAttachments", 40 | "description":"File activity related to email attachments being cached on disk. This can be used to determine names of email attachments.", 41 | "query":"CREATE VIEW EmailAttachments AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%Users/%/Attachments/%' OR fullpath LIKE '%Users/%/Library/Containers/com.apple.mail/Data/Mail Downloads/%' OR fullpath LIKE '%mobile/Library/Mail/%/Attachments/%';" 42 | }, 43 | { 44 | "report_name":"CloudStorageDropBoxActivity", 45 | "description":"Cloud storage activity related to files in the Dropbox folder for the Dropbox app.", 46 | "query":"CREATE VIEW CloudStorageDropBoxActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%Users/%/Dropbox/%';" 47 | }, 48 | { 49 | "report_name":"CloudStorageBoxActivity", 50 | "description":"Cloud storage activity related to files in the Box folder for the Box.com app.", 51 | "query":"CREATE VIEW CloudStorageBoxActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%Users/%/Box Sync/%';" 52 | }, 53 | { 54 | "report_name":"DSStoreActivity", 55 | "description":"DS_Store activity related to .DS_Store files which indicates file/folder accesses.", 56 | "query":"CREATE VIEW DSStoreActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%.DS_Store';" 57 | }, 58 | { 59 | "report_name":"SavedApplicationState", 60 | "description":"Indication that a window was open and the application state was saved.", 61 | "query":"CREATE VIEW SavedApplicationState AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%/Saved Application State/%windows.plist';" 62 | }, 63 | { 64 | "report_name":"RootShellActivity", 65 | "description":"Activity related to .sh_history file. This has been observed when the commands ‘sudo su’ or ‘sudo -I’ have been successfully executed. When the shell is closed the .sh_history file is modified.", 66 | "query":"CREATE VIEW RootShellActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%.sh_history%';" 67 | }, 68 | { 69 | "report_name":"GuestAccountActivity", 70 | "description":"Events related to usage of the Guest account. When the Guest account is enabled, users can log in to the Guest account and perform activities as a limited user. Once the user logs out the user data is deleted. These events provide insight into what a user was doing while logged in to the Guest account.", 71 | "query":"CREATE VIEW GuestAccountActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%Users/Guest/%';" 72 | }, 73 | { 74 | "report_name":"SudoUsageActivity", 75 | "description":"Activity related to a user using the sudo command in the terminal app. The name of the file is the user account issuing the sudo command.", 76 | "query":"CREATE VIEW SudoUsageActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%private/var/db/sudo/%';" 77 | }, 78 | { 79 | "report_name":"BashActivity", 80 | "description":"Activity related to .bash_sessions history files. The creating and modification of files with a User’s .bash_sessions folder indicates that commands were being run in the Terminal app.", 81 | "query":"CREATE VIEW BashActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE 'Users/%/.bash%';" 82 | }, 83 | { 84 | "report_name":"FailedPasswordActivity", 85 | "description":"Activity related to failed password attempts. The filename includes the user name that was used. These events indicate failed password attempts. This can be the result of running the su or sudo commands in the terminal and entering an incorrect password, being prompted with a dialog window to enter user credentials and entering an incorrect password, or even remote connection attempts with incorrect passwords.", 86 | "query":"CREATE VIEW FailedPasswordActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%rivate/var/db/dslocal/nodes/Default/users/.tmp.%.plist';" 87 | }, 88 | { 89 | "report_name":"iCloudSyncronizationActivity", 90 | "description":"Activity related to files synced to iCloud. These events can reveal the names of files that have been synced to iCloud from other devices.", 91 | "query":"CREATE VIEW iCloudSyncronizationActivity AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%.iCloud' OR fullpath LIKE '%Mobile Documents_com_apple_CloudDocs%';" 92 | }, 93 | { 94 | "report_name":"SharedFileLists", 95 | "description":"Activity related to Shared File List files (.sfl and .sfl2).", 96 | "query":"CREATE VIEW SharedFileLists AS SELECT * FROM fsevents_sorted_by_event_id WHERE fullpath LIKE '%.sfl' OR fullpath LIKE '%.sfl2';" 97 | } 98 | ] 99 | } 100 | --------------------------------------------------------------------------------