├── examples
    ├── filetree.rec
    ├── dumps
    │   └── just_to_create_dumps_folder.txt
    ├── carving.rec
    └── tree_explorer.rec
├── CHANGELOG.md
├── util
    ├── time.py
    ├── func_parser.py
    ├── table.py
    ├── hexdump.py
    ├── filetree.py
    └── carving.py
├── mmls.py
├── vol.py
├── LICENSE
├── .gitignore
├── part
    └── refs
    │   ├── object.py
    │   ├── allocator.py
    │   ├── object_tree.py
    │   ├── tree_control.py
    │   ├── vol.py
    │   ├── entry_block.py
    │   └── attribute.py
├── media
    ├── mbr.py
    └── gpt.py
├── list_entryblock.py
├── README.md
└── igor.py


/examples/filetree.rec:
--------------------------------------------------------------------------------
1 | file examples/dumps/Windows8ReFSImages/Physical_Image_9.001
2 | filetree
3 | 


--------------------------------------------------------------------------------
/examples/dumps/just_to_create_dumps_folder.txt:
--------------------------------------------------------------------------------
1 | This file is here only to make sure the dumps folder appears in the git repository.
2 | 


--------------------------------------------------------------------------------
/examples/carving.rec:
--------------------------------------------------------------------------------
1 | file examples/dumps/Windows8ReFSImages/Physical_Image_9.001
2 | find_entryblocks -f -F
3 | list_folders
4 | list_filenames
5 | list_dataruns
6 | datastream carving-example-linux-3.2.9.tar 0x13100000 0x7368,3220 0x7360,4 0x1000,16296
7 | datastream carving-example-README 0x73 0x7050,4
8 | bye
9 | 


--------------------------------------------------------------------------------
/examples/tree_explorer.rec:
--------------------------------------------------------------------------------
 1 | file examples/dumps/Windows8ReFSImages/Physical_Image_9.001
 2 | find_entryblocks
 3 | tree_control
 4 | tree_control_extension 0x167
 5 | object_tree 0x57
 6 | entryblock 0x72
 7 | entryblock 0x700e
 8 | entryblock 0x7012
 9 | datastream tree_explorer-README.txt 0x30 0x701c,4
10 | bye
11 | 


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | ## 0.1.1 (2018-09-19)
 2 | 
 3 |  - Added changelog file
 4 | 
 5 | ### Fixed (2 changes)
 6 | 
 7 |  - Options of igor `filetree` command
 8 |  - Text references to `find_entryblocks` command in igor
 9 | 
10 | ## 0.1 (2018-09-16)
11 | 
12 |  - First release of the `pyrefs` library including the `igor` command line utility.
13 | 


--------------------------------------------------------------------------------
/util/time.py:
--------------------------------------------------------------------------------
 1 | from datetime import datetime,timedelta
 2 | 
 3 | def bytes2time(dt):
 4 |     us = dt / 10
 5 |     try:
 6 |         td = timedelta(microseconds=us)
 7 |         time = datetime(1601,1,1) + td
 8 |     except:
 9 |         return '<bad time format>'
10 |     if time.year < 2000 or time.year > datetime.today().year:
11 |         return '<wrong date?> {}'.format(time)
12 |     return time
13 | 


--------------------------------------------------------------------------------
/util/func_parser.py:
--------------------------------------------------------------------------------
 1 | import argparse
 2 | 
 3 | class FuncArgumentParserError(Exception): pass
 4 | 
 5 | class FuncArgumentParserHelp(Exception): pass
 6 | 
 7 | class FuncArgumentParser(argparse.ArgumentParser):
 8 |     def exit(self, status=0, message=None):
 9 |         if message:
10 |             print(message)
11 |         raise FuncArgumentParserHelp()
12 | 
13 |     def error(self, message):
14 |         raise FuncArgumentParserError(message)
15 | 


--------------------------------------------------------------------------------
/mmls.py:
--------------------------------------------------------------------------------
 1 | import argparse
 2 | import media.mbr as mbr
 3 | import media.gpt as gpt
 4 | 
 5 | def mmls():
 6 |     parser = argparse.ArgumentParser(description='Read MBR from provided dump.')
 7 |     parser.add_argument('dump', action='store',
 8 |             type=argparse.FileType(mode='rb'))
 9 |     args = parser.parse_args()
10 |     mbr_data = mbr.readMBR(args.dump)
11 |     # we make the supposition that if it is a GPT media then only one
12 |     # partition is declared in the MBR
13 |     if len(mbr_data) == 1 and mbr_data[0][1]['ptype'] == mbr.MBR_PARTTYPE_GPT:
14 |         gpt_data = gpt.readGPT(args.dump, mbr_data[0][1]['start'])
15 |         gpt.print_gpt(gpt_data)
16 |     else:
17 |         print(mbr_data)
18 | 
19 | if __name__ == '__main__':
20 |     mmls()
21 | 


--------------------------------------------------------------------------------
/vol.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | import argparse
 3 | import media.mbr as mbr
 4 | import media.gpt as gpt
 5 | import part.refs.vol as refs
 6 | 
 7 | parser = argparse.ArgumentParser(description='Read from provided dump.')
 8 | parser.add_argument('dump', action='store',
 9 |         type=argparse.FileType(mode='rb'))
10 | args = parser.parse_args()
11 | mbr_data = mbr.readMBR(args.dump)
12 | # we make the supposition that if it is a GPT media then only one
13 | # partition is declared in the MBR
14 | if not (len(mbr_data) == 1 and mbr_data[0][1]['ptype'] == mbr.MBR_PARTTYPE_GPT):
15 |     print(mbr_data)
16 |     sys.exit()
17 | 
18 | gpt_data = gpt.readGPT(args.dump, mbr_data[0][1]['start'])
19 | gpt.print_gpt(gpt_data)
20 | 
21 | for part in gpt_data['parts']:
22 |     if part['type'] == gpt.GUID_PART_TYPE_W_BASIC_DATA_PART:
23 |         if refs.is_refs_part(args.dump, part['first_lba']):
24 |             refs_fsstat = refs.fsstat(args.dump, part['first_lba'], part['last_lba'])
25 |             refs.dump_fsstat(refs_fsstat)
26 |         else:
27 |             print('no ReFS partition')
28 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2018 Dafti
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/util/table.py:
--------------------------------------------------------------------------------
 1 | def print_table(_meta, _entries):
 2 |     # first we transform the entries
 3 |     entries = []
 4 |     for _e in _entries:
 5 |         entry = {}
 6 |         for col in _meta:
 7 |             entry[col['key']] = _e[col['key']] if 'transform' not in col.keys() else  col['transform'](_e[col['key']])
 8 |         entries.append(entry)
 9 |     # transform the values and compute columns lengths
10 |     lengths = {m['key']:len(m['header']) for m in _meta}
11 |     values = []
12 |     for e in entries:
13 |         value = {}
14 |         for m in _meta:
15 |             form = m['format'] if 'format' in m.keys() else ''
16 |             val = ('{:' + form + '}').format(e[m['key']])
17 |             value[m['key']] = val
18 |             lengths[m['key']] = max(len(val), lengths[m['key']])
19 |         values.append(value)
20 |     lengths = {k:v+1 for k,v in lengths.items()}
21 |     # print the table header
22 |     for m in _meta:
23 |         align = m['align'] if 'align' in m.keys() else ''
24 |         print(('{:' + align + str(lengths[m['key']]) + '} ').format(
25 |             m['header']), end='')
26 |     print('')
27 |     # print the table contents
28 |     for v in values:
29 |         for m in _meta:
30 |             align = m['align'] if 'align' in m.keys() else ''
31 |             print(('{:' + align + str(lengths[m['key']]) + '} ').format(
32 |                 v[m['key']]), end='')
33 |         print('')
34 | 
35 | 


--------------------------------------------------------------------------------
/util/hexdump.py:
--------------------------------------------------------------------------------
 1 | def hexdump(data, address=0, width=16):
 2 |     max_address = address + len(data)
 3 |     address_len = len('{:#x}'.format(max_address)) + 1
 4 |     data_format = ' {:02x}'
 5 |     text_format = '    {}'
 6 |     header_format = '{:<' + str(address_len) + '} ' + (data_format * width) + text_format
 7 |     buf = [ x % 0x100 for x in range(width) ]
 8 |     print(header_format.format('', *buf, ''))
 9 |     backup = None
10 |     skip = -1
11 |     line_range = list(range(0, len(data), width))
12 |     last_range = line_range[-1]
13 |     dot_line = '.' * len(('{:#0' + str(address_len) + 'x}:').format(0))
14 |     for index in line_range:
15 |         buf = data[index:index + width]
16 |         dec = ''.join([ chr(x) if x >= 32 and x <= 126 else '.' for x in buf ])
17 |         white_format = '   ' * (width - len(buf))
18 |         line_format = '{:#0' + str(address_len) + 'x}:' + (data_format * len(buf)) + white_format + text_format
19 |         line = line_format.format(address + index, *buf, dec)
20 |         if skip == -1:
21 |             print(line)
22 |             skip = 0
23 |         elif index == last_range:
24 |             if skip > 0:
25 |                 print(backup)
26 |             print(line)
27 |         elif list(filter(lambda x: x != 0, buf)):
28 |             if skip:
29 |                 print(backup)
30 |             skip = 0
31 |             print(line)
32 |         else:
33 |             if skip == 0:
34 |                 skip = 1
35 |                 backup = line
36 |             elif skip == 1:
37 |                 backup_format = dot_line + (data_format * len(buf)) + white_format + text_format
38 |                 backup = backup_format.format(*buf, dec)
39 |             else:
40 |                 skip = 2
41 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | *.egg-info/
 24 | .installed.cfg
 25 | *.egg
 26 | MANIFEST
 27 | 
 28 | # PyInstaller
 29 | #  Usually these files are written by a python script from a template
 30 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 31 | *.manifest
 32 | *.spec
 33 | 
 34 | # Installer logs
 35 | pip-log.txt
 36 | pip-delete-this-directory.txt
 37 | 
 38 | # Unit test / coverage reports
 39 | htmlcov/
 40 | .tox/
 41 | .coverage
 42 | .coverage.*
 43 | .cache
 44 | nosetests.xml
 45 | coverage.xml
 46 | *.cover
 47 | .hypothesis/
 48 | .pytest_cache/
 49 | 
 50 | # Translations
 51 | *.mo
 52 | *.pot
 53 | 
 54 | # Django stuff:
 55 | *.log
 56 | local_settings.py
 57 | db.sqlite3
 58 | 
 59 | # Flask stuff:
 60 | instance/
 61 | .webassets-cache
 62 | 
 63 | # Scrapy stuff:
 64 | .scrapy
 65 | 
 66 | # Sphinx documentation
 67 | docs/_build/
 68 | 
 69 | # PyBuilder
 70 | target/
 71 | 
 72 | # Jupyter Notebook
 73 | .ipynb_checkpoints
 74 | 
 75 | # pyenv
 76 | .python-version
 77 | 
 78 | # celery beat schedule file
 79 | celerybeat-schedule
 80 | 
 81 | # SageMath parsed files
 82 | *.sage.py
 83 | 
 84 | # Environments
 85 | .env
 86 | .venv
 87 | env/
 88 | venv/
 89 | ENV/
 90 | env.bak/
 91 | venv.bak/
 92 | 
 93 | # Spyder project settings
 94 | .spyderproject
 95 | .spyproject
 96 | 
 97 | # Rope project settings
 98 | .ropeproject
 99 | 
100 | # mkdocs documentation
101 | /site
102 | 
103 | # mypy
104 | .mypy_cache/
105 | 
106 | # vim
107 | .*.swp
108 | 


--------------------------------------------------------------------------------
/part/refs/object.py:
--------------------------------------------------------------------------------
 1 | from struct import Struct
 2 | 
 3 | OBJ_HEADER_1_FORMAT = Struct('<Q40sL28sL') # entryblock and node descriptor
 4 | OBJ_HEADER_2_FORMAT = Struct('<LLLLLLQ') # node header
 5 | OBJ_HEADER_3_FORMAT = Struct('<L20sQQ8sQ') # record
 6 | 
 7 | OBJ_NODE_DESC_OFFSET = 0x30
 8 | 
 9 | def read_object(dump, offset):
10 |     dump.seek(offset, 0)
11 |     data = dump.read(OBJ_HEADER_1_FORMAT.size)
12 |     fields = OBJ_HEADER_1_FORMAT.unpack_from(data, 0)
13 |     obj = {'_absolute_offset': offset,
14 |            'eb_number': fields[0],
15 |            'node_desc_length': fields[2],
16 |            'num_records_in_node': fields[4]}
17 |     node_header_offset = offset + OBJ_NODE_DESC_OFFSET + obj['node_desc_length']
18 |     dump.seek(node_header_offset)
19 |     data = dump.read(OBJ_HEADER_2_FORMAT.size)
20 |     fields = OBJ_HEADER_2_FORMAT.unpack_from(data, 0)
21 |     obj['node_header_length'] = fields[0]
22 |     obj['offset_next_free_rec'] = fields[1]
23 |     obj['free_space'] = fields[2]
24 |     obj['node_header_unknown'] = fields[3]
25 |     obj['offset_first_ptr'] = fields[4]
26 |     obj['num_ptrs_in_node'] = fields[5]
27 |     obj['offset_end_node'] = fields[6]
28 |     record_offset = node_header_offset + ot['node_header_length']
29 |     obj['records_offset'] = record_offset
30 |     records = []
31 |     for rec_i in range(ot['num_records_in_node']):
32 |         dump.seek(record_offset)
33 |         data = dump.read(OT_HEADER_3_FORMAT.size)
34 |         fields = OT_HEADER_3_FORMAT.unpack_from(data, 0)
35 |         rec = {'record_length': fields[0],
36 |                'nodeid': fields[2],
37 |                'eb_num': fields[3],
38 |                'id': fields[5]}
39 |         records.append(rec)
40 |         record_offset = record_offset + rec['record_length']
41 |     ot['records'] = records
42 |     ot['_structure_size'] = record_offset - offset
43 |     return ot
44 | 
45 | 
46 | 


--------------------------------------------------------------------------------
/part/refs/allocator.py:
--------------------------------------------------------------------------------
 1 | from struct import Struct
 2 | 
 3 | AL_HEADER_FORMAT = Struct('<QQ8sQ16sL28sL20sLLLLLLQ')
 4 | 
 5 | def read_allocator(dump, offset):
 6 |     dump.seek(offset, 0)
 7 |     data = dump.read(AL_HEADER_FORMAT.size)
 8 |     fields = AL_HEADER_FORMAT.unpack_from(data, 0)
 9 |     al = {'absolute_offset': offset,
10 |           'eb_number': fields[0],
11 |           'counter': fields[1],
12 |           'node_id': fields[3],
13 |           'node_desc_length': fields[5],
14 |           'num_records_in_node': fields[7],
15 |           'node_header_length': fields[9],
16 |           'offset_next_free_rec': fields[10],
17 |           'free_space': fields[11],
18 |           'node_header_unknown': fields[12],
19 |           'offset_first_ptr': fields[13],
20 |           'num_ptrs_in_node': fields[14],
21 |           'offset_end_node': fields[15]}
22 |     al['_structure_size'] = AL_HEADER_FORMAT.size
23 |     return al
24 | 
25 | def dump_allocator(al):
26 |     print('Allocator {:#x} ({size},{size:#x}):'.format(al['absolute_offset'], size=al['_structure_size']))
27 |     print('- entryblock number: {:#x}'.format(al['eb_number']))
28 |     print('- counter: {}'.format(al['counter']))
29 |     print('- node id: {:#x}'.format(al['node_id']))
30 |     print('- node descriptor length: {val:#x} ({val})'.format(val=al['node_desc_length']))
31 |     print('- number of records in node: {}'.format(al['num_records_in_node']))
32 |     print('- offset to next free record: {val:#x} ({val})'.format(val=al['offset_next_free_rec']))
33 |     print('- free space in node: {val:#x} ({val})'.format(val=al['free_space']))
34 |     print('- <node header unknown>: {val:#x} ({val})'.format(val=al['node_header_unknown']))
35 |     print('- offset to first pointer: {val:#x} ({val})'.format(val=al['offset_first_ptr']))
36 |     print('- number of pointers in this node: {}'.format(al['num_ptrs_in_node']))
37 |     print('- offset to end of node: {val:#x} ({val})'.format(val=al['offset_end_node']))
38 | 


--------------------------------------------------------------------------------
/media/mbr.py:
--------------------------------------------------------------------------------
 1 | from struct import unpack, Struct
 2 | from media.gpt import readGPT
 3 | 
 4 | SECTOR_SIZE = 512
 5 | MBR_SIZE = SECTOR_SIZE
 6 | MBR_TERMINATOR = 0x55aa
 7 | MBR_TERMINATOR_OFFSET = 510
 8 | MBR_PART_TABLE_OFFSET = 0x1be
 9 | MBR_PART_TABLE_SIZE = 4
10 | MBR_PART_ENTRY_SIZE = 16
11 | MBR_PARTTYPE_NOPART = 0
12 | MBR_PARTTYPE_REFS_EXFAT_NTFS = 7
13 | MBR_PARTTYPE_REFS_EXFAT_GPT = 0xee
14 | MBR_PARTTYPE_GPT = 0xee
15 | 
16 | '''
17 | MBR Partition format
18 | 
19 | offset,size,name, interpretation
20 | 
21 | 0   ubyte     boot flag        0x80 means bootable, 0 means not bootable
22 | 1   3 bytes   CHS              obsolete
23 | 4   ubyte     partition type   0x7 means NTFS, Exfat or ReFS
24 | 5   3 bytes   CHS              obsolete
25 | 8   ulong     1st sector       address of first sector
26 | 12  ulong     size             size in sectors
27 | '''
28 | #https://docs.python.org/3/library/struct.html
29 | MBR_PART_FORMAT = Struct('<B3sB3sLL') #'<' means little endian
30 | MBR_TERMINATOR_FORMAT = Struct('>H')
31 | 
32 | def readMBR(stream):
33 |     data = stream.read(MBR_SIZE)
34 |     mbr = []
35 |     terminator, = MBR_TERMINATOR_FORMAT.unpack_from(data, MBR_TERMINATOR_OFFSET)
36 |     if terminator != MBR_TERMINATOR:
37 |         print('not an MBR, missing MBR terminator ({})'.format(MBR_TERMINATOR))
38 |         return None #we should raise an Exception. https://docs.python.org/fr/3.5/tutorial/errors.html
39 |     else:
40 |         for p in range(MBR_PART_TABLE_SIZE):
41 |             # by convention, _ is the variable name for ignored values in Python
42 |             mbr_entry = MBR_PART_FORMAT.unpack_from(data, MBR_PART_TABLE_OFFSET + (p * MBR_PART_ENTRY_SIZE))
43 |             bflag, _, ptype, _, start, size = mbr_entry
44 |             if ptype != 0:
45 |                 mbr.append( (p, { 'bflag':bflag, 'ptype':ptype, 'start':start, 'size':size } ) )
46 |             # else:
47 |             #     print('Ignored entry')
48 |     # print(mbr)    
49 |     return mbr
50 | 
51 | 
52 | 


--------------------------------------------------------------------------------
/part/refs/object_tree.py:
--------------------------------------------------------------------------------
 1 | from struct import Struct
 2 | 
 3 | OT_HEADER_1_FORMAT = Struct('<QQ8sQ16sL28sL') # entryblock and node descriptor
 4 | OT_HEADER_2_FORMAT = Struct('<LLLLLLQ') # node header
 5 | OT_HEADER_3_FORMAT = Struct('<L20sQQ8sQ') # record
 6 | 
 7 | OT_NODE_DESC_OFFSET = 0x30
 8 | 
 9 | def read_object_tree(dump, offset):
10 |     dump.seek(offset, 0)
11 |     data = dump.read(OT_HEADER_1_FORMAT.size)
12 |     fields = OT_HEADER_1_FORMAT.unpack_from(data, 0)
13 |     ot = {'_dump_offset': offset,
14 |           '_absolute_offset': offset,
15 |           'eb_number': fields[0],
16 |           'counter': fields[1],
17 |           'node_id': fields[3],
18 |           'node_desc_length': fields[5],
19 |           'num_records_in_node': fields[7]}
20 |     node_header_offset = offset + OT_NODE_DESC_OFFSET + ot['node_desc_length']
21 |     dump.seek(node_header_offset)
22 |     data = dump.read(OT_HEADER_2_FORMAT.size)
23 |     fields = OT_HEADER_2_FORMAT.unpack_from(data, 0)
24 |     ot['node_header_length'] = fields[0]
25 |     ot['offset_next_free_rec'] = fields[1]
26 |     ot['free_space'] = fields[2]
27 |     ot['node_header_unknown'] = fields[3]
28 |     ot['offset_first_ptr'] = fields[4]
29 |     ot['num_ptrs_in_node'] = fields[5]
30 |     ot['offset_end_node'] = fields[6]
31 |     record_offset = node_header_offset + ot['node_header_length']
32 |     ot['records_offset'] = record_offset - offset
33 |     records = []
34 |     for rec_i in range(ot['num_records_in_node']):
35 |         dump.seek(record_offset, 0)
36 |         data = dump.read(OT_HEADER_3_FORMAT.size)
37 |         fields = OT_HEADER_3_FORMAT.unpack_from(data, 0)
38 |         rec = {'record_length': fields[0],
39 |                'nodeid': fields[2],
40 |                'eb_num': fields[3],
41 |                'id': fields[5],
42 |                '_record_offset': record_offset - offset}
43 |         records.append(rec)
44 |         record_offset = record_offset + rec['record_length']
45 |     ot['records'] = records
46 |     ot['_structure_size'] = record_offset - offset
47 |     return ot
48 | 
49 | def dump_object_tree(ot):
50 |     print('Object tree: <{:#x}> ({size},{size:#x})'.format(ot['_absolute_offset'], size=ot['_structure_size']))
51 |     print('- entryblock number: {:#x}'.format(ot['eb_number']))
52 |     print('- counter: {}'.format(ot['counter']))
53 |     print('- node id: {:#x}'.format(ot['node_id']))
54 |     print('- node descriptor length: {val:#x} ({val})'.format(val=ot['node_desc_length']))
55 |     print('- number of records in node: {}'.format(ot['num_records_in_node']))
56 |     print('- offset to next free record: {val:#x} ({val})'.format(val=ot['offset_next_free_rec']))
57 |     print('- free space in node: {val:#x} ({val})'.format(val=ot['free_space']))
58 |     print('- <node header unknown>: {val:#x} ({val})'.format(val=ot['node_header_unknown']))
59 |     print('- offset to first pointer: {val:#x} ({val})'.format(val=ot['offset_first_ptr']))
60 |     print('- number of pointers in this node: {}'.format(ot['num_ptrs_in_node']))
61 |     print('- offset to end of node: {val:#x} ({val})'.format(val=ot['offset_end_node']))
62 |     if ot['records']:
63 |         print('- records: <{:#x}>'.format(ot['records_offset']))
64 |         for i, rec in enumerate(ot['records']):
65 |             print('  - record {}: <{:#x}>'. format(i, rec['_record_offset']))
66 |             print('    - record length: {val:#x} ({val})'.format(val=rec['record_length']))
67 |             print('    - node id: {:#x}'.format(rec['nodeid']))
68 |             print('    - entryblock number: {:#x}'.format(rec['eb_num']))
69 |             print('    - identifier(?): {:#x}'.format(rec['id']))
70 | 


--------------------------------------------------------------------------------
/part/refs/tree_control.py:
--------------------------------------------------------------------------------
 1 | from struct import Struct
 2 | 
 3 | TC_HEADER_FORMAT = Struct('<Q72sLLLL')
 4 | TC_EXTENT_POINTER_FORMAT = Struct('<Q')
 5 | 
 6 | def read_tree_control(dump, offset):
 7 |     dump.seek(offset, 0)
 8 |     data = dump.read(TC_HEADER_FORMAT.size)
 9 |     fields = TC_HEADER_FORMAT.unpack_from(data, 0)
10 |     tc = {'_absolute_offset': offset,
11 |           'eb_number': fields[0],
12 |           'offset_extents': fields[2],
13 |           'num_extents': fields[3],
14 |           'offset_record': fields[4],
15 |           'length_record': fields[5]}
16 |     e_pts = []
17 |     for i in range(tc['num_extents']):
18 |         e_offset = tc['offset_extents'] + (i * TC_EXTENT_POINTER_FORMAT.size)
19 |         dump.seek(offset + e_offset, 0)
20 |         data = dump.read(TC_EXTENT_POINTER_FORMAT.size)
21 |         e_pt, = TC_EXTENT_POINTER_FORMAT.unpack_from(data, 0)
22 |         e_pts.append(e_pt)
23 |     tc['extent_pointers'] = e_pts
24 |     tc['_structure_size'] = tc['offset_record'] + tc['length_record']
25 |     return tc
26 | 
27 | def dump_tree_control(tc):
28 |     print('Tree control: <{:#x}> ({size},{size:#x})'.format(tc['_absolute_offset'],
29 |                                                             size=tc['_structure_size']))
30 |     print('- entryblock number: {:#x}'.format(tc['eb_number']))
31 |     print('- offset of extents: {val:#x} ({val})'.format(val=tc['offset_extents']))
32 |     print('- number of extents: {}'.format(tc['num_extents']))
33 |     print('- offset of record: {val:#x} ({val})'.format(val=tc['offset_record']))
34 |     print('- length of record: {val:#x} ({val})'.format(val=tc['length_record']))
35 |     if tc['extent_pointers']:
36 |         print('- extent pointers: ', end='')
37 |         for pt in tc['extent_pointers']:
38 |             print('{:#x} '.format(pt), end='')
39 |         print('')
40 | 
41 | TC_EXT_HEADER_FORMAT = Struct('<QQ8sQ28sL24sL')
42 | TC_EXT_RECORD_PTR_FORMAT = Struct('<L')
43 | TC_EXT_RECORD_OFFSET = 0x98
44 | TC_EXT_RECORD_FORMAT = Struct('<Q')
45 | 
46 | def read_tree_control_ext(dump, offset):
47 |     dump.seek(offset, 0)
48 |     data = dump.read(TC_EXT_HEADER_FORMAT.size)
49 |     fields = TC_EXT_HEADER_FORMAT.unpack_from(data, 0)
50 |     tc_e = {'_absolute_offset': offset,
51 |             'eb_number': fields[0],
52 |             'counter': fields[1],
53 |             'node_id': fields[3],
54 |             'length_record': fields[5],
55 |             'num_records': fields[7]}
56 |     r_pts = []
57 |     for i in range(tc_e['num_records']):
58 |         r_offset = TC_EXT_HEADER_FORMAT.size + (i * TC_EXT_RECORD_PTR_FORMAT.size)
59 |         dump.seek(offset + r_offset, 0)
60 |         data = dump.read(TC_EXT_RECORD_PTR_FORMAT.size)
61 |         r_pt, = TC_EXT_RECORD_PTR_FORMAT.unpack_from(data, 0)
62 |         r_pts.append(r_pt)
63 |     tc_e['record_offsets'] = r_pts
64 |     recs = []
65 |     tc_e['_records_offset'] = tc_e['record_offsets'][0] if tc_e['record_offsets'] else 0
66 |     for _rec_offset in tc_e['record_offsets']:
67 |         rec_offset = offset + _rec_offset
68 |         dump.seek(rec_offset, 0)
69 |         data = dump.read(TC_EXT_RECORD_FORMAT.size)
70 |         fields = TC_EXT_RECORD_FORMAT.unpack_from(data, 0)
71 |         rec = {'_record_offset': rec_offset,
72 |                'eb_number': fields[0]}
73 |         recs.append(rec)
74 |     tc_e['records'] = recs
75 |     tc_e['_structure_size'] = TC_EXT_RECORD_OFFSET + (tc_e['num_records'] * tc_e['length_record'])
76 |     return tc_e
77 | 
78 | def dump_tree_control_ext(tc_e):
79 |     print('Tree control extension: <{:#x}> ({size},{size:#x})'.format(tc_e['_absolute_offset'],
80 |                                                                       size=tc_e['_structure_size']))
81 |     print('- entryblock number: {:#x}'.format(tc_e['eb_number']))
82 |     print('- counter: {}'.format(tc_e['counter']))
83 |     print('- node id: {:#x}'.format(tc_e['node_id']))
84 |     print('- length of record: {val:#x} ({val})'.format(val=tc_e['length_record']))
85 |     print('- num_records: {}'.format(tc_e['num_records']))
86 |     if tc_e['record_offsets']:
87 |         print('- record offsets: ', end='')
88 |         for pt in tc_e['record_offsets']:
89 |             print('{:#x} '.format(pt), end='')
90 |         print('')
91 |     if tc_e['records']:
92 |         print('- records: <{:#x}>'.format(tc_e['_records_offset']))
93 |         for i, rec in enumerate(tc_e['records']):
94 |             print('  - record {}: <{:#x}>'.format(i, rec['_record_offset']))
95 |             print('    - entryblock number: {:#x}'.format(rec['eb_number']))
96 | 


--------------------------------------------------------------------------------
/media/gpt.py:
--------------------------------------------------------------------------------
  1 | from struct import unpack, Struct
  2 | from binascii import hexlify
  3 | 
  4 | SECTOR_SIZE = 512
  5 | GPT_HEADER_OFFSET = 0
  6 | GPT_HEADER_SIZE = SECTOR_SIZE    
  7 | GPT_HEADER_SIGNATURE = b'EFI PART'
  8 | 
  9 | '''
 10 | 0 signature 
 11 | 8 version, L
 12 | 12 size, L
 13 | 16 CRC32, L
 14 | 20 L
 15 | 24 header LBA, Q
 16 | '''
 17 | GPT_HEADER_FORMAT = Struct('<8sLLLLQQQQ16sQLLL')
 18 | GPT_PART_FORMAT = Struct('<16s16sQQQ72s')
 19 | 
 20 | GUID_LE_FORMAT = Struct('<LHH')
 21 | GUID_BG_FORMAT = Struct('>HHL')
 22 | GUID_UNUSED_PART_STRING = '00000000-0000-0000-0000-000000000000'
 23 | GUID_PART_TYPE_W_BASIC_DATA_PART = 'ebd0a0a2-b9e5-4433-87c0-68b6b72699c7'
 24 | GUID_PART_TYPE_W_MS_RESV_PART = 'e3c9e316-0b5c-4db8-817d-f92df00215ae'
 25 | GUID_TRANSLATION = {
 26 |     GUID_PART_TYPE_W_MS_RESV_PART: 'Windows - Microsoft Reserved Partition',
 27 |     GUID_PART_TYPE_W_BASIC_DATA_PART: 'Windows - Basic data partition'
 28 |     }
 29 | 
 30 | def guid_string(guid):
 31 |     _guid_le = GUID_LE_FORMAT.unpack_from(guid, 0)
 32 |     _guid_bg = GUID_BG_FORMAT.unpack_from(guid, 8)
 33 |     res = '{:08x}-{:04x}-{:04x}-{:04x}-{:04x}{:08x}'.format(
 34 |             _guid_le[0], _guid_le[1], _guid_le[2],
 35 |             _guid_bg[0], _guid_bg[1], _guid_bg[2])
 36 |     return res
 37 | 
 38 | def _gpt_init(header):
 39 |     gpt = {'signature': header[0],
 40 |            'revision': header[1],
 41 |            'header_size': header[2],
 42 |            'header_crc': header[3],
 43 |            'cur_lba': header[5],
 44 |            'bkp_lba': header[6],
 45 |            'first_lba': header[7],
 46 |            'last_lba': header[8],
 47 |            'disk_guid': guid_string(header[9]),
 48 |            'part_lba': header[10],
 49 |            'npart': header[11],
 50 |            'part_size': header[12],
 51 |            'part_crc': header[13]
 52 |            }
 53 |     return gpt
 54 | 
 55 | def _gpt_add_partitions(gpt, part_list):
 56 |     gpt['parts'] = part_list
 57 | 
 58 | def _gpt_part_init(entry, index):
 59 |     part = {'index': index,
 60 |             'type': guid_string(entry[0]),
 61 |             'guid': guid_string(entry[1]),
 62 |             'first_lba': entry[2],
 63 |             'last_lba': entry[3],
 64 |             'attr': entry[4],
 65 |             'name': entry[5].decode('utf-16le')}
 66 |     return part
 67 | 
 68 | def _read_gpt_partitions(stream, part_lba, num_part, part_size):
 69 |     parts = []
 70 |     for pi in range(num_part):
 71 |         part_offset = (part_lba * SECTOR_SIZE) + (pi * part_size)
 72 |         stream.seek(part_offset, 0)
 73 |         part_block = stream.read(part_size)
 74 |         part = GPT_PART_FORMAT.unpack_from(part_block, 0)
 75 |         part = _gpt_part_init(part, pi)
 76 |         if part['type'] != GUID_UNUSED_PART_STRING:
 77 |             parts.append(part)
 78 |     return parts
 79 | 
 80 | def readGPT(stream, offset=1):
 81 |     stream.seek(offset * SECTOR_SIZE, 0)
 82 |     _header = stream.read(GPT_HEADER_SIZE)
 83 |     header = GPT_HEADER_FORMAT.unpack_from(_header, GPT_HEADER_OFFSET)
 84 |     gpt = _gpt_init(header)
 85 |     if gpt['signature'] != GPT_HEADER_SIGNATURE:
 86 |         print('no gpt header magic found')
 87 |         return {}
 88 |     parts = _read_gpt_partitions(stream, gpt['part_lba'],
 89 |                                  gpt['npart'], gpt['part_size'])
 90 |     _gpt_add_partitions(gpt, parts)
 91 |     return gpt
 92 | 
 93 | def print_gpt_part(part, prefix=''):
 94 |     print('{}Partition {}:'.format(prefix, part['index']))
 95 |     print('{}  Type (GUID type): {} ({})'.format(prefix,
 96 |                                                  GUID_TRANSLATION[part['type']], part['type']))
 97 |     print('{}  GUID: {}'.format(prefix, part['guid']))
 98 |     print('{}  First LBA: {}'.format(prefix, part['first_lba']))
 99 |     print('{}  Last LBA: {}'.format(prefix, part['last_lba']))
100 |     print('{}  Attributes: {}'.format(prefix, part['attr']))
101 |     print('{}  Name: {}'.format(prefix, part['name']))
102 | 
103 | def print_gpt(gpt):
104 |     print('GPT media')
105 |     print('Signature: {}'.format(gpt['signature'].decode('utf-8')))
106 |     print('Revision: {}'.format(gpt['revision']))
107 |     print('Header size: {}'.format(gpt['header_size']))
108 |     print('Header CRC: {}'.format(gpt['header_crc']))
109 |     print('Current LBA: {}'.format(gpt['cur_lba']))
110 |     print('Backup LBA: {}'.format(gpt['bkp_lba']))
111 |     print('First LBA: {}'.format(gpt['first_lba']))
112 |     print('Last LBA: {}'.format(gpt['last_lba']))
113 |     print('Disk GUID: {}'.format(gpt['disk_guid']))
114 |     print('Partitions LBA: {}'.format(gpt['part_lba']))
115 |     print('Number of partitions: {}'.format(gpt['npart']))
116 |     print('Partition size: {}'.format(gpt['part_size']))
117 |     print('Partition CRC: {}'.format(gpt['part_crc']))
118 |     print('Partition list:')
119 |     for part in gpt['parts']:
120 |         print_gpt_part(part, '  ')
121 | 


--------------------------------------------------------------------------------
/part/refs/vol.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | from struct import Struct
  3 | from util.hexdump import hexdump
  4 | 
  5 | SECTOR_SIZE = 512
  6 | REFS_VR_JUMPINSTRUCTION_OFFSET = 0x0
  7 | REFS_VR_JUMPINSTRUCTION_SIZE = 3
  8 | REFS_VR_FILESYSTEMNAME_OFFSET = 0x3
  9 | REFS_VR_FILESYSTEMNAME_SIZE = 8
 10 | REFS_VR_RESV_OFFSET = 0x7
 11 | REFS_VR_RESV_SIZE = 5
 12 | REFS_VR_STRUCT_ID_OFFSET = 0x10
 13 | REFS_VR_STRUCT_ID_SIZE = 4
 14 | REFS_VR_NUMBYTES_OFFSET = 0x14
 15 | REFS_VR_NUMBYTES_SIZE = 2
 16 | REFS_VR_CHECKSUM_OFFSET = 0x16
 17 | REFS_VR_CHECKSUM_SIZE = 2
 18 | REFS_VR_BACKUP_OFFSET = 0x18
 19 | REFS_VR_BACKUP_SIZE = 8
 20 | REFS_VR_BYTES_PER_SECTOR_OFFSET = 0x20
 21 | REFS_VR_BYTES_PER_SECTOR_SIZE = 4
 22 | REFS_VR_SECTORS_PER_CLUSTER_OFFSET = 0x24
 23 | REFS_VR_SECTORS_PER_CLUSTER_SIZE = 4
 24 | REFS_VR_VERSION_MAJOR_OFFSET = 0x28
 25 | REFS_VR_VERSION_MAJOR_SIZE = 1
 26 | REFS_VR_VERSION_MINOR_OFFSET = 0x29
 27 | REFS_VR_VERSION_MINOR_SIZE = 1
 28 | REFS_VR_UNKNOWN_OFFSET = 0x2a
 29 | REFS_VR_UNKNOWN_SIZE = 14
 30 | REFS_VR_SN_OFFSET = 0x38
 31 | REFS_VR_SN_SIZE = 8
 32 | REFS_VR_SIZE = 64
 33 | 
 34 | REFS_VR_JUMPINSTRUCTION_POS = 0
 35 | REFS_VR_FILESYSTEMNAME_POS = 1
 36 | REFS_VR_RESV_POS = 2
 37 | REFS_VR_STRUCT_ID_POS = 3
 38 | REFS_VR_NUMBYTES_POS = 4
 39 | REFS_VR_CHECKSUM_POS = 5
 40 | REFS_VR_BACKUP_POS = 6
 41 | REFS_VR_BYTES_PER_SECTOR_POS = 7
 42 | REFS_VR_SECTORS_PER_CLUSTER_POS = 8
 43 | REFS_VR_VERSION_MAJOR_POS = 9
 44 | REFS_VR_VERSION_MINOR_POS = 10
 45 | REFS_VR_UNKNOWN_POS = 11
 46 | REFS_VR_SN_POS = 12
 47 | 
 48 | REFS_VR_FILESYSTEMNAME_SIGNATURE = 'ReFS\x00\x00\x00\x00'
 49 | REFS_VR_STRUCT_ID_SIGNATURE = 'FSRS'
 50 | 
 51 | REFS_VR_FORMAT = Struct('<3s8s5s4sHHQLLBB14sQ')
 52 | _JUMPINSTRUCTION_FORMAT = Struct('BBB')
 53 | 
 54 | def is_refs_part(dump, lba):
 55 |     """Check the given LBA for the ReFS signature and
 56 |     filesystem strings to decide if it is an ReFS volume."""
 57 |     dump.seek(lba * SECTOR_SIZE, 0)
 58 |     header = dump.read(REFS_VR_SIZE)
 59 |     signature = header[REFS_VR_FILESYSTEMNAME_OFFSET:REFS_VR_FILESYSTEMNAME_OFFSET + REFS_VR_FILESYSTEMNAME_SIZE].decode('ascii')
 60 |     fs = header[0x10:0x14].decode('ascii')
 61 |     return (signature == REFS_VR_FILESYSTEMNAME_SIGNATURE and
 62 |             fs == REFS_VR_STRUCT_ID_SIGNATURE)
 63 | 
 64 | def _check_is_refs_part(dump, lba):
 65 |     if not is_refs_part(dump, lba):
 66 |         print("ERROR(part.refs.fsstat): no ReFS partition")
 67 |         sys.exit(1)
 68 | 
 69 | def _dec_volume_record(dump, lba):
 70 |     dump.seek(lba * SECTOR_SIZE, 0)
 71 |     vr_raw = dump.read(REFS_VR_SIZE)
 72 |     vr_entries = REFS_VR_FORMAT.unpack_from(vr_raw, 0)
 73 |     ji_entries = _JUMPINSTRUCTION_FORMAT.unpack_from(vr_entries[REFS_VR_JUMPINSTRUCTION_OFFSET], 0)
 74 |     ji = ji_entries[0] + (ji_entries[1] * 256) + (ji_entries[2] * 256 * 256)
 75 |     vr = {'absolute_lba': lba,
 76 |           'jump_instruction': ji,
 77 |           'filesystem_name': vr_entries[REFS_VR_FILESYSTEMNAME_POS],
 78 |           'reserved': vr_entries[REFS_VR_RESV_POS],
 79 |           'struct_id': vr_entries[REFS_VR_STRUCT_ID_POS],
 80 |           'numbytes': vr_entries[REFS_VR_NUMBYTES_POS],
 81 |           'checksum': vr_entries[REFS_VR_CHECKSUM_POS],
 82 |           'backup': vr_entries[REFS_VR_BACKUP_POS],
 83 |           'bytes_per_sector': vr_entries[REFS_VR_BYTES_PER_SECTOR_POS],
 84 |           'sectors_per_cluster': vr_entries[REFS_VR_SECTORS_PER_CLUSTER_POS],
 85 |           'version_major': vr_entries[REFS_VR_VERSION_MAJOR_POS],
 86 |           'version_minor': vr_entries[REFS_VR_VERSION_MINOR_POS],
 87 |           'unknown': vr_entries[REFS_VR_UNKNOWN_POS],
 88 |           'serial_number': vr_entries[REFS_VR_SN_POS]}
 89 |     return vr
 90 | 
 91 | def fsstat(dump, lba, last_lba):
 92 |     _check_is_refs_part(dump, lba)
 93 |     vr = _dec_volume_record(dump, lba)
 94 |     vr_backup = _dec_volume_record(dump, last_lba)
 95 |     fs = {'volume_record': vr,
 96 |           'volume_record_backup': vr_backup}
 97 |     return fs
 98 | 
 99 | def _dump_volume_record(vr):
100 |     print('- Jump instruction: {:#08x}'.format(vr['jump_instruction']))
101 |     print('- Filesystem name: {}'.format(vr['filesystem_name'].decode('utf-8')))
102 |     print('- Struct identifier: {}'.format(vr['struct_id'].decode('utf-8')))
103 |     print('- Size of volume record: {}'.format(vr['numbytes']))
104 |     print('- FSRS checksum: {:#06x}'.format(vr['checksum']))
105 |     print('- Volume record backup LBA offset: {}'.format(vr['backup']))
106 |     print('- Bytes per sector: {}'.format(vr['bytes_per_sector']))
107 |     print('- Sectors per cluster: {}'.format(vr['sectors_per_cluster']))
108 |     print('- Filesystem major version: {}'.format(vr['version_major']))
109 |     print('- Filesystem minor version: {}'.format(vr['version_minor']))
110 |     print('- Volume serial number: {}'.format(vr['serial_number']))
111 | 
112 | def _dump_fsstat_volume_record_main(fs):
113 |     vr = fs['volume_record']
114 |     print('Volume record (lba: {}):'.format(vr['absolute_lba']))
115 |     _dump_volume_record(vr)
116 | 
117 | def _dump_fsstat_volume_record_backup(fs):
118 |     vr = fs['volume_record_backup']
119 |     print('Volume record backup (lba: {}):'.format(vr['absolute_lba']))
120 |     _dump_volume_record(vr)
121 | 
122 | def dump_fsstat(fs):
123 |     _dump_fsstat_volume_record_main(fs)
124 |     _dump_fsstat_volume_record_backup(fs)
125 | 


--------------------------------------------------------------------------------
/util/filetree.py:
--------------------------------------------------------------------------------
  1 | import part.refs.attribute as rattr
  2 | import part.refs.entry_block as reb
  3 | import util.carving as carving
  4 | 
  5 | def filetree(dump, v_offset, v_end_offset, nodeid, block_list = None, block_size = 16 * 1024):
  6 |     if not block_list:
  7 |         block_list = carving.find_blocks(dump, v_offset, v_end_offset, block_size)
  8 |     tree = _filetree(dump, nodeid, block_list)
  9 |     print(tree)
 10 |     return tree
 11 | 
 12 | def _filetree(dump, nodeid, complete_block_list):
 13 |     node_block_list = [ reb.read_entryblock(dump, x['offset'])
 14 |                         for x in complete_block_list
 15 |                         if x['nodeid'] == nodeid ]
 16 |     if not node_block_list:
 17 |         return None
 18 |     node_ext_block_list = [ x for x in node_block_list if x['_contains_extents'] ]
 19 |     node_rec_block_list = [ x for x in node_block_list if x['_contains_records'] ]
 20 |     rec_block_list = []
 21 |     if node_ext_block_list:
 22 |         root_ext_block_list = []
 23 |         for block in node_ext_block_list:
 24 |             root = True
 25 |             for _block in node_ext_block_list:
 26 |                 if block['eb_number'] in [ ext['eb_number'] for ext in _block['extents'] ]:
 27 |                     root = False
 28 |                     break
 29 |             if root:
 30 |                 root_ext_block_list.append(block)
 31 |         max_counter = max([ x['counter'] for x in root_ext_block_list ])
 32 |         ext_block = [ x for x in root_ext_block_list if x['counter'] == max_counter ][0]
 33 |         rec_block_list.append(ext_block)
 34 |         while [ x for x in rec_block_list if x['_contains_extents'] ]:
 35 |             for i in range(len(rec_block_list)):
 36 |                 block = rec_block_list[i]
 37 |                 if block['_contains_extents']:
 38 |                     blocks = []
 39 |                     for blockid in [ b['eb_number'] for b in block['extents'] ]:
 40 |                         blocks.append([ b for b in node_block_list if b['eb_number'] == blockid ][0])
 41 |                     rec_block_list[i] = blocks
 42 |                 else:
 43 |                     rec_block_list[i] = [rec_block_list[i]]
 44 |                 _flatten = [ i for sub in rec_block_list for i in sub ]
 45 |                 rec_block_list = _flatten
 46 |     else:
 47 |         if not node_rec_block_list:
 48 |             print('ERROR: didn\'t found any entryblock with extensions or records in any of the entryblocks of node {:#x} ({} entryblocks).'.format(nodeid, len(node_block_list)))
 49 |             return None
 50 |         max_counter = max([ x['counter'] for x in node_rec_block_list ])
 51 |         block = [ x for x in node_rec_block_list if x['counter'] == max_counter ][0]
 52 |         rec_block_list.append(block)
 53 |     dm = _get_directory_metadata(rec_block_list)
 54 |     if not dm:
 55 |         return None
 56 |     rootname = _get_directory_metadata_name(dm)
 57 |     files = _get_files(rec_block_list)
 58 |     folders = _get_folders(rec_block_list, dump, complete_block_list)
 59 |     folder = {'name': rootname,
 60 |               'nodeid': rec_block_list[0]['node_id'],
 61 |               # 'entryblocks': block_list,
 62 |               'files': files,
 63 |               'folders': folders}
 64 |     return folder
 65 | 
 66 | def _get_directory_metadata(blocks):
 67 |     for block in blocks:
 68 |         if 'pointers_data' in block.keys():
 69 |             ptrs = [ ptr
 70 |                      for ptr in block['pointers_data']
 71 |                      if ptr['type'] == rattr.ATTR_TYPE_DIRECTORY_METADATA ]
 72 |             if ptrs:
 73 |                 return ptrs[0]
 74 |         else:
 75 |             reb.dump_entryblock(block)
 76 |     # should never reach this point
 77 |     print('ERROR: no directory metadata record found for node {}.'.format(blocks[0]['node_id']))
 78 |     return None
 79 | 
 80 | def _get_directory_metadata_name(attr):
 81 |     return [ ptr['name']
 82 |              for ptr in attr['pointers_data']
 83 |              if ptr['type'] == rattr.DM_SUBATTR_TYPE_FOLDER ][0]
 84 | 
 85 | def _get_files(blocks):
 86 |     files = []
 87 |     for block in blocks:
 88 |         bfiles = [ {'name': ptr['filename'],
 89 |                     'blockid': block['eb_number']}
 90 |                    for ptr in block['pointers_data']
 91 |                    if ptr['type'] == rattr.ATTR_TYPE_FILENAME ]
 92 |         files.extend(bfiles)
 93 |     return files
 94 | 
 95 | def _get_folders(blocks, dump, complete_block_list):
 96 |     folders = []
 97 |     for block in blocks:
 98 |         bfolders = [ {'name': ptr['foldername'],
 99 |                       'blockid': block['eb_number'],
100 |                       'tree': _filetree(dump, ptr['nodeid'], complete_block_list)}
101 |                      for ptr in block['pointers_data']
102 |                      if ptr['type'] == rattr.ATTR_TYPE_FILENAME_FOLDER ]
103 |         folders.extend(bfolders)
104 |     return folders
105 | 
106 | def dump_filetree(tree, level=0, name='.'):
107 |     if not tree:
108 |         return
109 |     print('D {} {} ({}) {:#x}'.format(level * '.',
110 |         name,
111 |         tree['name'].decode('utf-16le') if tree['name'] else '<unknown>',
112 |         tree['nodeid']))
113 |     for f in tree['files']:
114 |         print('F {} {} {:#x}'.format((level + 1) * '.',
115 |             f['name'].decode('utf-16le') if f['name'] else '<unknown>',
116 |             f['blockid']))
117 |     for f in tree['folders']:
118 |         dump_filetree(f['tree'], level + 1, f['name'].decode('utf-16le'))
119 | 


--------------------------------------------------------------------------------
/util/carving.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import sys
  3 | from struct import Struct
  4 | import part.refs.entry_block as reb
  5 | import part.refs.tree_control as rtc
  6 | import part.refs.attribute as refs_attr
  7 | import part.refs.allocator as alloc
  8 | import part.refs.object_tree as rot
  9 | from util.hexdump import hexdump
 10 | from util.table import print_table
 11 | 
 12 | SECTOR_SIZE = 512
 13 | ENTRYBLOCK_SIZE = 16 * 1024
 14 | CLUSTER_SIZE = 4 * ENTRYBLOCK_SIZE
 15 | counter_offset=0x8
 16 | nodeid_offset=0x18
 17 | childid_offset=0x20
 18 | ENTRYBLOCK_FORMAT=Struct('<Q')
 19 | NODEID_FORMAT=Struct('<Q')
 20 | 
 21 | def find_bytes(entry, block):
 22 |     hits = []
 23 |     for i in range(0, len(block) - (len(entry) - 1)):
 24 |         match = True
 25 |         for l in range(0, len(entry)):
 26 |             match = entry[l] == block[i + l]
 27 |             if not match:
 28 |                 break
 29 |         if match:
 30 |             hits.append(i)
 31 |     return hits
 32 | 
 33 | def find_blocks(dump, lba_offset, lba_end, step = ENTRYBLOCK_SIZE):
 34 |     offset = lba_offset * SECTOR_SIZE
 35 |     end = lba_end * SECTOR_SIZE
 36 |     blocks = []
 37 |     for i in range(offset,end,step):
 38 |         dump.seek(i, 0)
 39 |         data = dump.read(8)
 40 |         entryblock, = ENTRYBLOCK_FORMAT.unpack_from(data, 0)
 41 |         if reb.is_entryblock(dump, i, offset):
 42 |             dump.seek(i+nodeid_offset,0)
 43 |             data = dump.read(8)
 44 |             nodeid, = NODEID_FORMAT.unpack_from(data, 0)
 45 |             dump.seek(i+childid_offset,0)
 46 |             data = dump.read(8)
 47 |             childid, = NODEID_FORMAT.unpack_from(data, 0)
 48 |             dump.seek(i+counter_offset, 0)
 49 |             counter = dump.read(1)[0]
 50 |             steps = int((i - offset) / step)
 51 |             block = {'offset': i, 'entryblock': entryblock,
 52 |                      'counter': counter, 'nodeid': nodeid,
 53 |                      'childid': childid, 'fnas': None,
 54 |                      'folderids': None}
 55 |             blocks.append(block)
 56 |     return blocks
 57 | 
 58 | def find_data_blocks_with_pattern(dump, pattern, lba_offset, lba_end, step=ENTRYBLOCK_SIZE):
 59 |     offset = lba_offset * SECTOR_SIZE
 60 |     end = lba_end * SECTOR_SIZE
 61 |     block_offsets = []
 62 |     for i in range(offset,end,step):
 63 |         dump.seek(i, 0)
 64 |         data = dump.read(128)
 65 |         addrs = find_bytes(pattern, data)
 66 |         if addrs:
 67 |             block_offsets.append({'block_offset': int((i - lba_offset)/step),
 68 |                                   'addr': i,
 69 |                                   'lba_offset': int((i - lba_offset)/SECTOR_SIZE)})
 70 |     return block_offsets
 71 | 
 72 | def print_blocks(blocks):
 73 |     columns = [ {'key': 'offset', 'header': 'Offset', 'align': '<', 'format': '#x'},
 74 |                 {'key': 'entryblock', 'header': 'Entryblock', 'align': '<', 'format': '#x'},
 75 |                 {'key': 'nodeid', 'header': 'NodeId', 'align': '<', 'format': '#x'},
 76 |                 {'key': 'childid', 'header': 'ChildId', 'align': '<', 'format': '#x'},
 77 |                 {'key': 'counter', 'header': 'Counter', 'align': '>'},
 78 |                 {'key': 'fnas', 'header': 'FNAs', 'align': '>',
 79 |                  'transform': lambda x: 'N/A' if x == None else len(x)},
 80 |                 {'key': 'folderids', 'header': 'Folders', 'align': '>',
 81 |                  'transform': lambda x: 'N/A' if x == None else len(x)} ]
 82 |     if blocks:
 83 |         print_table(columns, blocks)
 84 | 
 85 | def blocks_with_filename_attributes(dump, blocks, block_size = ENTRYBLOCK_SIZE):
 86 |     blocks_found = []
 87 |     for block in blocks:
 88 |         dump.seek(block['offset'])
 89 |         data = dump.read(block_size)
 90 |         fileids = find_bytes([0x30,0,1,0], data)
 91 |         if fileids:
 92 |             block['fnas'] = [ x + block['offset'] - 0x10 for x in fileids ]
 93 |             blocks_found.append(block)
 94 |         else:
 95 |             block['fnas'] = []
 96 |     return blocks_found
 97 | 
 98 | def blocks_with_folder_attributes(dump, blocks, block_size = ENTRYBLOCK_SIZE):
 99 |     blocks_found = []
100 |     for block in blocks:
101 |         dump.seek(block['offset'], 0)
102 |         data = dump.read(block_size)
103 |         _folderids = find_bytes([0x30,0,2,0], data)
104 |         folderids = []
105 |         # this loop tries to check that the pattern found is in a filename_folder
106 |         # attribute
107 |         # TODO: find a better way to determine if the pattern is in a filename_folder
108 |         # attribute or in a folder attribute
109 |         for fid in _folderids:
110 |             dump.seek(fid + block['offset'] - 0x10 + 0x4, 0)
111 |             data = dump.read(2)
112 |             offset = data[0] + (data[1] * 256)
113 |             if offset == 0x10:
114 |                 folderids.append(fid)
115 |         if folderids:
116 |             block['folderids'] = [ x + block['offset'] - 0x10 for x in folderids ]
117 |             blocks_found.append(block)
118 |         else:
119 |             block['folderids'] = []
120 |     return blocks_found
121 | 
122 | def blocks_with_child_attributes(dump, blocks, block_size = ENTRYBLOCK_SIZE):
123 |     blocks_found = []
124 |     for block in blocks:
125 |         dump.seek(block['offset'])
126 |         data = dump.read(block_size)
127 |         ids = find_bytes([0x20,0,0,0x80], data)
128 |         if ids:
129 |             block['childids'] = [ x + block['offset'] - 0x10 for x in ids ]
130 |             blocks_found.append(block)
131 |     return blocks_found
132 | 


--------------------------------------------------------------------------------
/list_entryblock.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import sys
  3 | from struct import Struct
  4 | import part.refs.entry_block as reb
  5 | import part.refs.tree_control as rtc
  6 | import part.refs.attribute as refs_attr
  7 | import part.refs.allocator as alloc
  8 | import part.refs.object_tree as rot
  9 | from util.hexdump import hexdump
 10 | 
 11 | SECTOR_SIZE = 512
 12 | ENTRYBLOCK_SIZE = 16 * 1024
 13 | 
 14 | def blocks_with_file_attributes(dump, blocks, block_size):
 15 |     blocks_found = []
 16 |     for block in blocks:
 17 |         dump.seek(block['offset'])
 18 |         data = dump.read(block_size)
 19 |         fileids = find_bytes([0x30,0,1,0], data)
 20 |         folderids = find_bytes([0x30,0,2,0], data)
 21 |         if fileids or folderids:
 22 |             block['fileids'] = [ x + block['offset'] - 0x10 for x in fileids ]
 23 |             block['folderids'] = [ x + block['offset'] - 0x10 for x in folderids ]
 24 |             blocks_found.append(block)
 25 |     return blocks_found
 26 | 
 27 | def blocks_with_child_attributes(dump, blocks, block_size):
 28 |     blocks_found = []
 29 |     for block in blocks:
 30 |         dump.seek(block['offset'])
 31 |         data = dump.read(block_size)
 32 |         ids = find_bytes([0x20,0,0,0x80], data)
 33 |         if ids:
 34 |             block['childids'] = [ x + block['offset'] - 0x10 for x in ids ]
 35 |             blocks_found.append(block)
 36 |     return blocks_found
 37 | 
 38 | def find_bytes(entry, block):
 39 |     hits = []
 40 |     for i in range(0, len(block) - (len(entry) - 1)):
 41 |         match = True
 42 |         for l in range(0, len(entry)):
 43 |             match = entry[l] == block[i + l]
 44 |             if not match:
 45 |                 break
 46 |         if match:
 47 |             hits.append(i)
 48 |     return hits
 49 | 
 50 | def find_blocks(dump, offset, end, step):
 51 |     blocks = []
 52 |     for i in range(offset,end,step):
 53 |         dump.seek(i, 0)
 54 |         data = dump.read(4)
 55 |         entryblock, = ENTRYBLOCK_FORMAT.unpack_from(data, 0)
 56 |         if (i - offset) / step == entryblock:
 57 |         # if entryblock != 0:
 58 |         # if reb.is_entry_block(dump, i, offset):
 59 |             dump.seek(i+nodeid_offset,0)
 60 |             data = dump.read(4)
 61 |             nodeid, = NODEID_FORMAT.unpack_from(data, 0)
 62 |             dump.seek(i+childid_offset,0)
 63 |             data = dump.read(4)
 64 |             childid, = NODEID_FORMAT.unpack_from(data, 0)
 65 |             dump.seek(i+counter_offset, 0)
 66 |             counter = dump.read(1)[0]
 67 |             steps = int((i - offset) / step)
 68 |             block = {'offset': i, 'entryblock': entryblock,
 69 |                      'counter': counter, 'nodeid': nodeid,
 70 |                      'childid': childid}
 71 |             blocks.append(block)
 72 |     return blocks
 73 | 
 74 | parser = argparse.ArgumentParser(description='ReFS carving on provided dump.')
 75 | parser.add_argument('dump', action='store',
 76 |         type=argparse.FileType(mode='rb'))
 77 | args = parser.parse_args()
 78 | 
 79 | # ref
 80 | offset = 0x8100000
 81 | end = (41940991*512)+512 # complete partition
 82 | # phys 9
 83 | # offset = 65664 * 512
 84 | # end = 1114239 * 512
 85 | # end = offset + (20000 * 1024 * 16)
 86 | # end = 0x16d34000
 87 | step= 16 * 1024
 88 | counter_offset=0x8
 89 | nodeid_offset=0x18
 90 | childid_offset=0x20
 91 | ENTRYBLOCK_FORMAT=Struct('<L')
 92 | NODEID_FORMAT=Struct('<L')
 93 | 
 94 | blocks = find_blocks(args.dump, offset, end, step)
 95 | print("==================================================================")
 96 | print("Blocks found ({})".format(len(blocks)))
 97 | print("==================================================================")
 98 | for b in blocks:
 99 |     print('{:#010x} - {:#010x}: {:#010x} {:>3} {:#010x} {:#010x}'.format(b['offset'],
100 |         b['offset'] + step, b['entryblock'], b['counter'], b['nodeid'], b['childid']))
101 | 
102 | blocks_with_files = blocks_with_file_attributes(args.dump, blocks, step)
103 | print("==================================================================")
104 | print("Blocks found with files found")
105 | print("==================================================================")
106 | for b in blocks_with_files:
107 |     print('{:#010x} - {:#010x}: {:#06x} {:>3} {:#06x} {:#06x} {:4} {:4}'.format(b['offset'],
108 |         b['offset'] + step, b['entryblock'], b['counter'], b['nodeid'], b['childid'],
109 |         len(b['fileids']), len(b['folderids'])))
110 | #
111 | print("==================================================================")
112 | print("Files found")
113 | print("==================================================================")
114 | for block in blocks_with_files:
115 |     for fid in block['fileids']:
116 |         attr = refs_attr.read_attribute(args.dump, fid)
117 |         try:
118 |             filename = attr['filename'].decode('utf-16le')
119 |         except:
120 |             filename = attr['filename']
121 |         print('{:#010x} {:#06x} {:#06x} {:#06x} {:#010x} {}'.format(
122 |             block['offset'], block['entryblock'],
123 |             block['nodeid'], block['childid'], attr['absolute_offset'], filename))
124 | #
125 | print("==================================================================")
126 | print("Folders found")
127 | print("==================================================================")
128 | for block in blocks_with_files:
129 |     for fid in block['folderids']:
130 |         attr = refs_attr.read_attribute(args.dump, fid)
131 |         try:
132 |             foldername = attr['foldername'].decode('utf-16le')
133 |         except:
134 |             foldername = attr['foldername']
135 |         print('{:#010x} {:#06x} {:#06x} {:#06x} {:#010x} {}'.format(
136 |             block['offset'], block['entryblock'],
137 |             block['nodeid'], block['childid'], attr['absolute_offset'], foldername))
138 | 
139 | args.dump.close()
140 | 


--------------------------------------------------------------------------------
/part/refs/entry_block.py:
--------------------------------------------------------------------------------
  1 | from struct import Struct
  2 | import part.refs.attribute as rattr
  3 | 
  4 | SECTOR_SIZE = 512
  5 | ENTRYBLOCK_SIZE = 16 * 1024
  6 | 
  7 | EB_NUM_OFFSET = 0
  8 | EB_OWN_OFFSET_OFFSET = 0x38
  9 | EB_LENGTH_OWN_OFFSET_OFFSET = 0x3c
 10 | EB_NUM_POINTERS_EXTENT_OFFSET = 0x58
 11 | EB_NUM_SIZE = 8
 12 | EB_OWN_OFFSET_SIZE = 4
 13 | EB_LENGTH_OWN_OFFSET_SIZE = 4
 14 | EB_NUM_POINTERS_EXTENT_SIZE = 4
 15 | 
 16 | EB_HEADER_FORMAT = Struct('<QQ8sQ16s')
 17 | EB_NODE_DESC_FORMAT = Struct('<L20sH6sL')
 18 | EB_NODE_HEADER_FORMAT = Struct('<LLLLLLQ')
 19 | 
 20 | EB_EXTENT_TABLE_FORMAT = Struct('<LLLLLLLL')
 21 | 
 22 | def is_entryblock(dump, offset, vbr_offset, block_size = 16 * 1024):
 23 |     dump.seek(offset, 0)
 24 |     data = dump.read(EB_HEADER_FORMAT.size)
 25 |     eb_num = EB_HEADER_FORMAT.unpack_from(data, 0)
 26 |     return (offset - vbr_offset) / block_size == eb_num[0]
 27 |     # return eb_num != 0
 28 | 
 29 | def read_entryblock(dump, offset):
 30 |     dump.seek(offset, 0)
 31 |     data = dump.read(EB_HEADER_FORMAT.size)
 32 |     fields = EB_HEADER_FORMAT.unpack_from(data, 0)
 33 |     eb = {'_absolute_offset': offset,
 34 |           'eb_number': fields[0],
 35 |           'counter': fields[1],
 36 |           'node_id': fields[3]}
 37 |     eb['_structure_size'] = EB_HEADER_FORMAT.size
 38 |     data = dump.read(EB_NODE_DESC_FORMAT.size)
 39 |     fields = EB_NODE_DESC_FORMAT.unpack_from(data, 0)
 40 |     eb['node_desc_length'] = fields[0]
 41 |     if eb['node_desc_length'] != 0x08:
 42 |         eb['num_extents'] = fields[2]
 43 |         eb['num_records'] = fields[4]
 44 |     eb['_structure_size'] = (eb['_structure_size'] +
 45 |                              eb['node_desc_length'])
 46 |     eb['_contains_records'] = False
 47 |     eb['_contains_extents'] = False
 48 |     if (eb['node_desc_length'] == 0x08 or
 49 |             eb['num_extents'] == 0):
 50 |         eb['_contains_records'] = True
 51 |         eb['node_header_offset'] = eb['_structure_size']
 52 |         dump.seek(offset + eb['_structure_size'], 0)
 53 |         data = dump.read(EB_NODE_HEADER_FORMAT.size)
 54 |         fields = EB_NODE_HEADER_FORMAT.unpack_from(data, 0)
 55 |         eb['header_length'] = fields[0]
 56 |         eb['offset_free_record'] = fields[1]
 57 |         eb['free_space'] = fields[2]
 58 |         eb['header_unknown'] = fields[3]
 59 |         eb['offset_first_pointer'] = fields[4]
 60 |         eb['num_pointers'] = fields[5]
 61 |         eb['offset_end_node'] = fields[6]
 62 |         if eb['num_pointers']:
 63 |             pointers_format = Struct('<' + ('L' * eb['num_pointers']))
 64 |             dump.seek(offset + eb['node_header_offset'] + eb['offset_first_pointer'], 0)
 65 |             data = dump.read(pointers_format.size)
 66 |             fields = pointers_format.unpack_from(data, 0)
 67 |             eb['pointers'] = fields
 68 |             eb['pointers_data'] = []
 69 |             for ptr in eb['pointers']:
 70 |                 ptr_addr = offset + eb['node_header_offset'] + ptr
 71 |                 attr = rattr.read_attribute(dump, ptr_addr)
 72 |                 eb['pointers_data'].append(attr)
 73 |                 eb['_structure_size'] = eb['_structure_size'] + attr['size']
 74 |         else:
 75 |             eb['pointers'] = None
 76 |             eb['pointers_data'] = None
 77 |         eb['_structure_size'] = (eb['_structure_size'] +
 78 |                                  eb['header_length'])
 79 |     elif (eb['node_desc_length'] != 0x08 and eb['num_extents'] != 0):
 80 |         eb['_contains_extents'] = True
 81 |         eb['extent_table_offset'] = eb['_structure_size']
 82 |         dump.seek(offset + eb['_structure_size'], 0)
 83 |         data = dump.read(EB_EXTENT_TABLE_FORMAT.size)
 84 |         fields = EB_EXTENT_TABLE_FORMAT.unpack_from(data, 0)
 85 |         eb['extent_table_length'] = fields[0]
 86 |         eb['extent_table_unknown0'] = fields[1]
 87 |         eb['extent_table_unknown1'] = fields[2]
 88 |         eb['extent_table_unknown2'] = fields[3]
 89 |         eb['offset_first_extent_pointer'] = fields[4]
 90 |         eb['num_extent_pointers'] = fields[5]
 91 |         eb['offset_end_of_extent_pointers'] = fields[6]
 92 |         eb['extent_table_unknown3'] = fields[7]
 93 |         eb['_structure_size'] = eb['_structure_size'] + eb['extent_table_length']
 94 |         pointers_format = Struct('<' + ('L' * eb['num_extent_pointers']))
 95 |         dump.seek(offset + eb['extent_table_offset'] + eb['offset_first_extent_pointer'], 0)
 96 |         data = dump.read(pointers_format.size)
 97 |         fields = pointers_format.unpack_from(data, 0)
 98 |         eb['extent_pointers'] = fields
 99 |         eb['extents'] = []
100 |         for ptr in eb['extent_pointers']:
101 |             ptr_addr = offset + eb['extent_table_offset'] + ptr
102 |             ext = _read_extent(dump, ptr_addr)
103 |             eb['extents'].append(ext)
104 |             eb['_structure_size'] = eb['_structure_size'] + ext['_structure_size']
105 |     return eb
106 | 
107 | EB_EXTENT_HEADER_FORMAT = Struct('<LLHHH')
108 | EB_EXTENT_BODY_FORMAT = Struct('<QQQ')
109 | 
110 | def _read_extent(dump, offset):
111 |     dump.seek(offset)
112 |     data = dump.read(EB_EXTENT_HEADER_FORMAT.size)
113 |     fields = EB_EXTENT_HEADER_FORMAT.unpack_from(data, 0)
114 |     ext = {'_absolute_offset': offset,
115 |            'size': fields[0],
116 |            'unknown0': fields[1],
117 |            'unknown1': fields[2],
118 |            'header_length': fields[3],
119 |            'body_length': fields[4],
120 |            '_structure_size': fields[0]}
121 |     dump.seek(offset + ext['header_length'])
122 |     data = dump.read(EB_EXTENT_BODY_FORMAT.size)
123 |     fields = EB_EXTENT_BODY_FORMAT.unpack_from(data, 0)
124 |     ext['eb_number'] = fields[0]
125 |     ext['0x0000000808020000'] = fields[1]
126 |     ext['crc'] = fields[2]
127 |     return ext
128 | 
129 | def dump_entryblock(eb):
130 |     print('Entryblock: <{:#x}> ({size},{size:#x})'.format(eb['_absolute_offset'],
131 |                                                           size=eb['_structure_size']))
132 |     print('- entryblock number: {:#x}'.format(eb['eb_number']))
133 |     print('- counter: {}'.format(eb['counter']))
134 |     print('- node id: {:#x}'.format(eb['node_id']))
135 |     print('- node descriptor length: {val:#x} ({val})'.format(val=eb['node_desc_length']))
136 |     if 'num_extents' in eb:
137 |         print('- number of extents: {}'.format(eb['num_extents']))
138 |     if 'num_records' in eb:
139 |         print('- number of records: {}'.format(eb['num_records']))
140 |     if eb['_contains_records']:
141 |         print('- node header offset: {val:#x} ({val})'.format(val=eb['node_header_offset']))
142 |         print('- node header length: {val:#x} ({val})'.format(val=eb['header_length']))
143 |         print('- offset to next free record: {val:#x} ({val})'.format(val=eb['offset_free_record']))
144 |         print('- free space in node: {val:#x} ({val})'.format(val=eb['free_space']))
145 |         print('- node header unknown: {val:#x} ({val})'.format(val=eb['header_unknown']))
146 |         print('- offset to first pointer: {val:#x} ({val}) -> entryblock offset: {ebo:#x}'.format(
147 |             val=eb['offset_first_pointer'],
148 |             ebo=eb['offset_first_pointer']+eb['node_header_offset']))
149 |         print('- number of pointers in node: {}'.format(eb['num_pointers']))
150 |         print('- offset to end node: {val:#x} ({val}) -> entryblock offset: {ebo:#x}'.format(
151 |             val=eb['offset_end_node'],
152 |             ebo=eb['offset_end_node']+eb['node_header_offset']))
153 |         print('- pointers:', end='')
154 |         if eb['pointers']:
155 |             for ptr in eb['pointers']:
156 |                 print(' {:#x}'.format(ptr), end='')
157 |             print('')
158 |             print('- pointers data:')
159 |             for ptr_i, ptr in enumerate(eb['pointers_data']):
160 |                 print('  - pointer {}: <{:#x}>'.format(ptr_i, ptr['_absolute_offset']))
161 |                 rattr.dump_attribute(ptr, '    ')
162 |         else:
163 |             print('None')
164 |             print('- pointers data: None')
165 |     elif eb['_contains_extents']:
166 |         print('- extent table offset: {val:#x} ({val})'.format(val=eb['extent_table_offset']))
167 |         print('- extent table length: {val:#x} ({val})'.format(val=eb['extent_table_length']))
168 |         print('- extent table unknown 0: {val:#x} ({val})'.format(val=eb['extent_table_unknown0']))
169 |         print('- extent table unknown 1: {val:#x} ({val})'.format(val=eb['extent_table_unknown1']))
170 |         print('- extent table unknown 2: {val:#x} ({val})'.format(val=eb['extent_table_unknown2']))
171 |         print('- offset to first extent pointer: {val:#x} ({val})'.format(
172 |             val=eb['offset_first_extent_pointer']))
173 |         print('- number of extent pointers: {}'.format(eb['num_extent_pointers']))
174 |         print('- offset en of extent pointers: {val:#x} ({val})'.format(
175 |             val=eb['offset_end_of_extent_pointers']))
176 |         print('- extent table unknown 3: {val:#x} ({val})'.format(val=eb['extent_table_unknown3']))
177 |         print('- extent pointers:', end='')
178 |         for ptr in eb['extent_pointers']:
179 |             print(' {:#x}'.format(ptr), end='')
180 |         print('')
181 |         print('- extents:')
182 |         for ext_i, ext in enumerate(eb['extents']):
183 |             print('  - extent {}: <{:#x}>'.format(ext_i, ext['_absolute_offset']))
184 |             _dump_extent(ext, '    ')
185 | 
186 | def _dump_extent(ext, prefix):
187 |     print('{}- size: {val:#x} ({val})'.format(prefix, val=ext['size']))
188 |     print('{}- unknown 0: {val:#x} ({val})'.format(prefix, val=ext['unknown0']))
189 |     print('{}- unknown 1: {val:#x} ({val})'.format(prefix, val=ext['unknown1']))
190 |     print('{}- header length: {val:#x} ({val})'.format(prefix, val=ext['header_length']))
191 |     print('{}- body length: {val:#x} ({val})'.format(prefix, val=ext['body_length']))
192 |     print('{}- entryblock id: {val:#x} ({val})'.format(prefix, val=ext['eb_number']))
193 |     print('{}- 0x0000000808020000: {:#018x}'.format(prefix, ext['0x0000000808020000']))
194 |     print('{}- crc(?): {:#018x}'.format(prefix, ext['crc']))
195 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Introduction
  2 | 
  3 | `pyrefs` provides a series of libraries helpful to parse and analyze Resilient
  4 | FileSystems (ReFS).
  5 | An interactive application to analyze a ReFS dump named `Igor` is also
  6 | provided.
  7 | 
  8 | `pyrefs` is still very immature, as of today it has been only tested with
  9 | partitions using ReFS version 1.1.
 10 | The code is neither far from perfect, and it would need a good refactoring.
 11 | We are working on that, but be aware when you retrieve `pyrefs`.
 12 | 
 13 | Finally, `pyrefs` naming convention for ReFS structures is mainly guided by
 14 | Henry Georges thesis work.
 15 | You can find Henry Georges's thesis and the code he developed to analyze ReFS
 16 | here <https://brage.bibsys.no/xmlui/handle/11250/2502565>.
 17 | 
 18 | ## Authors
 19 | 
 20 |  - Daniel Gracia Pérez \<<daniel.gracia-perez@cfa-afti.fr>\>
 21 |  - Eric Jouenne \<<eric.jouenne@cfa-afti.fr>\>
 22 |  - Tony Gerard \<<tony.gerard@cfa-afti.fr>\>
 23 |  - Francois-Xavier Babin \<<francois-xavier.babin@cfa-afti.fr>\>
 24 | 
 25 | ## Thanks
 26 | 
 27 | We would like to thank: Henry Georges, his thesis document has been the
 28 | foundation of this project; Willi Ballenthin, for the ReFS dump images on his
 29 | website and his ReFS articles; and all the other authors providing information
 30 | on the ReFS filesystem.
 31 | 
 32 | We would like to also thank Juan Romero, Olivier Gilles, Isabelle Isabelle
 33 | Moukala-Mouberi and Claudia Duhoo, who performed a similar work than ours.
 34 | Your feedback was unvaluable.
 35 | 
 36 | # Dependencies
 37 | 
 38 | `pyrefs` was developed using Python 3 (Python 3.6.6) at the time of this
 39 | release, and has been tested to work on Windows and Linux platforms.
 40 | 
 41 | No special dependencies, if you find any please let us know.
 42 | 
 43 | # Sources organization
 44 | 
 45 | ```
 46 | .
 47 | +-- README.md
 48 | +-- LICENSE
 49 | +-- igor.py
 50 | +-- part
 51 | |   +-- refs
 52 | |       +-- allocator.py
 53 | |       +-- attribute.py
 54 | |       +-- entry_block.py
 55 | |       +-- object.py
 56 | |       +-- object_tree.py
 57 | |       +-- tree_control.py
 58 | |       +-- vol.py
 59 | +-- media
 60 | |   +-- gpt.py
 61 | |   +-- mbr.py
 62 | +-- util
 63 | |   +-- carving.py
 64 | |   +-- filetree.py
 65 | |   +-- hexdump.py
 66 | |   +-- table.py
 67 | |   +-- time.py
 68 | +-- examples
 69 |     +-- carving.rec
 70 |     +-- filetree.rec
 71 |     +-- tree_explorer.rec
 72 | ```
 73 | 
 74 | # Igor
 75 | 
 76 | An interactive command line tool is provided to facilitate the analysis of
 77 | those not willing to play with the code.
 78 | We call it `Igor` and it can be launched with Python.
 79 | 
 80 | `Igor` provides a limited toolset to analyze ReFS partitions exploiting the
 81 | `pyrefs` library, the following sections enumerate them.
 82 | 
 83 | ## Examples
 84 | 
 85 | Some examples of Igor usage are provided in the `examples` folder as Igor
 86 | records.
 87 | To execute/playback the examples contained in the `examples` folder download
 88 | the `Windows8ReFSImages.zip` file provided by Willi Ballenthin at
 89 | <http://www.williballenthin.com/forensics/refs/test_images/>, copy it to the
 90 | `examples/dump` folder and uncompress it.
 91 | Then you can use igor to test the records in the examples folder with the
 92 | command `playback examples/<example_file>`.
 93 | 
 94 | ## Command reference
 95 | 
 96 | ### file
 97 | 
 98 | Usage: `file [-h] [-i] [-f] [-F] dump`
 99 | 
100 | Load the provided dump file for analysis, and automatically select the ReFS
101 | partition for you.
102 | You can also initialize the list of entryblocks of the partition.
103 | 
104 | Positional arguments:
105 | 
106 |  - `dump`: File to use as dump for the analysis.
107 | 
108 | Optional arguments:
109 | 
110 |  - `-h`, `--help`: show this help message and exit
111 |  - `-i`, `--initiliaze-entryblocks`: find blocks in provided dump
112 |  - `-f`, `--files`: find files in provided dump (only considered if -i defined)
113 |  - `-F`, `--folders`: find folders in provided dump (only considered if -i
114 |    defined)
115 | 
116 | ### vol
117 | 
118 | Usage: `vol [-h]`
119 | 
120 | Dump the volume record information from the current ReFS partition.
121 | 
122 | Optional arguments:
123 | 
124 |  - `-h`, `--help`: show this help message and exit
125 | 
126 | Dump the volume record information from the current ReFS partition.
127 | 
128 | ### part
129 | 
130 | Usage: `part [-h] [partidx]`
131 | 
132 | Show available partitions in the currently loaded dump if no parameter is
133 | given, switch to the provided partition if any provided.
134 | 
135 | Positional arguments:
136 | 
137 |  - `partidx`: Partition index of the partition to use for analysis
138 | 
139 | Optional arguments:
140 | 
141 |  - `-h`, `--help`: show this help message and exit
142 | 
143 | ### find\_entryblocks
144 | 
145 | Usage: `find_entryblocks [-h] [-f] [-F]`
146 | 
147 | Find and show all the entryblocks in current partition. If requested number of
148 | files and folders information will be also collected.
149 | 
150 | Optional arguments:
151 | 
152 |  - `-h`, `--help`: show this help message and exit
153 |  - `-f`, `--files`: Collect information on the number of files in the
154 |    entryblocks.
155 |  - `-F`, `--folders`: Collect information on the number of folders in the
156 |    entryblocks.
157 | 
158 | ### find\_pattern
159 | 
160 | Usage: `find_pattern [-h] pattern`
161 | 
162 | Find a data pattern in all the blocks of the current partition. Special
163 | characters (including spaces, carriage return, etc.) are not allowed in the
164 | pattern, think of escaping them.
165 | 
166 | Positional arguments:
167 | 
168 |  - `pattern`: Pattern to find in the current partition blocks.
169 | 
170 | Optional arguments:
171 | 
172 |  - `-h`, `--help`: show this help message and exit
173 | 
174 | ### list\_filenames
175 | 
176 | Usage: `list_filenames [-h]`
177 | 
178 | List the found filenames from the list of entryblocks with filenames.
179 | 
180 | Optional arguments:
181 | 
182 |  - `-h`, `--help`: show this help message and exit
183 | 
184 | ### list\_folders
185 | 
186 | Usage: `list_folders [-h]`
187 | 
188 | List the found filename folders from the list of entryblocks with folders.
189 | 
190 | Optional arguments:
191 | 
192 |  - `-h`, `--help`: show this help message and exit
193 | 
194 | ### entryblock
195 | 
196 | Usage: `entryblock [-h] entryblock_identifier`
197 | 
198 | Dump the provided entryblock identifier as EntryBlock.
199 | 
200 | Positional arguments:
201 | 
202 |  - `entryblock_identifier`: entryblock identifier of the EntryBlock to dump
203 | 
204 | Optional arguments:
205 | 
206 |  - `-h`, `--help`: show this help message and exit
207 | 
208 | ### tree\_control
209 | 
210 | Usage: `tree_control [-h] [entryblock_identifier]`
211 | 
212 | Parse the given entryblock identifier as a TreeControl.
213 | If no entryblock identifier is provided it uses 0x1e as entryblock identifier.
214 | 
215 | Positional arguments:
216 | 
217 |  - `entryblock_identifier`: entryblock identifier of the TreeControl to dump
218 | 
219 | Optional arguments:
220 | 
221 |  - `-h`, `--help`: show this help message and exit
222 | 
223 | ### tree\_control\_extension
224 | 
225 | Usage: `tree_control_extension [-h] entryblock_identifier`
226 | 
227 | Parse the given entryblock identifier as a TreeControlExtension.
228 | 
229 | Positional arguments:
230 | 
231 |  - `entryblock_identifier`: entryblock identifier of the TreeControlExtension
232 |    to dump
233 | 
234 | Optional arguments:
235 | 
236 |  - `-h`, `--help`: show this help message and exit
237 | 
238 | ### object\_tree
239 | 
240 | Usage: `object_tree [-h] entryblock_identifier`
241 | 
242 | Parse the given entryblock identifier as an ObjectTree.
243 | 
244 | Positional arguments:
245 | 
246 |  - `entryblock_identifier`: entryblock identifier of the ObjectTree to dump
247 | 
248 | Optional arguments:
249 | 
250 |  - `-h`, `--help`: show this help message and exit
251 | 
252 | ### allocator
253 | 
254 | Usage: `allocator [-h] entryblock_identifier`
255 | 
256 | Parse the given entryblock identifier as an Allocator.
257 | 
258 | Positional arguments:
259 | 
260 |  - `entryblock_identifier`: entryblock identifier of the Allocator to dump
261 | 
262 | Optional arguments:
263 | 
264 |  - `-h`, `--help`: show this help message and exit
265 | 
266 | ### attribute
267 | 
268 | Usage: `attribute [-h] dump_offset`
269 | 
270 | Parse the given dump offset (in bytes) as an Attribute.
271 | 
272 | Positional arguments:
273 | 
274 |  - `dump_offset`: dump offset (in bytes) of the Attribute to dump
275 | 
276 | Optional arguments:
277 | 
278 |  - `-h`, `--help`: show this help message and exit
279 | 
280 | ### datastream
281 | 
282 | Usage:
283 | ```
284 | datastream [-h] output_filename output_size
285 |            entryblock_identifier,number_of_blocks
286 |            [entryblock_identifier,number_of_blocks ...]
287 | ```
288 | 
289 | Extract data from dump from the given datarun.
290 | Dataruns can be extracted by exploring the EntryBlocks (command `entryblock`)
291 | or using the `list_dataruns` command.
292 | 
293 | Positional arguments:
294 | 
295 |  - `output_filename`: name of the generated output file
296 |  - `output_size`: total amount of data to extract
297 |  - `entryblock_identifier,number_of_blocks`: datarun to follow for the data extraction
298 | 
299 | Optional arguments:
300 | 
301 |  - `-h`, `--help`: show this help message and exit
302 | 
303 | ### list\_dataruns
304 | 
305 | Usage: `list_dataruns [-h]`
306 | 
307 | Retrieve list of all the files dataruns.
308 | 
309 | Optional arguments:
310 | 
311 |  - `-h`, `--help`: show this help message and exit
312 | 
313 | ### filetree
314 | 
315 | Usage: `filetree [-h] node_id`
316 | 
317 | Extract the file tree structure from the given node (use node 0x600 by
318 | default).
319 | 
320 | Positional arguments:
321 | 
322 |  - `node_id`: node identifier of the node to extract the file tree structure
323 |    from
324 | 
325 | Optional arguments:
326 | 
327 |  - `-h`, `--help`: show this help message and exit
328 | 
329 | ### hexdump
330 | 
331 | Usage: `hexdump [-h] dump_offset size`
332 | 
333 | Hexdump the number of bytes at the provided offset.
334 | 
335 | Positional arguments:
336 | 
337 |  - `dump_offset`: dump offset of the hexdump start
338 |  - `size`: number of bytes to dump
339 | 
340 | Optional arguments:
341 | 
342 |  - `-h`, `--help`: show this help message and exit
343 | 
344 | ### hexblock
345 | 
346 | Usage: `hexblock [-h] entryblock_id`
347 | 
348 | Hexdump the block with the provided entryblock identifier.
349 | 
350 | Positional arguments:
351 | 
352 |  - `entryblock_id`: entryblock identifier to hexdump
353 | 
354 | Optional arguments:
355 | 
356 |  - `-h`, `--help`: show this help message and exit
357 | 
358 | ### bye or \<ctrl-d\>
359 | 
360 | Usage: `bye [-h]`
361 | 
362 | Exit the program. Are you sure?
363 | 
364 | Optional arguments:
365 | 
366 |  - `-h`, `--help`: show this help message and exit
367 | 
368 | ### record
369 | 
370 | Usage: `record [-h] output_file`
371 | 
372 | Save the following commands to selected file.
373 | Saved file can be used afterwards for replay with the `playback` command.
374 | 
375 | Positional arguments:
376 | 
377 |  - `output_file`: file onto which commands will be saved
378 | 
379 | Optional arguments:
380 | 
381 |  - `-h`, `--help`: show this help message and exit
382 | 
383 | ### playback
384 | 
385 | Usage: `playback [-h] input_file`
386 | 
387 | Execute the sequence of commands defined in the input file.
388 | 
389 | Positional arguments:
390 | 
391 |  - `input_file`: file from which commands to execute will be read
392 | 
393 | Optional arguments:
394 | 
395 |  - `-h`, `--help`: show this help message and exit
396 | 


--------------------------------------------------------------------------------
/part/refs/attribute.py:
--------------------------------------------------------------------------------
  1 | from util.time import bytes2time
  2 | from struct import Struct
  3 | 
  4 | ATTR_SIZE_OFFSET = 0
  5 | ATTR_SIZE_SIZE = 4
  6 | ATTR_SIZE_FORMAT = Struct('<L')
  7 | ATTR_TYPE_OFFSET = 0x10
  8 | ATTR_TYPE_SIZE = 4
  9 | ATTR_TYPE_FORMAT = Struct('<L')
 10 | 
 11 | ATTR_FN_METADATA_FORMAT = Struct('<LH34sQQQQB7sQQQQQ')
 12 | 
 13 | def read_filename_attribute_metadata(dump, offset):
 14 |     dump.seek(offset, 0)
 15 |     data = dump.read(ATTR_FN_METADATA_FORMAT.size)
 16 |     fields = ATTR_FN_METADATA_FORMAT.unpack_from(data, 0)
 17 |     attr = {'_absolute_offset': offset,
 18 |             'size': fields[0],
 19 |             'offset_to_val?': fields[1],
 20 |             'created': fields[3],
 21 |             'modified': fields[4],
 22 |             'metadata_modified': fields[5],
 23 |             'last_accessed': fields[6],
 24 |             'flags': fields[7],
 25 |             'parentid': fields[9],
 26 |             'childid': fields[10],
 27 |             'unknown': fields[11],
 28 |             'logical_size': fields[12],
 29 |             'physical_size': fields[13]}
 30 |     return attr
 31 | 
 32 | def _dump_filename_attribute_metadata(attr, prefix=''):
 33 |     print('{}- size: {val:#x} ({val})'.format(prefix, val=attr['size']))
 34 |     print('{}- offset to first value(?): {val:#x} ({val})'.format(prefix, val=attr['offset_to_val?']))
 35 |     print('{}- created: {} ({:#x})'.format(prefix, bytes2time(attr['created']), attr['created']))
 36 |     print('{}- modified: {} ({:#x})'.format(prefix, bytes2time(attr['modified']), attr['modified']))
 37 |     print('{}- metadata modified: {} ({:#x})'.format(prefix, bytes2time(attr['metadata_modified']),
 38 |                                                      attr['metadata_modified']))
 39 |     print('{}- last accessed: {} ({:#x})'.format(prefix, bytes2time(attr['last_accessed']),
 40 |                                                  attr['last_accessed']))
 41 |     print('{}- flags: {:#x}'.format(prefix, attr['flags']))
 42 |     print('{}- parent node id: {:#x}'.format(prefix, attr['parentid']))
 43 |     print('{}- child node id: {:#x}'.format(prefix, attr['childid']))
 44 |     print('{}- unknown field: {:#x}'.format(prefix, attr['unknown']))
 45 |     print('{}- logical size: {val:#x} ({val})'.format(prefix, val=attr['logical_size']))
 46 |     print('{}- physical size: {val:#x} ({val})'.format(prefix, val=attr['physical_size']))
 47 | 
 48 | ATTR_FN_DATARUN_ENTRY_HEADER_FORMAT = Struct('<LHHHHLLLL')
 49 | ATTR_FN_DATARUN_ENTRY_BODY_FORMAT = Struct('<L48sQQ')
 50 | ATTR_FN_DATARUN_ENTRY_BODY_LIST_FORMAT = Struct('<LLL4sLLL')
 51 | ATTR_FN_DATARUN_ENTRY_BODY_LIST_ENTRY_FORMAT = Struct('<L20sQQ')
 52 | 
 53 | def read_filename_attribute_datarun_entry(dump, offset):
 54 |     dump.seek(offset, 0)
 55 |     data = dump.read(ATTR_FN_DATARUN_ENTRY_HEADER_FORMAT.size)
 56 |     fields = ATTR_FN_DATARUN_ENTRY_HEADER_FORMAT.unpack_from(data, 0)
 57 |     attr = {'_absolute_offset': offset,
 58 |             'size': fields[0],
 59 |             'unknown0': fields[1],
 60 |             'unknown1': fields[2],
 61 |             'unknown2': fields[3],
 62 |             'header_size': fields[4],
 63 |             'header_body_size': fields[5],
 64 |             'body_size_copy': fields[6],
 65 |             'unknown3': fields[7],
 66 |             'attributeid?': fields[8]}
 67 |     attr['_body_offset'] = attr['header_size']
 68 |     dump.seek(offset + attr['_body_offset'], 0)
 69 |     data = dump.read(ATTR_FN_DATARUN_ENTRY_BODY_FORMAT.size)
 70 |     fields = ATTR_FN_DATARUN_ENTRY_BODY_FORMAT.unpack_from(data, 0)
 71 |     attr['body_size'] = fields[0]
 72 |     # NOTE: This is wrong in the thesis, logical comes before physical
 73 |     attr['physical_size'] = fields[2]
 74 |     attr['logical_size'] = fields[3]
 75 |     attr['_body_list_offset'] = attr['_body_offset'] + attr['body_size']
 76 |     dump.seek(offset + attr['_body_list_offset'], 0)
 77 |     data = dump.read(ATTR_FN_DATARUN_ENTRY_BODY_LIST_FORMAT.size)
 78 |     fields = ATTR_FN_DATARUN_ENTRY_BODY_LIST_FORMAT.unpack_from(data, 0)
 79 |     attr['body_list_size'] = fields[0]
 80 |     attr['body_list_offset_next_record'] = fields[1]
 81 |     attr['body_list_free_space'] = fields[2]
 82 |     attr['offset_pointers'] = fields[4]
 83 |     attr['num_pointers'] = fields[5]
 84 |     attr['body_list_end_struct'] = fields[6]
 85 |     if attr['num_pointers']:
 86 |         pointers_format = Struct('<' + ('L' * attr['num_pointers']))
 87 |         dump.seek(offset + attr['_body_list_offset'] + attr['offset_pointers'], 0)
 88 |         data = dump.read(pointers_format.size)
 89 |         fields = pointers_format.unpack_from(data, 0)
 90 |         attr['pointers'] = fields
 91 |         attr['pointers_data'] = []
 92 |     else:
 93 |         attr['pointers'] = None
 94 |         attr['pointers_data'] = None
 95 |     attr['_structure_size'] = attr['_body_list_offset'] + attr['body_list_size']
 96 |     if attr['pointers']:
 97 |         for ptr in attr['pointers']:
 98 |             ptr_addr = offset + attr['_body_list_offset'] + ptr
 99 |             dump.seek(ptr_addr, 0)
100 |             data = dump.read(ATTR_FN_DATARUN_ENTRY_BODY_LIST_ENTRY_FORMAT.size)
101 |             fields = ATTR_FN_DATARUN_ENTRY_BODY_LIST_ENTRY_FORMAT.unpack_from(data, 0)
102 |             entry = {'_absolute_offset': ptr_addr,
103 |                      'size': fields[0],
104 |                      'num_blocks': fields[2],
105 |                      'blockid': fields[3]}
106 |             attr['pointers_data'].append(entry)
107 |     return attr
108 | 
109 | def _dump_filename_attribute_datarun_entry(attr, prefix=''):
110 |     print('{}- size: {val:#x} ({val})'.format(prefix, val=attr['size']))
111 |     print('{}- unknown field 0: {val:#x} ({val})'.format(prefix, val=attr['unknown0']))
112 |     print('{}- unknown field 1: {val:#x} ({val})'.format(prefix, val=attr['unknown1']))
113 |     print('{}- unknown field 2: {val:#x} ({val})'.format(prefix, val=attr['unknown2']))
114 |     print('{}- header size: {val:#x} ({val})'.format(prefix, val=attr['header_size']))
115 |     print('{}- body size (info in header): {val:#x} ({val})'.format(prefix, val=attr['header_body_size']))
116 |     print('{}- body size (copy): {val:#x} ({val})'.format(prefix, val=attr['body_size_copy']))
117 |     print('{}- unknown field 3: {val:#x} ({val})'.format(prefix, val=attr['unknown3']))
118 |     print('{}- attribute type identifier: {:#010x}'.format(prefix, attr['attributeid?']))
119 |     print('{}- body size: {val:#x} ({val})'.format(prefix, val=attr['body_size']))
120 |     print('{}- logical size: {val:#x} ({val})'.format(prefix, val=attr['logical_size']))
121 |     print('{}- physical size: {val:#x} ({val})'.format(prefix, val=attr['physical_size']))
122 |     print('{}- body list size: {val:#x} ({val})'.format(prefix, val=attr['body_list_size']))
123 |     print('{}- body list offset to next record: {val:#x} ({val})'.format(prefix,
124 |         val=attr['body_list_offset_next_record']))
125 |     print('{}- body list free space: {val:#x} ({val})'.format(prefix, val=attr['body_list_free_space']))
126 |     print('{}- offset to pointers: {val:#x} ({val})'.format(prefix, val=attr['offset_pointers']))
127 |     print('{}- number of pointers: {}'.format(prefix, attr['num_pointers']))
128 |     print('{}- body list end of structure: {val:#x} ({val})'.format(prefix,
129 |         val=attr['body_list_end_struct']))
130 |     if attr['pointers']:
131 |         print('{}- pointers:'.format(prefix), end='')
132 |         for ptr in attr['pointers']:
133 |             print(' {:#x}'.format(ptr), end='')
134 |         print('')
135 |         print('{}- pointers data:'.format(prefix))
136 |         for ptr_i, ptr in enumerate(attr['pointers_data']):
137 |             print('{}  - pointer {}: <{:#x}>'.format(prefix, ptr_i, ptr['_absolute_offset']))
138 |             print('{}    - entry size: {val:#x} ({val})'.format(prefix, val=ptr['size']))
139 |             print('{}    - number of blocks/clusters: {}'.format(prefix, ptr['num_blocks']))
140 |             print('{}    - block identifier: {:#x}'.format(prefix, ptr['blockid']))
141 |     else:
142 |         print('{}- pointers: None'.format(prefix))
143 |         print('{}- pointers data: None'.format(prefix))
144 | 
145 | ATTR_FN_DATARUN_HEADER_FORMAT = Struct('<LLLLLLL')
146 | 
147 | def read_filename_attribute_datarun(dump, offset):
148 |     dump.seek(offset, 0)
149 |     data = dump.read(ATTR_FN_DATARUN_HEADER_FORMAT.size)
150 |     fields = ATTR_FN_DATARUN_HEADER_FORMAT.unpack_from(data, 0)
151 |     attr = {'_absolute_offset': offset,
152 |             'size': fields[0],
153 |             'offset_next_free_record': fields[1],
154 |             'free_space': fields[2],
155 |             'unknown0': fields[3],
156 |             'offset_first_pointer': fields[4],
157 |             'num_pointers': fields[5],
158 |             'end_of_struct': fields[6]}
159 |     if attr['num_pointers']:
160 |         pointers_format = Struct('<' + ('L' * attr['num_pointers']))
161 |         dump.seek(offset + attr['offset_first_pointer'], 0)
162 |         data = dump.read(pointers_format.size)
163 |         fields = pointers_format.unpack_from(data, 0)
164 |         attr['pointers'] = fields
165 |         attr['pointers_data'] = []
166 |         for ptr in attr['pointers']:
167 |             ptr_addr = offset + ptr
168 |             attr['pointers_data'].append(read_filename_attribute_datarun_entry(dump, ptr_addr))
169 |     else:
170 |         attr['pointers'] = None
171 |         attr['pointers_data'] = None
172 |     return attr
173 | 
174 | def _dump_filename_attribute_datarun(attr, prefix=''):
175 |     print('{}- size: {val:#x} ({val})'.format(prefix, val=attr['size']))
176 |     print('{}- offset next free record: {:#x}'.format(prefix, attr['offset_next_free_record']))
177 |     print('{}- free space: {val:#x} ({val})'.format(prefix, val=attr['free_space']))
178 |     print('{}- unknown value: {val:#x} ({val})'.format(prefix, val=attr['unknown0']))
179 |     print('{}- offset to first pointer: {:#x}'.format(prefix, attr['offset_first_pointer']))
180 |     print('{}- number of pointers: {}'.format(prefix, attr['num_pointers']))
181 |     print('{}- end of structure: {val:#x} ({val})'.format(prefix, val=attr['end_of_struct']))
182 |     if attr['pointers']:
183 |         print('{}- pointers:'.format(prefix), end='')
184 |         for ptr in attr['pointers']:
185 |             print(' {:#x}'.format(ptr), end='')
186 |         print('')
187 |         for ptr_i, ptr in enumerate(attr['pointers_data']):
188 |             print('{}  - pointer {}: <{:#x}>'.format(prefix, ptr_i, ptr['_absolute_offset']))
189 |             _dump_filename_attribute_datarun_entry(ptr, prefix + '    ')
190 |     else:
191 |         print('{}- pointers: None'.format(prefix))
192 |         print('{}- pointers data: None'.format(prefix))
193 | 
194 | ATTR_FILENAME_HEADER_FORMAT = Struct('<LHHHHHHL')
195 | 
196 | ATTR_TYPE_FILENAME = 0x00010030
197 | 
198 | def read_filename_attribute(dump, offset):
199 |     dump.seek(offset, 0)
200 |     # TODO: the header should be read in two steps using header_length
201 |     data = dump.read(ATTR_FILENAME_HEADER_FORMAT.size)
202 |     fields = ATTR_FILENAME_HEADER_FORMAT.unpack_from(data, 0)
203 |     attr = {'_absolute_offset': offset,
204 |             'size': fields[0],
205 |             'header_length': fields[1],
206 |             'length': fields[2],
207 |             'next_struct_offset': fields[4],
208 |             'record_rem_data': fields[5],
209 |             'type': fields[7]}
210 |     f_size = attr['length'] - ATTR_TYPE_SIZE
211 |     if f_size > 0:
212 |         dump.seek(offset + ATTR_TYPE_OFFSET + ATTR_TYPE_SIZE, 0)
213 |         fn = dump.read(f_size)
214 |     else:
215 |         fn = 'DEADBEEF'.encode('utf-16le')
216 |     attr['filename'] = fn
217 |     attr['_offset_metadata'] = attr['next_struct_offset']
218 |     attr['metadata'] = read_filename_attribute_metadata(dump, offset + attr['_offset_metadata'])
219 |     attr['_offset_datarun'] = attr['_offset_metadata'] + attr['metadata']['size']
220 |     if attr['metadata']['physical_size'] == 0:
221 |         attr['datarun'] = None
222 |     else:
223 |         attr['datarun'] = read_filename_attribute_datarun(dump, offset + attr['_offset_datarun'])
224 |     return attr
225 | 
226 | def _dump_filename_attribute(attr, prefix=''):
227 |     print('{}- size: {val:#x} ({val})'.format(prefix, val=attr['size']))
228 |     print('{}- header length: {val:#x} ({val})'.format(prefix, val=attr['header_length']))
229 |     print('{}- filename + attribute type length: {val:#x} ({val})'.format(prefix, val=attr['length']))
230 |     print('{}- next structure offset: {val:#x} ({val})'.format(prefix, val=attr['next_struct_offset']))
231 |     print('{}- record remaining data: {val:#x} ({val})'.format(prefix, val=attr['record_rem_data']))
232 |     print('{}- record type: {:#x} (filename attribute)'.format(prefix, attr['type']))
233 |     print('{}- filename: {}'.format(prefix, attr['filename'].decode('utf-16le')))
234 |     print('{}- metadata: <{:#x}>'.format(prefix, attr['metadata']['_absolute_offset']))
235 |     _dump_filename_attribute_metadata(attr['metadata'], prefix + '  ')
236 |     if attr['datarun']:
237 |         print('{}- datarun: <{:#x}>'.format(prefix, attr['datarun']['_absolute_offset']))
238 |         _dump_filename_attribute_datarun(attr['datarun'], prefix + '  ')
239 |     else:
240 |         print('{}- datarun: None'.format(prefix))
241 | 
242 | ATTR_FILENAME_FOLDER_HEADER_1_FORMAT = Struct('<LHHHHHH')
243 | ATTR_FILENAME_FOLDER_HEADER_2_FORMAT = Struct('<L')
244 | ATTR_FILENAME_FOLDER_BODY_FORMAT = Struct('<Q8sQQQQ')
245 | 
246 | ATTR_TYPE_FILENAME_FOLDER   = 0x00020030
247 | 
248 | def read_filename_folder_attribute(dump, offset):
249 |     dump.seek(offset, 0)
250 |     data = dump.read(ATTR_FILENAME_FOLDER_HEADER_1_FORMAT.size)
251 |     fields = ATTR_FILENAME_FOLDER_HEADER_1_FORMAT.unpack_from(data, 0)
252 |     attr = {'_absolute_offset': offset,
253 |             'size': fields[0],
254 |             'offset_identifier': fields[1],
255 |             'header_rem_data': fields[2],
256 |             'unknown0': fields[3],
257 |             'header_length': fields[4],
258 |             'record_rem_data': fields[5],
259 |             'unknown1': fields[6]}
260 |     dump.seek(offset + attr['offset_identifier'], 0)
261 |     data = dump.read(ATTR_FILENAME_FOLDER_HEADER_2_FORMAT.size)
262 |     fields = ATTR_FILENAME_FOLDER_HEADER_2_FORMAT.unpack_from(data, 0)
263 |     attr['type'] = fields[0]
264 |     # f_size = attr['header_length'] - (attr['offset_identifier'] + ATTR_TYPE_SIZE)
265 |     f_size = attr['header_rem_data'] - ATTR_TYPE_SIZE
266 |     if f_size > 0:
267 |         dump.seek(offset + attr['offset_identifier'] + ATTR_TYPE_SIZE, 0)
268 |         fn = dump.read(f_size)
269 |     else:
270 |         fn = 'DEADBEEF'.encode('utf-16le')
271 |     attr['foldername'] = fn
272 |     dump.seek(offset + attr['header_length'], 0)
273 |     data = dump.read(ATTR_FILENAME_FOLDER_BODY_FORMAT.size)
274 |     fields = ATTR_FILENAME_FOLDER_BODY_FORMAT.unpack_from(data, 0)
275 |     attr['nodeid'] = fields[0]
276 |     attr['created'] = fields[2]
277 |     attr['modified'] = fields[3]
278 |     attr['metadata_modified'] = fields[4]
279 |     attr['last_accessed'] = fields[5]
280 |     return attr
281 | 
282 | def _dump_filename_folder_attribute(attr, prefix=''):
283 |     print('{}- size: {val:#x} ({val})'.format(prefix, val=attr['size']))
284 |     print('{}- offset of identifier: {val:#x} ({val})'.format(prefix, val=attr['offset_identifier']))
285 |     print('{}- header remaining bytes: {val:#x} ({val})'.format(prefix, val=attr['header_rem_data']))
286 |     print('{}- unknown field 0: {:#x}'.format(prefix, attr['unknown0']))
287 |     print('{}- header length: {val:#x} ({val})'.format(prefix, val=attr['header_length']))
288 |     print('{}- record remaining data: {val:#x} ({val})'.format(prefix, val=attr['record_rem_data']))
289 |     print('{}- unknown field 1: {:#x}'.format(prefix, attr['unknown1']))
290 |     print('{}- type: {:#x} (filename folder attribute)'.format(prefix, attr['type']))
291 |     print('{}- foldername: {}'.format(prefix, attr['foldername'].decode('utf-16le')))
292 |     print('{}- node identifier: {:#x}'.format(prefix, attr['nodeid']))
293 |     print('{}- created: {} ({:#x})'.format(prefix, bytes2time(attr['created']), attr['created']))
294 |     print('{}- modified: {} ({:#x})'.format(prefix, bytes2time(attr['modified']), attr['modified']))
295 |     print('{}- metadata modified: {} ({:#x})'.format(prefix, bytes2time(attr['metadata_modified']),
296 |                                                      attr['metadata_modified']))
297 |     print('{}- last accessed: {} ({:#x})'.format(prefix, bytes2time(attr['last_accessed']),
298 |                                                  attr['last_accessed']))
299 | 
300 | ATTR_CHILD_HEADER_FORMAT = Struct('<LHH2sHH2sL4sL4sQ8sHH')
301 | 
302 | ATTR_TYPE_CHILD    = 0x80000020
303 | 
304 | def read_child_attribute(dump, offset):
305 |     dump.seek(offset, 0)
306 |     data = dump.read(ATTR_CHILD_HEADER_FORMAT.size)
307 |     data_fields = ATTR_CHILD_HEADER_FORMAT.unpack_from(data, 0)
308 |     attr = {'_absolute_offset': offset,
309 |             'size': data_fields[0],
310 |             'offset_identifier': data_fields[1],
311 |             'header_rem_data': data_fields[2],
312 |             'header_length': data_fields[4],
313 |             'record_rem_data': data_fields[5],
314 |             'type': data_fields[7],
315 |             'parentid': data_fields[9],
316 |             'childid': data_fields[11],
317 |             '000c': data_fields[13],
318 |             'length_name': data_fields[14]}
319 |     fn = dump.read(attr['length_name'])
320 |     attr['filename'] = fn
321 |     return attr
322 | 
323 | def _dump_child_attribute(attr, prefix=''):
324 |     print('{}- size: {val:#x} ({val})'.format(prefix, val=attr['size']))
325 |     print('{}- offset identifier: {val:#x} ({val})'.format(prefix, val=attr['offset_identifier']))
326 |     print('{}- header remaining data: {val:#x} ({val})'.format(prefix, val=attr['header_rem_data']))
327 |     print('{}- header length: {val:#x} ({val})'.format(prefix, val=attr['header_length']))
328 |     print('{}- record remaining data: {val:#x} ({val})'.format(prefix, val=attr['record_rem_data']))
329 |     print('{}- record type: {:#x} (child attribute)'.format(prefix, attr['type']))
330 |     print('{}- parent id: {:#x}'.format(prefix, attr['parentid']))
331 |     print('{}- child id: {:#x}'.format(prefix, attr['childid']))
332 |     print('{}- 0x000c: {:#06x}'.format(prefix, attr['000c']))
333 |     print('{}- filename: {}'.format(prefix, attr['filename'].decode('utf-16le')))
334 | 
335 | ATTR_DIR_METADATA_HEADER_FORMAT = Struct('<LHHHHHH')
336 | ATTR_DIR_METADATA_HEADER_2_FORMAT = Struct('<L')
337 | ATTR_DIR_METADATA_BODY_FORMAT = Struct('<LH34sQQQQ8sQ')
338 | ATTR_DIR_METADATA_PSEC_FORMAT = Struct('<L12sLLL')
339 | 
340 | ATTR_TYPE_DIRECTORY_METADATA = 0x00000010
341 | 
342 | def read_directory_metadata_attribute(dump, offset):
343 |     dump.seek(offset)
344 |     data = dump.read(ATTR_DIR_METADATA_HEADER_FORMAT.size)
345 |     fields = ATTR_DIR_METADATA_HEADER_FORMAT.unpack_from(data, 0)
346 |     attr = {'_absolute_offset': offset,
347 |             'size': fields[0],
348 |             'offset_identifier': fields[1],
349 |             'header_rem_data': fields[2],
350 |             'header_length': fields[4],
351 |             'record_rem_data': fields[5],
352 |             '_structure_size': fields[0]}
353 |     dump.seek(offset + attr['offset_identifier'])
354 |     data = dump.read(ATTR_DIR_METADATA_HEADER_2_FORMAT.size)
355 |     fields = ATTR_DIR_METADATA_HEADER_2_FORMAT.unpack_from(data, 0)
356 |     attr['type'] = fields[0]
357 |     dump.seek(offset + attr['header_length'])
358 |     data = dump.read(ATTR_DIR_METADATA_BODY_FORMAT.size)
359 |     fields = ATTR_DIR_METADATA_BODY_FORMAT.unpack_from(data, 0)
360 |     attr['_offset_body'] = attr['header_length']
361 |     attr['body_length'] = fields[0]
362 |     attr['offset_first_timestamp'] = fields[1]
363 |     attr['created'] = fields[3]
364 |     attr['modified'] = fields[4]
365 |     attr['metadata_modified'] = fields[5]
366 |     attr['last_accessed'] = fields[6]
367 |     attr['nodeid'] = fields[8]
368 |     attr['_offset_psec'] = attr['_offset_body'] + attr['body_length']
369 |     dump.seek(offset + attr['_offset_psec'], 0)
370 |     data = dump.read(ATTR_DIR_METADATA_PSEC_FORMAT.size)
371 |     fields = ATTR_DIR_METADATA_PSEC_FORMAT.unpack_from(data, 0)
372 |     attr['psec_length'] = fields[0]
373 |     attr['offset_first_pointer'] = fields[2]
374 |     attr['num_pointers'] = fields[3]
375 |     attr['offset_end_pointers_area?'] = fields[4]
376 |     if attr['num_pointers']:
377 |         pointers_format = Struct('<' + ('L' * attr['num_pointers']))
378 |         dump.seek(offset + attr['_offset_psec'] + attr['offset_first_pointer'], 0)
379 |         data = dump.read(pointers_format.size)
380 |         fields = pointers_format.unpack_from(data, 0)
381 |         attr['pointers'] = fields
382 |         attr['pointers_data'] = []
383 |     else:
384 |         attr['pointers'] = None
385 |         attr['pointers_data'] = None
386 |     attr['_offset_rec_area'] = attr['_offset_psec'] + attr['psec_length']
387 |     rec_offset = attr['_offset_rec_area']
388 |     for ptr in attr['pointers']:
389 |         ptr_addr = offset + attr['_offset_psec'] + ptr
390 |         attr['pointers_data'].append(read_directory_metadata_subattribute(dump, ptr_addr))
391 |     return attr
392 | 
393 | def _dump_directory_metadata_attribute(attr, prefix=''):
394 |     print('{}- size: {val:#x} ({val})'.format(prefix, val=attr['size']))
395 |     print('{}- offset type identifier: {val:#x} ({val})'.format(prefix,
396 |                                                                 val=attr['offset_identifier']))
397 |     print('{}- header remaining bytes: {val:#x} ({val})'.format(prefix,
398 |                                                                 val=attr['header_rem_data']))
399 |     print('{}- header length/offset metadata: {val:#x} ({val})'.format(prefix,
400 |                                                                        val=attr['header_length']))
401 |     print('{}- record remaining data: {val:#x} ({val})'.format(prefix,
402 |                                                                val=attr['record_rem_data']))
403 |     print('{}- type: {:#x} (directory metadata attribute)'.format(prefix, attr['type']))
404 |     print('{}- body: <{:#x}>'.format(prefix, attr['_absolute_offset'] + attr['_offset_body']))
405 |     print('{}  - length: {val:#x} ({val})'.format(prefix, val=attr['body_length']))
406 |     print('{}  - offset_first_timestamp: {val:#x} ({val})'.format(prefix,
407 |                                                                   val=attr['offset_first_timestamp']))
408 |     print('{}  - created: {} ({:#x})'.format(prefix, bytes2time(attr['created']), attr['created']))
409 |     print('{}  - modified: {} ({:#x})'.format(prefix, bytes2time(attr['modified']),
410 |                                                                       attr['modified']))
411 |     print('{}  - metadata modified: {} ({:#x})'.format(prefix,
412 |                                                        bytes2time(attr['metadata_modified']),
413 |                                                        attr['metadata_modified']))
414 |     print('{}  - last accessed: {} ({:#x})'.format(prefix, bytes2time(attr['last_accessed']),
415 |                                                    attr['last_accessed']))
416 |     print('{}  - node id: {:#x}'.format(prefix, attr['nodeid']))
417 |     print('{}- pointers section: <{:#x}>'.format(prefix,
418 |                                                  attr['_absolute_offset'] + attr['_offset_psec']))
419 |     print('{}  - length: {val:#x} ({val})'.format(prefix, val=attr['psec_length']))
420 |     print('{}  - offset to first pointer: {:#x}'.format(prefix, attr['offset_first_pointer']))
421 |     print('{}  - number of pointers: {}'.format(prefix, attr['num_pointers']))
422 |     print('{}  - offset end pointers area(?): {val:#x} ({val})'.format(prefix,
423 |         val=attr['offset_end_pointers_area?']))
424 |     print('{}  - pointers:'.format(prefix), end='')
425 |     if attr['pointers']:
426 |         for ptr in attr['pointers']:
427 |             print(' {:#x}'.format(ptr), end='')
428 |         print('')
429 |         print('{}- pointers data:'.format(prefix))
430 |         for ptr_i, ptr in enumerate(attr['pointers_data']):
431 |             print('{}  - pointer {}: <{:#x}>'.format(prefix, ptr_i, ptr['_absolute_offset']))
432 |             _dump_directory_metadata_subattribute(ptr, prefix + '    ')
433 |     else:
434 |         print(' None')
435 |         print('{}  - pointers data: None'.format(prefix))
436 | 
437 | DM_SUBATTR_SI30_HEADER_FORMAT = Struct('<LHHHHLLLL8s')
438 | 
439 | DM_SUBATTR_TYPE_SI30 = 0x00000090
440 | 
441 | def read_dm_si30_subattribute(dump, offset):
442 |     dump.seek(offset)
443 |     data = dump.read(DM_SUBATTR_SI30_HEADER_FORMAT.size)
444 |     fields = DM_SUBATTR_SI30_HEADER_FORMAT.unpack_from(data, 0)
445 |     attr = {'_absolute_offset': offset,
446 |             'size': fields[0],
447 |             'unknown0': fields[1],
448 |             'header_rem_data': fields[2],
449 |             'header_length': fields[4],
450 |             'record_rem_data': fields[5],
451 |             'record_rem_data_copy': fields[6],
452 |             'unknown1': fields[7],
453 |             'type': fields[8],
454 |             '$I30': fields[9],
455 |             '_structure_size': fields[0]}
456 |     dump.seek(offset + attr['header_length'])
457 |     # TODO: find what is at the end of the $I30
458 |     return attr
459 | 
460 | def _dump_dm_si30_subattribute(attr, prefix=''):
461 |     print('{}- size: {val:#x} ({val})'.format(prefix, val=attr['size']))
462 |     print('{}- unknown field 0: {val:#x} ({val})'.format(prefix, val=attr['unknown0']))
463 |     print('{}- header remaining bytes: {val:#x} ({val})'.format(prefix, val=attr['header_rem_data']))
464 |     print('{}- header length: {val:#x} ({val})'.format(prefix, val=attr['header_length']))
465 |     print('{}- record remaining data: {val:#x} ({val})'.format(prefix, val=attr['record_rem_data']))
466 |     print('{}- record remaining data (copy?): {val:#x} ({val})'.format(prefix,
467 |                                                                        val=attr['record_rem_data_copy']))
468 |     print('{}- unknown field 1: {val:#x} ({val})'.format(prefix, val=attr['unknown1']))
469 |     print('{}- type: {:#x} ($I30 type)'.format(prefix, attr['type']))
470 |     print('{}- $I30 string: {}'.format(prefix, attr['$I30'].decode('utf-16le')))
471 | 
472 | DM_SUBATTR_FOLDER_HEADER_FORMAT = Struct('<LHHHHHH')
473 | DM_SUBATTR_FOLDER_HEADER_2_FORMAT = Struct('<L')
474 | DM_SUBATTR_FOLDER_HEADER_FORMAT = Struct('<LHHHHLLLL8s')
475 | 
476 | DM_SUBATTR_FOLDER_BODY_FORMAT = Struct('<LLHHQQQQQQ24sH')
477 | 
478 | DM_SUBATTR_TYPE_FOLDER = 0x00000038
479 | 
480 | def read_dm_folder_subattribute(dump, offset):
481 |     dump.seek(offset)
482 |     data = dump.read(DM_SUBATTR_FOLDER_HEADER_FORMAT.size)
483 |     fields = DM_SUBATTR_FOLDER_HEADER_FORMAT.unpack_from(data, 0)
484 |     attr = {'_absolute_offset': offset,
485 |             'size': fields[0],
486 |             'unknown0': fields[1],
487 |             'header_rem_data': fields[2],
488 |             'header_length': fields[4],
489 |             'record_rem_data': fields[5],
490 |             'record_rem_data_copy': fields[6],
491 |             'unknown1': fields[7],
492 |             'type': fields[8],
493 |             '_structure_size': fields[0]}
494 |     dump.seek(offset + attr['header_length'])
495 |     data = dump.read(DM_SUBATTR_FOLDER_BODY_FORMAT.size)
496 |     fields = DM_SUBATTR_FOLDER_BODY_FORMAT.unpack_from(data, 0)
497 |     attr['parentid'] = fields[4]
498 |     attr['created'] = fields[6]
499 |     attr['modified'] = fields[7]
500 |     attr['metadata_modified'] = fields[8]
501 |     attr['last_accessed'] = fields[9]
502 |     attr['name_size'] = fields[11]
503 |     dump.seek(offset + attr['header_length'] + 0x5e)
504 |     data = dump.read(attr['name_size'] * 2)
505 |     attr['name'] = data
506 |     return attr
507 | 
508 | def _dump_dm_folder_subattribute(attr, prefix=''):
509 |     print('{}- size: {val:#x} ({val})'.format(prefix, val=attr['size']))
510 |     print('{}- unknown field 0: {val:#x} ({val})'.format(prefix, val=attr['unknown0']))
511 |     print('{}- header remaining bytes: {val:#x} ({val})'.format(prefix, val=attr['header_rem_data']))
512 |     print('{}- header length: {val:#x} ({val})'.format(prefix, val=attr['header_length']))
513 |     print('{}- record remaining data: {val:#x} ({val})'.format(prefix, val=attr['record_rem_data']))
514 |     print('{}- record remaining data (copy?): {val:#x} ({val})'.format(prefix,
515 |                                                                        val=attr['record_rem_data_copy']))
516 |     print('{}- unknown field 1: {val:#x} ({val})'.format(prefix, val=attr['unknown1']))
517 |     print('{}- type: {:#x} (directory metadata folder type)'.format(prefix, attr['type']))
518 |     print('{}- parent node id: {:#x}'.format(prefix, attr['parentid']))
519 |     print('{}- created: {} ({:#x})'.format(prefix, bytes2time(attr['created']), attr['created']))
520 |     print('{}- modified: {} ({:#x})'.format(prefix, bytes2time(attr['modified']), attr['modified']))
521 |     print('{}- metadata modified: {} ({:#x})'.format(prefix, bytes2time(attr['metadata_modified']),
522 |                                                      attr['metadata_modified']))
523 |     print('{}- last accessed: {} ({:#x})'.format(prefix, bytes2time(attr['last_accessed']),
524 |                                                  attr['last_accessed']))
525 |     print('{}- folder name size: {val:#x} ({val})'.format(prefix, val=attr['name_size']))
526 |     print('{}- folder name: {}'.format(prefix, attr['name'].decode('utf-16le')))
527 | 
528 | DM_SUBATTR_HEADER_FORMAT = Struct('<LHHHHLLLLL')
529 | 
530 | def read_directory_metadata_subattribute(dump, offset):
531 |     dump.seek(offset, 0)
532 |     data = dump.read(DM_SUBATTR_HEADER_FORMAT.size)
533 |     header = DM_SUBATTR_HEADER_FORMAT.unpack_from(data, 0)
534 |     if header[8] == DM_SUBATTR_TYPE_SI30:
535 |         attr = read_dm_si30_subattribute(dump, offset)
536 |     elif header[8] == DM_SUBATTR_TYPE_FOLDER:
537 |         attr = read_dm_folder_subattribute(dump, offset)
538 |     else:
539 |         attr = {'_absolute_offset': offset,
540 |                 'size': header[0],
541 |                 'unknown0': header[1],
542 |                 'header_rem_data': header[2],
543 |                 'header_length': header[4],
544 |                 'record_rem_data': header[5],
545 |                 'record_rem_data_copy': header[6],
546 |                 'unknown1': header[7],
547 |                 'type': header[8]}
548 |     return attr
549 | 
550 | def _dump_directory_metadata_subattribute(attr, prefix=''):
551 |     if attr['type'] == DM_SUBATTR_TYPE_SI30:
552 |         _dump_dm_si30_subattribute(attr, prefix)
553 |     elif attr['type'] == DM_SUBATTR_TYPE_FOLDER:
554 |         _dump_dm_folder_subattribute(attr, prefix)
555 |     else:
556 |         print('{}- size: {val:#x} ({val})'.format(prefix, val=attr['size']))
557 |         print('{}- unknown field 0: {val:#x} ({val})'.format(prefix, val=attr['unknown0']))
558 |         print('{}- header remaining bytes: {val:#x} ({val})'.format(prefix,
559 |                                                                     val=attr['header_rem_data']))
560 |         print('{}- header length: {val:#x} ({val})'.format(prefix, val=attr['header_length']))
561 |         print('{}- record remaining data: {val:#x} ({val})'.format(prefix,
562 |                                                                     val=attr['record_rem_data']))
563 |         print('{}- record remaining data (copy?): {val:#x} ({val})'.format(prefix,
564 |                                                                     val=attr['record_rem_data_copy']))
565 |         print('{}- unknown field 1: {val:#x} ({val})'.format(prefix, val=attr['unknown1']))
566 |         print('{}- type: {:#x} (unknown type)'.format(prefix, attr['type']))
567 | 
568 | ATTR_HEADER_FORMAT = Struct('<LHHHHHH')
569 | ATTR_HEADER_2_FORMAT = Struct('<L')
570 | 
571 | def read_attribute(dump, offset):
572 |     dump.seek(offset, 0)
573 |     data = dump.read(ATTR_HEADER_FORMAT.size)
574 |     header1 = ATTR_HEADER_FORMAT.unpack_from(data, 0)
575 |     dump.seek(offset + header1[1], 0)
576 |     data = dump.read(ATTR_HEADER_2_FORMAT.size)
577 |     header2 = ATTR_HEADER_2_FORMAT.unpack_from(data, 0)
578 |     if header2[0] == ATTR_TYPE_FILENAME:
579 |         attr = read_filename_attribute(dump, offset)
580 |     elif header2[0] == ATTR_TYPE_FILENAME_FOLDER:
581 |         attr = read_filename_folder_attribute(dump, offset)
582 |     elif header2[0] == ATTR_TYPE_CHILD:
583 |         attr = read_child_attribute(dump, offset)
584 |     elif header2[0] == ATTR_TYPE_DIRECTORY_METADATA:
585 |         attr = read_directory_metadata_attribute(dump, offset)
586 |     else:
587 |         attr = {'_absolute_offset': offset,
588 |                 'size': header1[0],
589 |                 'offset_identifier': header1[1],
590 |                 'header_rem_data': header1[2],
591 |                 'header_length': header1[4],
592 |                 'record_rem_data': header1[5],
593 |                 'type': header2[0]}
594 |     return attr
595 | 
596 | def dump_attribute(attr, prefix=''):
597 |     if attr['type'] == ATTR_TYPE_FILENAME:
598 |         _dump_filename_attribute(attr, prefix)
599 |     elif attr['type'] == ATTR_TYPE_FILENAME_FOLDER:
600 |         _dump_filename_folder_attribute(attr, prefix)
601 |     elif attr['type'] == ATTR_TYPE_CHILD:
602 |         _dump_child_attribute(attr, prefix)
603 |     elif attr['type'] == ATTR_TYPE_DIRECTORY_METADATA:
604 |         _dump_directory_metadata_attribute(attr, prefix)
605 |     else:
606 |         print('{}- size: {val:#x} ({val})'.format(prefix, val=attr['size']))
607 |         print('{}- offset type identifier: {val:#x} ({val})'.format(prefix,
608 |                                                                     val=attr['offset_identifier']))
609 |         print('{}- header remaining bytes: {val:#x} ({val})'.format(prefix,
610 |                                                                     val=attr['header_rem_data']))
611 |         print('{}- header length: {val:#x} ({val})'.format(prefix, val=attr['header_length']))
612 |         print('{}- record remaining data: {val:#x} ({val})'.format(prefix,
613 |                                                                     val=attr['record_rem_data']))
614 |         print('{}- type: {:#x} (unknown type)'.format(prefix, attr['type']))
615 | 


--------------------------------------------------------------------------------
/igor.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import cmd
  3 | import sys
  4 | import ast
  5 | 
  6 | import media.mbr as mbr
  7 | import media.gpt as gpt
  8 | import part.refs.vol as vol
  9 | import part.refs.entry_block as reb
 10 | import part.refs.tree_control as rtc
 11 | import part.refs.allocator as ralloc
 12 | import part.refs.object_tree as rot
 13 | import part.refs.attribute as rattr
 14 | from util.hexdump import hexdump
 15 | from util.filetree import filetree, dump_filetree
 16 | import util.carving as carving
 17 | from util.func_parser import FuncArgumentParser, FuncArgumentParserError, FuncArgumentParserHelp
 18 | from util.table import print_table
 19 | 
 20 | def _datarun_arg(s):
 21 |     try:
 22 |         ebid, n = map(lambda x: int(x, 0), s.split(','))
 23 |         return (ebid, n)
 24 |     except:
 25 |         raise argparse.ArgumentTypeError('Datarun entry must be <entryblock_identifier>,<number>.')
 26 | 
 27 | class PyReFSShell(cmd.Cmd):
 28 |     intro = None
 29 |     #     intro = '''Hello master! Welcome home.
 30 |     # I will try to help you to analyze your ReFS partition dumps.
 31 |     # Type 'help' or '?' to list commands.\n'''
 32 |     prompt = '> '
 33 |     rec_file = None
 34 |     dump_filename = None
 35 |     dump_file = None
 36 |     part = None
 37 |     parts = None
 38 |     blocks = None
 39 | 
 40 |     def __init__(self, **args):
 41 |         cmd.Cmd.__init__(self, args)
 42 |         self._init_func_args()
 43 | 
 44 |     def _init_func_args(self):
 45 |         file_argparser = FuncArgumentParser(
 46 |                 prog='file',
 47 |                 description='Load the provided dump file for analysis, and ' +
 48 |                             'automatically select the ReFS partition for ' +
 49 |                             'you. You can also initialize the list of ' +
 50 |                             'entryblocks of the partition.')
 51 |         file_argparser.add_argument('dump', action='store',
 52 |                 help='file to use as dump for the analysis')
 53 |         file_argparser.add_argument('-i', '--initiliaze-entryblocks', action='store_true',
 54 |                 default=False, dest='initialize_entryblocks',
 55 |                 help='find blocks in provided dump')
 56 |         file_argparser.add_argument('-f', '--files', action='store_true',
 57 |                 default=False,
 58 |                 help='find files in provided dump (only considered if -i defined)')
 59 |         file_argparser.add_argument('-F', '--folders', action='store_true',
 60 |                 default=False,
 61 |                 help='find folders in provided dump (only considered if -i defined)')
 62 |         vol_argparser = FuncArgumentParser(
 63 |                 prog='vol',
 64 |                 description='Dump the volume record information from the current ReFS partition.')
 65 |         part_argparser = FuncArgumentParser(
 66 |                 prog='part',
 67 |                 description='Show available partitions in the currently loaded dump if no parameter' +
 68 |                             ' is given, switch to the provided partition if any provided.')
 69 |         part_argparser.add_argument('partidx', action='store', type=int,
 70 |                 default=None, nargs='?',
 71 |                 help='Partition index of the partition to use for analysis')
 72 |         feb_argparser = FuncArgumentParser(
 73 |                 prog='find_entryblocks',
 74 |                 description='Find and show all the entryblocks in current partition.' +
 75 |                             ' If requested number of files and folders information will be also' +
 76 |                             ' collected.')
 77 |         feb_argparser.add_argument('-f', '--files', action='store_true',
 78 |                 default=False, dest='files',
 79 |                 help='Collect information on the number of files in the entryblocks.')
 80 |         feb_argparser.add_argument('-F', '--folders', action='store_true',
 81 |                 default=False, dest='folders',
 82 |                 help='Collect information on the number of folders in the entryblocks.')
 83 |         fp_argparser = FuncArgumentParser(
 84 |                 prog='find_pattern',
 85 |                 description='Find a data pattern in all the blocks of the current partition.' +
 86 |                             ' Special characters (including spaces, carriage return, etc.)' +
 87 |                             ' are not allowed in the pattern, think of escaping them.')
 88 |         fp_argparser.add_argument('pattern', action='store',
 89 |                 type=str,
 90 |                 help='Pattern to find in the current partition blocks.')
 91 |         lfiles_argparser = FuncArgumentParser(
 92 |                 prog='list_filenames',
 93 |                 description='List the found filenames from the list of ' +
 94 |                             'entryblocks with filenames.')
 95 |         lfolders_argparser = FuncArgumentParser(
 96 |                 prog='list_folders',
 97 |                 description='List the found filename folders from the list ' +
 98 |                             'of entryblocks with folders.')
 99 |         eb_argparser = FuncArgumentParser(
100 |                 prog='entryblock',
101 |                 description='Dump the provided entryblock identifier as ' +
102 |                 'EntryBlock.')
103 |         eb_argparser.add_argument('entryblock_identifier', action='store',
104 |                 type=lambda x: int(x, 0),
105 |                 help='entryblock identifier of the EntryBlock to dump')
106 |         tc_argparser = FuncArgumentParser(
107 |                 prog='tree_control',
108 |                 description='Parse the given entryblock identifier as a ' +
109 |                             'TreeControl. If no entryblock identifier is ' +
110 |                             'provided it uses 0x1e as entryblock identifier.')
111 |         tc_argparser.add_argument('entryblock_identifier', action='store',
112 |                 type=lambda x: int(x, 0), nargs='?', default=0x1e,
113 |                 help='entryblock identifier of the TreeControl to dump')
114 |         tce_argparser = FuncArgumentParser(
115 |                 prog='tree_control_extension',
116 |                 description='Parse the given entryblock identifier as a ' +
117 |                             'TreeControlExtension.')
118 |         tce_argparser.add_argument('entryblock_identifier', action='store',
119 |                 type=lambda x: int(x, 0),
120 |                 help='entryblock identifier of the TreeControlExtension to ' +
121 |                      'dump')
122 |         ot_argparser = FuncArgumentParser(
123 |                 prog='object_tree',
124 |                 description='Parse the given entryblock identifier as an ' +
125 |                             'ObjectTree.')
126 |         ot_argparser.add_argument('entryblock_identifier', action='store',
127 |                 type=lambda x: int(x, 0),
128 |                 help='entryblock identifier of the ObjectTree to ' +
129 |                      'dump')
130 |         alloc_argparser = FuncArgumentParser(
131 |                 prog='allocator',
132 |                 description='Parse the given entryblock identifier as an ' +
133 |                             'Allocator.')
134 |         alloc_argparser.add_argument('entryblock_identifier', action='store',
135 |                 type=lambda x: int(x, 0),
136 |                 help='entryblock identifier of the Allocator to dump')
137 |         attr_argparser = FuncArgumentParser(
138 |                 prog='attribute',
139 |                 description='Parse the given dump offset (in bytes) as an ' +
140 |                             'Attribute.')
141 |         attr_argparser.add_argument('dump_offset', action='store',
142 |                 type=lambda x: int(x, 0),
143 |                 help='dump offset (in bytes) of the Attribute to dump')
144 |         ds_argparser = FuncArgumentParser(
145 |                 prog='datastream',
146 |                 description='Extract data from dump from the given datarun. ' +
147 |                             'Dataruns can be extracted by exploring the ' +
148 |                             'EntryBlocks (command `entryblock`) or using ' +
149 |                             'the `list_dataruns` command.'
150 |                             )
151 |         ds_argparser.add_argument('output_filename', action='store',
152 |                 help='name of the generated output file')
153 |         ds_argparser.add_argument('output_size', action='store',
154 |                 type=lambda x: int(x, 0),
155 |                 help='total amount of data to extract')
156 |         ds_argparser.add_argument('dataruns', action='store',
157 |                 type=_datarun_arg, nargs='+',
158 |                 metavar='entryblock_identifier,number_of_blocks',
159 |                 help='datarun to follow for the data extraction')
160 |         ldr_argparser = FuncArgumentParser(
161 |                 prog='list_dataruns',
162 |                 description='Retrieve list of all the files dataruns.')
163 |         ft_argparser = FuncArgumentParser(
164 |                 prog='filetree',
165 |                 description='Extract the file tree structure from the given ' +
166 |                             'node (use node 0x600 by default).')
167 |         ft_argparser.add_argument('node_id', action='store',
168 |                 type=lambda x: int(x, 0), nargs='?', default=0x600,
169 |                 help='node identifier of the node to extract the file tree ' +
170 |                      'structure from')
171 |         hd_argparser = FuncArgumentParser(
172 |                 prog='hexdump',
173 |                 description='Hexdump the number of bytes at the provided ' +
174 |                             'offset.')
175 |         hd_argparser.add_argument('dump_offset', action='store',
176 |                 type=lambda x: int(x, 0),
177 |                 help='dump offset of the hexdump start')
178 |         hd_argparser.add_argument('size', action='store',
179 |                 type=lambda x: int(x, 0),
180 |                 help='number of bytes to dump')
181 |         hb_argparser = FuncArgumentParser(
182 |                 prog='hexblock',
183 |                 description='Hexdump the block with the provided ' +
184 |                             'entryblock identifier.')
185 |         hb_argparser.add_argument('entryblock_id', action='store',
186 |                 type=lambda x: int(x, 0),
187 |                 help='entryblock identifier to hexdump')
188 |         bye_argparser = FuncArgumentParser(
189 |                 prog='bye',
190 |                 description='Exit the program. Are you sure?')
191 |         record_argparser = FuncArgumentParser(
192 |                 prog='record',
193 |                 description='Save the following commands to selected file. ' +
194 |                             'Saved file can be used afterwards for replay ' +
195 |                             'with the \'playback\' command.')
196 |         record_argparser.add_argument('output_file', action='store',
197 |                 help='file onto which commands will be saved')
198 |         playback_argparser = FuncArgumentParser(
199 |                 prog='playback',
200 |                 description='Execute the sequence of commands defined in ' +
201 |                             'the input file.')
202 |         playback_argparser.add_argument('input_file', action='store',
203 |                 help='file from which commands to execute will be read')
204 |         self._args = {file_argparser.prog: file_argparser,
205 |                       vol_argparser.prog: vol_argparser,
206 |                       part_argparser.prog: part_argparser,
207 |                       feb_argparser.prog: feb_argparser,
208 |                       fp_argparser.prog: fp_argparser,
209 |                       lfiles_argparser.prog: lfiles_argparser,
210 |                       lfolders_argparser.prog: lfolders_argparser,
211 |                       eb_argparser.prog: eb_argparser,
212 |                       tc_argparser.prog: tc_argparser,
213 |                       tce_argparser.prog: tce_argparser,
214 |                       ot_argparser.prog: ot_argparser,
215 |                       alloc_argparser.prog: alloc_argparser,
216 |                       attr_argparser.prog: attr_argparser,
217 |                       ds_argparser.prog: ds_argparser,
218 |                       ldr_argparser.prog: ldr_argparser,
219 |                       ft_argparser.prog: ft_argparser,
220 |                       hd_argparser.prog: hd_argparser,
221 |                       hb_argparser.prog: hb_argparser,
222 |                       bye_argparser.prog: bye_argparser,
223 |                       record_argparser.prog: record_argparser,
224 |                       playback_argparser.prog: playback_argparser
225 |                      }
226 | 
227 |     def _check_func_args(self, func, arg):
228 |         parser = self._args[func]
229 |         out = {'return': True, 'args': None}
230 |         try:
231 |             args = parser.parse_args(arg.split())
232 |         except FuncArgumentParserHelp:
233 |             return out
234 |         except FuncArgumentParserError:
235 |             print('Master your command is badly formatted.')
236 |             return out
237 |         out = {'return': False, 'args': args}
238 |         return out
239 | 
240 |     def preloop(self):
241 |         print('''Hello master! Welcome home.
242 | I will try to help you to analyze your ReFS partition dumps.
243 | Type 'help' or '?' to list commands, and 'bye' or <ctrl-d> to exit.
244 | Type 'help <command>' for a short description of the command.
245 | Type '<command> -h' for the usage instructions of the command.''')
246 |         if not self.dump_filename:
247 |             print('''Master I just realized you haven't set a dump to analyze.
248 | Please use the 'file' command to set it.''')
249 | 
250 |     def do_file(self, arg):
251 |         '''Use the provided dump file for your ReFS analysis.
252 | I will try to automatically select the right partition for you.
253 | You can also initialize the list of entryblocks of the partition.'''
254 |         cargs = self._check_func_args('file', arg)
255 |         if cargs['return']:
256 |             return
257 |         args = cargs['args']
258 |         self.dump_filename = args.dump
259 |         print('Master I will try to follow your wishes by loading `{}`.'.format(self.dump_filename))
260 |         try:
261 |             self.dump_file = open(self.dump_filename, 'rb')
262 |         except:
263 |             print('I tried hard Master, but I couldn\'t open the requested file.')
264 |             print('Are you sure it exists?')
265 |             return
266 |         mbr_data = mbr.readMBR(self.dump_file)
267 |         gpt_part = None
268 |         for part_index, part in mbr_data:
269 |             if part['ptype'] == mbr.MBR_PARTTYPE_GPT:
270 |                 gpt_part = part
271 |                 break
272 |         if not gpt_part:
273 |             print('ARGH Master. I couldn\'t find a GPT volume in the dump' +
274 |                   ' you provided me. Please, try with another file.')
275 |             self.dump_filename = None
276 |             self.dump_file.close()
277 |             self.dump_file = None
278 |             return
279 |         gpt_data = gpt.readGPT(self.dump_file, gpt_part['start'])
280 |         parts = [ p for p in gpt_data['parts']
281 |                   if p['type'] == gpt.GUID_PART_TYPE_W_BASIC_DATA_PART
282 |                   and vol.is_refs_part(self.dump_file, p['first_lba']) ]
283 |         if not parts:
284 |             print('ARGH Master. I couldn\'t find a ReFS' +
285 |                   ' partition in the dump file you provided.' +
286 |                   ' Please, try with another file.',
287 |                   gpt.GUID_TRANSLATION[gpt.GUID_PART_TYPE_W_BASIC_DATA_PART])
288 |             self.dump_filename = None
289 |             self.dump_file.close()
290 |             self.dump_file = None
291 |             return
292 |         print(('Master, {} ReFS partitions were found.\n' +
293 |                'I will be using partition {}, but you can list the available' +
294 |                ' partitions and switch between them with the \'part\' command.').format(
295 |                    len(parts), parts[0]['index']))
296 |         for part in parts:
297 |             gpt.print_gpt_part(part)
298 |         self.parts = parts
299 |         self.part = parts[0]
300 |         self.prompt = '\n{} part {} - first lba: {} last lba: {}\n> '.format(
301 |                 self.dump_filename,
302 |                 self.part['index'],
303 |                 self.part['first_lba'],
304 |                 self.part['last_lba'])
305 |         if args.initialize_entryblocks:
306 |             f_args = ''
307 |             if args.files:
308 |                 f_args = '-f'
309 |             if args.folders:
310 |                 f_args = f_args + ' -F' if f_args != '' else '-F'
311 |             self.do_find_entryblocks(f_args)
312 |         return
313 | 
314 |     def do_vol(self, arg):
315 |         'Dump the volume record information from the current ReFS partition.'
316 |         cargs = self._check_func_args('vol', arg)
317 |         if cargs['return']:
318 |             return
319 |         args = cargs['args']
320 |         _vol = vol.fsstat(self.dump_file,
321 |                           self.part['first_lba'],
322 |                           self.part['last_lba'])
323 |         vol.dump_fsstat(_vol)
324 |         return
325 | 
326 |     def do_part(self, arg):
327 |         '''Show available partitions information ('part') and select from them
328 | ('part <partition index>').'''
329 |         cargs = self._check_func_args('part', arg)
330 |         if cargs['return']:
331 |             return
332 |         args = cargs['args']
333 |         if args.partidx is None:
334 |             print('Master you have {} partition{} available, here they are.'.format(
335 |                 len(self.parts), 's' if len(self.parts) > 1 else ''))
336 |             for p in self.parts:
337 |                 gpt.print_gpt_part(self.part)
338 |             return
339 |         if args.partidx not in [ p['index'] for p in self.parts ]:
340 |             print('Master I don\'t have the partition index {} you provided.'.format(
341 |                 args.partidx))
342 |             return
343 |         if args.partidx == self.part['index']:
344 |             print('Nothing done, as we are already using partition {}.'.format(
345 |                 args.partidx))
346 |             return
347 |         self.part = self.parts[args.partidx]
348 |         print('Switched to partition {}, enjoy Master.'.format(arg))
349 |         self.prompt = '\n{} part {} - first lba: {} last lba: {}\n> '.format(
350 |                 self.dump_filename,
351 |                 self.part['index'],
352 |                 self.part['first_lba'],
353 |                 self.part['last_lba'])
354 |         gpt.print_gpt_part(self.part)
355 | 
356 |     def do_find_entryblocks(self, arg):
357 |         '''Extract and show all the entryblocks used in the current partition.
358 | If requested also extract an estimation of the number of filenames and folders
359 | found per entryblock.'''
360 |         cargs = self._check_func_args('find_entryblocks', arg)
361 |         if cargs['return']:
362 |             return
363 |         args = cargs['args']
364 |         if not self.dump_file:
365 |             print('Master you have not defined a dump to analyze.')
366 |             print('Please provide a dump file.')
367 |             return
368 |         offset = self.part['first_lba']
369 |         end_offset = self.part['last_lba']
370 |         if self.blocks == None:
371 |             print(('Looking for blocks between lba {} and lba {}.' +
372 |                    ' This may take a while Master. A coffee?').format(
373 |                        offset,
374 |                        end_offset))
375 |             self.blocks = carving.find_blocks(self.dump_file, offset, end_offset)
376 |         print('Master I found {} blocks.'.format(len(self.blocks)))
377 |         if args.files:
378 |             blocks = carving.blocks_with_filename_attributes(self.dump_file, self.blocks)
379 |             print('Master I found {} blocks with the filename attribute.'.format(len(blocks)))
380 |         if args.folders:
381 |             blocks = carving.blocks_with_folder_attributes(self.dump_file, self.blocks)
382 |             print('Master I found {} blocks with the filename folder attribute.'.format(len(blocks)))
383 |         carving.print_blocks(self.blocks)
384 | 
385 |     def do_find_pattern(self, arg):
386 |         '''Find a data pattern in all the blocks of the current partition.
387 | Special characters (including spaces, carriage return, etc.) are not allowed in
388 | the pattern, think of escaping them.'''
389 |         cargs = self._check_func_args('find_pattern', arg)
390 |         if cargs['return']:
391 |             return
392 |         args = cargs['args']
393 |         _pattern = ast.literal_eval('"{}"'.format(args.pattern))
394 |         offset = self.part['first_lba']
395 |         end_offset = self.part['last_lba']
396 |         pattern = [ ord(x) for x in _pattern ]
397 |         print('Master I will be analyzing your pattern \'{}\' ({}) but it may take a while.'.format(
398 |             args.pattern, [ '{:#x}'.format(x) for x in pattern ]))
399 |         print('Do you want a cup of tea?')
400 |         blocks = carving.find_data_blocks_with_pattern(self.dump_file, pattern, offset, end_offset)
401 |         print('Master I found {} blocks with your wiseful pattern.'.format(len(blocks)))
402 |         # print table of found blocks
403 |         columns = [ {'key': 'addr', 'header': 'Address', 'align': '<', 'format': '#x'},
404 |                     {'key': 'lba_offset', 'header': 'LBA offset', 'align': '<', 'format': '#x'},
405 |                     {'key': 'block_offset', 'header': 'Block offset', 'align': '<', 'format': '#x'} ]
406 |         if blocks:
407 |             print_table(columns, blocks)
408 | 
409 |     def do_list_filenames(self, arg):
410 |         'List the found filenames from the list of entryblocks with filenames.'
411 |         cargs = self._check_func_args('list_filenames', arg)
412 |         if cargs['return']:
413 |             return
414 |         args = cargs['args']
415 |         if not self.blocks:
416 |             print('Master, first you need to look for the blocks and blocks with filenames.')
417 |             print('Please Master, use \'find_entryblocks -f\' first.')
418 |             return
419 |         listed = 0
420 |         for block in self.blocks:
421 |             if block['fnas']:
422 |                 for fid in block['fnas']:
423 |                     attr = rattr.read_attribute(self.dump_file, fid)
424 |                     try:
425 |                         filename = attr['filename'].decode('utf-16le')
426 |                     except:
427 |                         filename = attr['filename']
428 |                     listed = listed + 1
429 |                     print('{:#010x} {:#06x} {:#06x} {:#06x} {:#010x} {}'.format(
430 |                         block['offset'], block['entryblock'],
431 |                         block['nodeid'], block['childid'], attr['_absolute_offset'], filename))
432 |         print('Master I listed {} filenames.'.format(listed))
433 |         if not listed:
434 |             print('Master are you sure you performed \'find_entryblocks -F\' before?')
435 | 
436 |     def do_list_folders(self, arg):
437 |         'List the found filename folders from the list of entryblocks with folders.'
438 |         cargs = self._check_func_args('list_folders', arg)
439 |         if cargs['return']:
440 |             return
441 |         args = cargs['args']
442 |         if not self.blocks:
443 |             print('Master, first you need to look for the blocks and blocks with folders.')
444 |             print('Please Master, use \'find_entryblocks -F\' first.')
445 |             return
446 |         listed = 0
447 |         for block in self.blocks:
448 |             if block['folderids']:
449 |                 for fid in block['folderids']:
450 |                     attr = rattr.read_attribute(self.dump_file, fid)
451 |                     try:
452 |                         foldername = attr['foldername'].decode('utf-16le')
453 |                     except:
454 |                         foldername = attr['foldername']
455 |                     listed = listed + 1
456 |                     print('{:#010x} {:#06x} {:#06x} {:#06x} {:#010x} {}'.format(
457 |                         block['offset'], block['entryblock'],
458 |                         block['nodeid'], block['childid'], attr['_absolute_offset'], foldername))
459 |         print('Master I listed {} folders.'.format(listed))
460 |         if not listed:
461 |             print('Master are you sure you performed \'find_entryblocks -F\' before?')
462 | 
463 |     def do_hexdump(self, arg):
464 |         'Hexdump the number of bytes at the provided offset.'
465 |         cargs = self._check_func_args('hexdump', arg)
466 |         if cargs['return']:
467 |             return
468 |         args = cargs['args']
469 |         offset = args.dump_offset
470 |         size = args.size
471 |         self.dump_file.seek(offset, 0)
472 |         data = self.dump_file.read(size)
473 |         hexdump(data, offset)
474 | 
475 |     def do_hexblock(self, arg):
476 |         'Hexdump the block with the provided entryblock identifier.'
477 |         cargs = self._check_func_args('hexblock', arg)
478 |         if cargs['return']:
479 |             return
480 |         args = cargs['args']
481 |         ebid = args.entryblock_id
482 |         if not self.blocks:
483 |             print('Master, first you need to look for the blocks.')
484 |             print('Please Master use \'find_entryblocks\' first or use \'hexdump\'.')
485 |             return
486 |         blks = [ b for b in self.blocks if b['entryblock'] == ebid ]
487 |         if not blks:
488 |             print('Master, are you sure such an entryblock exist?')
489 |             return
490 |         blk = blks[0]
491 |         self.dump_file.seek(blk['offset'], 0)
492 |         data = self.dump_file.read(16 * 1024)
493 |         hexdump(data, blk['offset'])
494 | 
495 |     def do_entryblock(self, arg):
496 |         'Dump the provided entryblock identifier as entryblock.'
497 |         cargs = self._check_func_args('entryblock', arg)
498 |         if cargs['return']:
499 |             return
500 |         args = cargs['args']
501 |         ebid = args.entryblock_identifier
502 |         if not self.blocks:
503 |             print('Master, first you need to look for the blocks.')
504 |             print('Please Master use \'find_entryblocks\' first.')
505 |             return
506 |         blocks = [ b for b in self.blocks if b['entryblock'] == ebid ]
507 |         if len(blocks) != 1:
508 |             print('Master, I couldn\'t find the block you asked for.')
509 |             return
510 |         block = blocks[0]
511 |         eb = reb.read_entryblock(self.dump_file, block['offset'])
512 |         reb.dump_entryblock(eb)
513 | 
514 |     def do_tree_control(self, arg):
515 |         '''Parse the given entryblock identifier as a TreeControl.
516 | If no entryblock identifier is provided it uses 0x1e as entryblock
517 | identifier.'''
518 |         cargs = self._check_func_args('tree_control', arg)
519 |         if cargs['return']:
520 |             return
521 |         args = cargs['args']
522 |         ebid = args.entryblock_identifier
523 |         if not self.blocks:
524 |             print('Master, first you need to look for the blocks.')
525 |             print('Please Master use \'find_entryblocks\' first.')
526 |             return
527 |         blocks = [ b for b in self.blocks if b['entryblock'] == ebid ]
528 |         if len(blocks) != 1:
529 |             print('Master, I couldn\'t find the block you asked for.')
530 |             return
531 |         block = blocks[0]
532 |         tc = rtc.read_tree_control(self.dump_file, block['offset'])
533 |         rtc.dump_tree_control(tc)
534 | 
535 |     def do_tree_control_extension(self, arg):
536 |         'Parse the given entryblock identifier as a TreeControlExtension.'
537 |         cargs = self._check_func_args('tree_control_extension', arg)
538 |         if cargs['return']:
539 |             return
540 |         args = cargs['args']
541 |         ebid = args.entryblock_identifier
542 |         if not self.blocks:
543 |             print('Master, first you need to look for the blocks.')
544 |             print('Please Master use \'find_entryblocks\' first.')
545 |             return
546 |         blocks = [ b for b in self.blocks if b['entryblock'] == ebid ]
547 |         if len(blocks) != 1:
548 |             print('Master, I couldn\'t find the block you asked for.')
549 |             return
550 |         block = blocks[0]
551 |         tce = rtc.read_tree_control_ext(self.dump_file, block['offset'])
552 |         rtc.dump_tree_control_ext(tce)
553 | 
554 |     def do_object_tree(self, arg):
555 |         'Parse the given entryblock identifier as an ObjectTree.'
556 |         cargs = self._check_func_args('object_tree', arg)
557 |         if cargs['return']:
558 |             return
559 |         args = cargs['args']
560 |         ebid = args.entryblock_identifier
561 |         if not self.blocks:
562 |             print('Master, first you need to look for the blocks.')
563 |             print('Please Master use \'find_entryblocks\' first.')
564 |             return
565 |         blocks = [ b for b in self.blocks if b['entryblock'] == ebid ]
566 |         if len(blocks) != 1:
567 |             print('Master, I couldn\'t find the block you asked for.')
568 |             return
569 |         block = blocks[0]
570 |         ot = rot.read_object_tree(self.dump_file, block['offset'])
571 |         rot.dump_object_tree(ot)
572 | 
573 |     def do_allocator(self, arg):
574 |         'Parse the given entryblock identifier as an Allocator.'
575 |         cargs = self._check_func_args('allocator', arg)
576 |         if cargs['return']:
577 |             return
578 |         args = cargs['args']
579 |         ebid = args.entryblock_identifier
580 |         if not self.blocks:
581 |             print('Master, first you need to look for the blocks.')
582 |             print('Please Master use \'find_entryblocks\' first.')
583 |             return
584 |         blocks = [ b for b in self.blocks if b['entryblock'] == ebid ]
585 |         if len(blocks) != 1:
586 |             print('Master, I couldn\'t find the block you asked for.')
587 |             return
588 |         block = blocks[0]
589 |         al = ralloc.read_allocator(self.dump_file, block['offset'])
590 |         ralloc.dump_allocator(al)
591 | 
592 |     def do_attribute(self, arg):
593 |         'Parse the given dump offset (in bytes) as an Attribute.'
594 |         cargs = self._check_func_args('attribute', arg)
595 |         if cargs['return']:
596 |             return
597 |         args = cargs['args']
598 |         offset = args.dump_offset
599 |         attr = rattr.read_attribute(self.dump_file, offset)
600 |         rattr.dump_attribute(attr)
601 | 
602 |     def do_datastream(self, arg):
603 |         '''Extract data from dump from the given datarun.
604 | Dataruns can be extracted by exploring the EntryBlocks (command `entryblock`)
605 | or using the `list_dataruns` command.'''
606 |         cargs = self._check_func_args('datastream', arg)
607 |         if cargs['return']:
608 |             return
609 |         args = cargs['args']
610 |         ofn = args.output_filename
611 |         size = args.output_size
612 |         dataruns = args.dataruns
613 |         of = open(ofn, 'wb')
614 |         for offset, length in dataruns:
615 |             offset = (offset * 1024 * 16) + (self.part['first_lba'] * 512)
616 |             length = length * 1024 * 16
617 |             if length > size:
618 |                 length = size
619 |             size = size - length
620 |             self.dump_file.seek(offset, 0)
621 |             data = self.dump_file.read(length)
622 |             of.write(data)
623 |         of.close()
624 | 
625 |     def do_list_dataruns(self, arg):
626 |         'Retrieve list of all the files dataruns.'
627 |         cargs = self._check_func_args('list_dataruns', arg)
628 |         if cargs['return']:
629 |             return
630 |         files = 0
631 |         drs = 0
632 |         for block in self.blocks:
633 |             if block['fnas']:
634 |                 for fid in block['fnas']:
635 |                     files = files + 1
636 |                     attr = rattr.read_attribute(self.dump_file, fid)
637 |                     dataruns = []
638 |                     if attr['datarun'] and attr['datarun']['pointers_data']:
639 |                         for ptr in attr['datarun']['pointers_data']:
640 |                             if ptr['pointers_data']:
641 |                                 datarun = [ (x['blockid'], x['num_blocks'])
642 |                                             for x in ptr['pointers_data'] ]
643 |                                 dataruns.append((ptr['logical_size'], datarun))
644 |                                 drs = drs + 1
645 |                     try:
646 |                         filename = attr['filename'].decode('utf-16le')
647 |                     except:
648 |                         filename = attr['filename']
649 |                     print('{:#010x} {:#06x} {:#06x} {:6} {}'.format(
650 |                         block['offset'], block['entryblock'],
651 |                         block['nodeid'], block['counter'], filename))
652 |                     for length, datarun in dataruns:
653 |                         print('  size: {:#x} datarun: '.format(length), end='')
654 |                         for run in datarun:
655 |                             blockid,num = run
656 |                             print(' {:#x},{}'.format(blockid,num), end='')
657 |                         print('')
658 |         if files:
659 |             print('Master I listed {} data runs from {} files.'.format(drs, files))
660 |         else:
661 |             print('Master I could not find any file, did you already execute \'find_entryblocks -f\'?')
662 | 
663 |     def do_filetree(self, arg):
664 |         'Extract the file tree structure from the given node (use node 0x600 by default).'
665 |         cargs = self._check_func_args('filetree', arg)
666 |         if cargs['return']:
667 |             return
668 |         args = cargs['args']
669 |         nodeid = args.node_id
670 |         block_list = None
671 |         if self.blocks:
672 |             block_list = self.blocks
673 |         offset = self.part['first_lba']
674 |         end_offset = self.part['last_lba']
675 |         tree = filetree(self.dump_file, offset, end_offset, nodeid, block_list)
676 |         dump_filetree(tree)
677 | 
678 |     def do_bye(self, arg):
679 |         'Exit the program. Are you sure?'
680 |         cargs = self._check_func_args('bye', arg)
681 |         if cargs['return']:
682 |             return
683 |         print('''It's been a great to serve you master.
684 | Bye!!!!''')
685 |         self.close()
686 |         return True
687 | 
688 |     def do_EOF(self, arg):
689 |         'Exit the program. Really are you sure?'
690 |         print('\nI would have liked a \'bye\' Master, but I follow your orders.')
691 |         return self.do_bye(arg)
692 | 
693 |     # ----- record and playback -----
694 |     def do_record(self, arg):
695 |         ''''Save the following commands to selected file.
696 | Saved file can be used afterwards for replay with the \'playback\' command.'''
697 |         cargs = self._check_func_args('record', arg)
698 |         if cargs['return']:
699 |             return
700 |         args = cargs['args']
701 |         self.close()
702 |         try:
703 |             self.rec_file = open(args.output_file, 'w')
704 |         except:
705 |             print('Master I couldn\'t open file to save commands.')
706 |             self.rec_file = None
707 | 
708 |     def do_playback(self, arg):
709 |         'Execute the sequence of commands defined in the input file.'
710 |         cargs = self._check_func_args('playback', arg)
711 |         if cargs['return']:
712 |             return
713 |         args = cargs['args']
714 |         self.close()
715 |         try:
716 |             with open(args.input_file) as f:
717 |                 self.cmdqueue.extend(f.read().splitlines())
718 |         except:
719 |             print('Master I couldn\'t open file to read commands to execute.')
720 | 
721 |     def precmd(self, line):
722 |         if self.rec_file and 'playback' not in line:
723 |             print(line, file=self.rec_file)
724 |         return line
725 | 
726 |     def close(self):
727 |         if self.rec_file:
728 |             self.rec_file.close()
729 |             self.rec_file = None
730 | 
731 | if __name__ == '__main__':
732 |     parser = argparse.ArgumentParser(description='ReFS carving on provided dump.')
733 |     parser.add_argument('-d', '--dump', action='store', required=False, type=str,
734 |                         help='ReFS dump to analyze')
735 |     args = parser.parse_args()
736 | 
737 |     if args.dump:
738 |         rshell = PyReFSShell()
739 |         rshell.onecmd('file ' + args.dump)
740 |         rshell.cmdloop()
741 |     else:
742 |         PyReFSShell().cmdloop()
743 | 


--------------------------------------------------------------------------------