├── fishy ├── __init__.py ├── APFS │ ├── __init__.py │ └── APFS_filesystem │ │ ├── __init__.py │ │ ├── APFS_Detector.py │ │ ├── Object_Header.py │ │ ├── APFS_Parser.py │ │ ├── APFS.py │ │ ├── Object_Map.py │ │ ├── Container_Superblock.py │ │ ├── InodeTable.py │ │ ├── Volume_Superblock.py │ │ └── Checkpoints.py ├── ext4 │ ├── __init__.py │ ├── tests │ │ ├── __init__.py │ │ └── test_superblock.py │ ├── ext4_filesystem │ │ ├── __init__.py │ │ ├── ext4_detector.py │ │ ├── inode_table.py │ │ ├── EXT4.py │ │ ├── parser.py │ │ ├── gdt.py │ │ └── inode.py │ ├── osd2.py │ └── obso_faddr.py ├── ntfs │ ├── __init__.py │ ├── ntfs_filesystem │ │ ├── __init__.py │ │ ├── record_reference.py │ │ ├── record_header.py │ │ ├── bootsector.py │ │ ├── attribute_header.py │ │ ├── t_ntfs.py │ │ └── attributes.py │ └── ntfs_detector.py ├── wrapper │ ├── __init__.py │ ├── write_gen.py │ ├── inode_padding.py │ ├── xfield_padding.py │ ├── timestamp_hiding.py │ ├── osd2.py │ ├── obso_faddr.py │ ├── reserved_gdt_blocks.py │ ├── mft_slack.py │ ├── bad_cluster.py │ ├── cluster_allocation.py │ └── superblock_slack.py ├── fat │ ├── __init__.py │ └── fat_filesystem │ │ ├── __init__.py │ │ ├── fs_info.py │ │ ├── fat_wrapper.py │ │ ├── fat_16.py │ │ ├── fat_detector.py │ │ ├── bootsector.py │ │ ├── fattools.py │ │ ├── fat_entry.py │ │ ├── fat_32.py │ │ └── fat_12.py └── filesystem_detector.py ├── utils ├── fs-files │ ├── another │ ├── testfile.txt │ ├── areallylongfilenamethatiwanttoreadcorrectly.txt │ └── onedirectory │ │ ├── afileinadirectory.txt │ │ ├── nested_directory │ │ └── royce.txt │ │ └── areallylongfilenamethatialsowanttoreadassoonaspossible.txt ├── mount-fs │ └── placeholder.txt ├── fs-files-stable1 │ ├── another │ ├── testfile.txt │ ├── areallylongfilenamethatiwanttoreadcorrectly.txt │ └── onedirectory │ │ ├── afileinadirectory.txt │ │ ├── nested_directory │ │ └── royce.txt │ │ └── areallylongfilenamethatialsowanttoreadassoonaspossible.txt └── fs-files-stable2 │ ├── another │ ├── testfile.txt │ ├── areallylongfilenamethatiwanttoreadcorrectly.txt │ ├── onedirectory │ ├── afileinadirectory.txt │ ├── nested_directory │ │ └── royce.txt │ └── areallylongfilenamethatialsowanttoreadassoonaspossible.txt │ ├── big_mft_file.txt │ └── other_file.txt ├── requirements.txt ├── doc ├── source │ ├── _static │ │ ├── mft_entry.png │ │ ├── ext4_extents.png │ │ ├── ext4_structure.png │ │ ├── fileslack_image.png │ │ └── module_flowchart.png │ ├── 07_appendix.rst │ ├── index.rst │ ├── toc.rst │ ├── 01_options.rst │ ├── resources │ │ └── module_flowchart.dot │ ├── 07_2_responsibilities.rst │ ├── 05_future_work.rst │ ├── 07_1_create_testfs.rst │ ├── 01_introduction.rst │ ├── 03_reference_api.rst │ ├── 02_module_overview.rst │ └── conf.py ├── README.md ├── fat-cluster-allocation.dot ├── Makefile ├── fat-fileslack.dot └── make.bat ├── setup.cfg ├── tests ├── test_doctest.py ├── test_filesystem_detector.py ├── test_fat_detector.py ├── test_ntfs_cluster_allocator.py ├── conftest.py ├── test_ntfs.py ├── test_fat_bad_cluster.py ├── test_ntfs_file_slack.py └── test_ntfs_mft_slack.py ├── setup.py ├── LICENSE ├── .gitignore └── TODO.md /fishy/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fishy/APFS/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fishy/ext4/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fishy/ntfs/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fishy/ext4/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fishy/wrapper/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/fs-files/another: -------------------------------------------------------------------------------- 1 | 222 2 | -------------------------------------------------------------------------------- /utils/mount-fs/placeholder.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fishy/APFS/APFS_filesystem/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fishy/ext4/ext4_filesystem/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fishy/ntfs/ntfs_filesystem/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/fs-files-stable1/another: -------------------------------------------------------------------------------- 1 | 222 2 | -------------------------------------------------------------------------------- /utils/fs-files-stable2/another: -------------------------------------------------------------------------------- 1 | 222 2 | -------------------------------------------------------------------------------- /utils/fs-files/testfile.txt: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /utils/fs-files-stable1/testfile.txt: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /utils/fs-files-stable2/testfile.txt: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /fishy/fat/__init__.py: -------------------------------------------------------------------------------- 1 | from . import file_slack 2 | -------------------------------------------------------------------------------- /utils/fs-files/areallylongfilenamethatiwanttoreadcorrectly.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/fs-files/onedirectory/afileinadirectory.txt: -------------------------------------------------------------------------------- 1 | eine datei 2 | -------------------------------------------------------------------------------- /utils/fs-files/onedirectory/nested_directory/royce.txt: -------------------------------------------------------------------------------- 1 | Test123 2 | -------------------------------------------------------------------------------- /utils/fs-files-stable1/areallylongfilenamethatiwanttoreadcorrectly.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/fs-files-stable1/onedirectory/afileinadirectory.txt: -------------------------------------------------------------------------------- 1 | eine datei 2 | -------------------------------------------------------------------------------- /utils/fs-files-stable2/areallylongfilenamethatiwanttoreadcorrectly.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/fs-files-stable2/onedirectory/afileinadirectory.txt: -------------------------------------------------------------------------------- 1 | eine datei 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | argparse 2 | construct < 2.9 3 | pytsk3 4 | simple-crypt 5 | -------------------------------------------------------------------------------- /utils/fs-files-stable1/onedirectory/nested_directory/royce.txt: -------------------------------------------------------------------------------- 1 | Test123 2 | -------------------------------------------------------------------------------- /utils/fs-files-stable2/onedirectory/nested_directory/royce.txt: -------------------------------------------------------------------------------- 1 | Test123 2 | -------------------------------------------------------------------------------- /utils/fs-files/onedirectory/areallylongfilenamethatialsowanttoreadassoonaspossible.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fishy/fat/fat_filesystem/__init__.py: -------------------------------------------------------------------------------- 1 | from . import fat, fattools, fat_detector, fat_wrapper 2 | -------------------------------------------------------------------------------- /utils/fs-files-stable1/onedirectory/areallylongfilenamethatialsowanttoreadassoonaspossible.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/fs-files-stable2/onedirectory/areallylongfilenamethatialsowanttoreadassoonaspossible.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /doc/source/_static/mft_entry.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dasec/fishy/HEAD/doc/source/_static/mft_entry.png -------------------------------------------------------------------------------- /doc/source/_static/ext4_extents.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dasec/fishy/HEAD/doc/source/_static/ext4_extents.png -------------------------------------------------------------------------------- /doc/source/_static/ext4_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dasec/fishy/HEAD/doc/source/_static/ext4_structure.png -------------------------------------------------------------------------------- /doc/source/_static/fileslack_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dasec/fishy/HEAD/doc/source/_static/fileslack_image.png -------------------------------------------------------------------------------- /doc/source/_static/module_flowchart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dasec/fishy/HEAD/doc/source/_static/module_flowchart.png -------------------------------------------------------------------------------- /doc/source/07_appendix.rst: -------------------------------------------------------------------------------- 1 | Appendix 2 | ======== 3 | 4 | .. include:: 07_1_create_testfs.rst 5 | 6 | .. include:: 07_2_responsibilities.rst 7 | -------------------------------------------------------------------------------- /doc/source/index.rst: -------------------------------------------------------------------------------- 1 | fishy - filesystem based data hiding techniques 2 | =============================================== 3 | 4 | .. include:: 01_description.rst 5 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | doc=build_sphinx 4 | 5 | [build_sphinx] 6 | project = 'fishy' 7 | version = 0.1 8 | release = 0.1.0 9 | source-dir = doc/source 10 | build-dir = doc/build 11 | -------------------------------------------------------------------------------- /fishy/ntfs/ntfs_filesystem/record_reference.py: -------------------------------------------------------------------------------- 1 | """ 2 | Declaration of a record reference 3 | """ 4 | 5 | from construct import Struct, Int16ul, Int32ul 6 | 7 | RECORD_REFERENCE = Struct( 8 | "segment_number_low_part" / Int32ul, 9 | "segment_number_high_part" / Int16ul, 10 | "sequence_number" / Int16ul 11 | ) 12 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | As long as we don't have any documentation strategy this doc folder might serve 2 | as a place for leaving random notes alone... 3 | 4 | 5 | 27.05.2019: As of right now, this documentation only describes the FAT, NTFS and ext4 fishy modules. 6 | While this documentation is being expanded, the github wiki (https://github.com/dasec/fishy/wiki) can be read for updated information. -------------------------------------------------------------------------------- /utils/fs-files-stable2/big_mft_file.txt: -------------------------------------------------------------------------------- 1 | 1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111 2 | -------------------------------------------------------------------------------- /fishy/APFS/APFS_filesystem/APFS_Detector.py: -------------------------------------------------------------------------------- 1 | # Calls function returning magic bytes from Container 2 | 3 | # basic syntax: if [magic bytes] == [predetermined magic bytes] return true 4 | 5 | def is_apfs(fs_stream): 6 | fs_stream.seek(0) 7 | offset = fs_stream.tell() 8 | fs_stream.seek(offset + 32) 9 | fs_type = fs_stream.read(4) 10 | fs_stream.seek(offset) 11 | if fs_type == b'NXSB': 12 | return True 13 | 14 | elif object_type != b'NXSB': 15 | return False 16 | 17 | 18 | -------------------------------------------------------------------------------- /doc/fat-cluster-allocation.dot: -------------------------------------------------------------------------------- 1 | digraph { 2 | node[shape=hexagon] 3 | begin, end 4 | node[shape=diamond] 5 | "data left to write" 6 | "FAT32?" 7 | node[shape=box] 8 | "begin" -> "get filepath" -> "traverse FAT directories" -> "get last cluster_id of file" -> "find free data cluster and extend cluster chain" -> "write into allocated cluster" -> "data left to write" 9 | "data left to write" -> "find free data cluster and extend cluster chain"[label="[yes]"] 10 | "data left to write" -> "FAT32?" -> "end"[label="[no]"] 11 | "FAT32?" -> "Update FS INFO sector" -> "end" 12 | } 13 | -------------------------------------------------------------------------------- /fishy/fat/fat_filesystem/fs_info.py: -------------------------------------------------------------------------------- 1 | """ 2 | FAT32 Filesystem information sector. 3 | This structure is part of the reserved sectors area. 4 | """ 5 | from construct import Struct, Padding, Bytes, Int32ul 6 | 7 | # Filesystem information sector for FAT32 8 | FS_INFORMATION_SECTOR = Struct( 9 | "fsinfo_sector_signature" / Bytes(4), 10 | Padding(480), # reserved 11 | "fsinfo_sector_signature" / Bytes(4), 12 | "free_data_cluster_count" / Int32ul, 13 | "last_allocated_data_cluster" / Int32ul, 14 | Padding(12), # reserved 15 | "fsinfo_sector_signature" / Bytes(4), 16 | ) 17 | -------------------------------------------------------------------------------- /doc/source/toc.rst: -------------------------------------------------------------------------------- 1 | .. fishy documentation master file, created by 2 | sphinx-quickstart on Tue Dec 5 12:49:58 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | fishy - filesystem based data hiding techniques 7 | =============================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | 01_introduction 14 | 01_getting_started 15 | 02_filesystem_datastructures 16 | 02_module_overview 17 | 03_reference_api 18 | 04_evaluation 19 | 05_future_work 20 | -------------------------------------------------------------------------------- /tests/test_doctest.py: -------------------------------------------------------------------------------- 1 | """ 2 | This test module runs doctests against selected modules 3 | """ 4 | 5 | import doctest 6 | import unittest 7 | import xmlrunner 8 | import fishy.metadata 9 | 10 | def load_tests(loader, tests, ignore): 11 | # add doctests for metadata module 12 | tests.addTests(doctest.DocTestSuite(fishy.metadata)) 13 | return tests 14 | 15 | if __name__ == '__main__': 16 | with open('doctests.xml', 'wb') as output: 17 | unittest.main( 18 | testRunner=xmlrunner.XMLTestRunner(output=output), 19 | failfast=False, buffer=False, catchbreak=False) 20 | -------------------------------------------------------------------------------- /doc/source/01_options.rst: -------------------------------------------------------------------------------- 1 | Command Line Option Reference 2 | ============================= 3 | 4 | .. argparse:: 5 | :module: fishy.cli 6 | :func: build_parser 7 | :prog: fishy 8 | 9 | fileslack 10 | Implemented for following filesystems: FAT, NTFS 11 | 12 | Warning: When using the -d option in combination with a directory, 13 | make sure that no other filename in this directory is specified. Otherwise 14 | this would lead to multiple writes into the same slack space of this file 15 | and result in data loss. 16 | 17 | 18 | addcluster 19 | Implemented for following filesystems: FAT 20 | -------------------------------------------------------------------------------- /fishy/ext4/ext4_filesystem/ext4_detector.py: -------------------------------------------------------------------------------- 1 | class UnsupportedFilesystemError(Exception): 2 | pass 3 | 4 | 5 | def is_ext4(stream): 6 | """ 7 | checks if a given stream is of type ext4 or not 8 | :param stream: stream of filesystem 9 | :return: bool, True if it is a ext4 filesystem 10 | False if it is not a ext4 filesystem 11 | """ 12 | # get start position to reset after check 13 | offset = stream.tell() 14 | stream.seek(offset + 1024 + 56) 15 | fs_type = stream.read(2) 16 | stream.seek(offset) 17 | if fs_type.hex() == '53ef': 18 | return True 19 | else: 20 | return False -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from setuptools import setup, find_packages, Distribution 4 | 5 | 6 | setup( 7 | name='fishy', 8 | version='0.2', 9 | packages=find_packages(), 10 | entry_points={ 11 | 'console_scripts': [ 12 | 'fishy = fishy.cli:main', 13 | ], 14 | }, 15 | install_requires=[ 16 | "argparse", 17 | "construct < 2.9", 18 | "pytsk3", 19 | "simple-crypt", 20 | ], 21 | setup_requires=['pytest-runner'], 22 | tests_require=['pytest'], 23 | extras_require={ 24 | 'build_sphinx': ['sphinx', 'sphinx-argparse'], 25 | }, 26 | ) 27 | -------------------------------------------------------------------------------- /fishy/ext4/tests/test_superblock.py: -------------------------------------------------------------------------------- 1 | import pprint 2 | from unittest import TestCase 3 | 4 | from fishy.ext4.ext4_filesystem.superblock import Superblock 5 | 6 | pp = pprint.PrettyPrinter(indent=4) 7 | 8 | 9 | class TestSuperblock(TestCase): 10 | 11 | image = "ext4_example_wc.img" 12 | 13 | def test_parse_superblock(self): 14 | sb = Superblock(self.image) 15 | 16 | pp.pprint(sb.data) 17 | keys = sb.structure.keys() 18 | 19 | hasAllKeys = True 20 | for key in keys: 21 | if not key in sb.data: 22 | hasAllKeys = False 23 | break 24 | 25 | self.assertTrue(hasAllKeys) 26 | -------------------------------------------------------------------------------- /fishy/ntfs/ntfs_filesystem/record_header.py: -------------------------------------------------------------------------------- 1 | """ 2 | Declaration of the header of MFT Records 3 | """ 4 | 5 | from construct import Struct, Bytes, Int16ul, Int32ul 6 | from .record_reference import RECORD_REFERENCE 7 | 8 | RECORD_HEADER = Struct( 9 | "signature" / Bytes(4), 10 | "update_sequence_array_offset" / Int16ul, 11 | "update_sequence_array_size" / Int16ul, 12 | "reserved1" / Bytes(8), 13 | "sequence_number" / Int16ul, 14 | "reserved2" / Bytes(2), 15 | "first_attribute_offset" / Int16ul, 16 | "flags" / Bytes(2), 17 | "reserved3" / Bytes(8), 18 | "base_record_segment" / RECORD_REFERENCE, 19 | "reserved4" / Bytes(2) 20 | ) 21 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = fishy 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /fishy/ext4/ext4_filesystem/inode_table.py: -------------------------------------------------------------------------------- 1 | from .inode import Inode 2 | 3 | class InodeTable: 4 | """ 5 | This class represents a single inode table. 6 | """ 7 | 8 | inodes = [] 9 | 10 | def __init__(self, fs_stream, superblock, offset, blocksize): 11 | self.blocksize = blocksize 12 | self.table_start = offset 13 | self.inode_size = superblock.data["inode_size"] 14 | self.table_size = superblock.data['inodes_per_group'] * self.inode_size 15 | 16 | for addr in range(self.table_start, self.table_start+self.table_size, self.inode_size): 17 | inode = Inode(fs_stream, addr, self.inode_size, self.blocksize) 18 | self.inodes.append(inode) 19 | -------------------------------------------------------------------------------- /doc/fat-fileslack.dot: -------------------------------------------------------------------------------- 1 | digraph { 2 | node[shape=hexagon] 3 | begin, end 4 | node[shape=diamond] 5 | "free slack space found" 6 | "data left to write" 7 | node[shape=box] 8 | "begin" -> "get filepaths" -> "traverse FAT directories" -> "calculate free slack space" -> "free slack space found" 9 | "free slack space found" -> "find last custer of the file"[label="[yes]"] 10 | "find last custer of the file" -> "write into slack space" 11 | "free slack space found" -> "traverse FAT directories"[label="[no]"] 12 | "write into slack space" -> "save cluster_id, cluster offset and written length" -> "data left to write" 13 | "data left to write" -> "traverse FAT directories"[label="[yes]"] 14 | "data left to write" -> "end"[label="[no]"] 15 | } 16 | -------------------------------------------------------------------------------- /fishy/ntfs/ntfs_detector.py: -------------------------------------------------------------------------------- 1 | # TODO: This file should be moved into ntfs module when it exists 2 | # TODO: This is a duplicate of fat.fat_filesystem.fat_detector 3 | # we should deduplicate this 4 | class UnsupportedFilesystemError(Exception): 5 | pass 6 | 7 | 8 | def is_ntfs(stream): 9 | """ 10 | checks if a given stream is of type ntfs or not 11 | :param stream: stream of filesystem 12 | :return: bool, True if it is a NTFS filesystem 13 | False if it is not a NTFS filesystem 14 | """ 15 | # get start position to reset after check 16 | offset = stream.tell() 17 | stream.seek(offset + 3) 18 | fs_type = stream.read(8) 19 | stream.seek(offset) 20 | if fs_type == b'NTFS ': 21 | return True 22 | return False 23 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | set SPHINXPROJ=fishy 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /fishy/ext4/ext4_filesystem/EXT4.py: -------------------------------------------------------------------------------- 1 | import typing as typ 2 | from pytsk3 import FS_Info, Img_Info 3 | 4 | from fishy.ext4.ext4_filesystem.gdt import GDT 5 | from fishy.ext4.ext4_filesystem.inode_table import InodeTable 6 | from fishy.ext4.ext4_filesystem.superblock import Superblock 7 | 8 | 9 | class EXT4: 10 | 11 | def __init__(self, stream: typ.BinaryIO, dev: str): 12 | self.blocksize = self._get_blocksize(dev) 13 | self.superblock = Superblock(stream) 14 | self.gdt = GDT(stream, self.superblock, self.blocksize) 15 | 16 | # get inode table for each gdt entry 17 | self.inode_tables = [] 18 | for gdt_entry in self.gdt.data: 19 | table_start = gdt_entry['inode_table_lo'] * self.blocksize 20 | self.inode_tables.append(InodeTable(stream, self.superblock, table_start, self.blocksize)) 21 | 22 | def _get_blocksize(self, dev): 23 | img_info = Img_Info(dev) 24 | fs_info = FS_Info(img_info, offset=0) 25 | return fs_info.info.block_size 26 | -------------------------------------------------------------------------------- /fishy/ntfs/ntfs_filesystem/bootsector.py: -------------------------------------------------------------------------------- 1 | """ 2 | Bootsector definition for NTFS 3 | """ 4 | 5 | from construct import Struct, Byte, Bytes,\ 6 | Int8ul, Int8sl, Int16ul, Int32ul, Int64ul 7 | 8 | NTFS_BOOTSECTOR = Struct( 9 | "jump_instruction" / Bytes(3), 10 | "oem_name" / Bytes(8), 11 | "sector_size" / Int16ul, 12 | "cluster_size" / Int8ul, 13 | "reserved_sectors" / Bytes(2), 14 | "reserved_1" / Bytes(3), 15 | "unused_1" / Bytes(2), 16 | "media_descriptor" / Byte, 17 | "unused_2" / Bytes(2), 18 | "sectors_per_track" / Int16ul, 19 | "num_heads" / Int16ul, 20 | "hidden_sectors" / Int32ul, 21 | "unused_3" / Bytes(4), 22 | "unused_5" / Bytes(4), 23 | "total_sectors" / Int64ul, 24 | "mft_cluster" / Int64ul, 25 | "mft_mirr_cluster" / Int64ul, 26 | "clusters_per_file_record" / Int8sl, 27 | "unused_6" / Bytes(3), 28 | "clusters_per_index_buffer" / Int8sl, 29 | "unused_7" / Bytes(3), 30 | "volume_serial" / Bytes(8), 31 | "checksum" / Bytes(4), 32 | "bootstrap" / Bytes(426), 33 | "eos_marker" / Bytes(2) 34 | ) 35 | 36 | -------------------------------------------------------------------------------- /fishy/fat/fat_filesystem/fat_wrapper.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains a wrapper for FAT filesystems, which 3 | detects the FAT filesystem an uses the right class 4 | 5 | >>> f = open('testfs.dd', 'rb') 6 | >>> fs = create_fat(f) 7 | """ 8 | import typing as typ 9 | from .fat_detector import get_filesystem_type 10 | from .fat import FAT 11 | from .fat_12 import FAT12 12 | from .fat_16 import FAT16 13 | from .fat_32 import FAT32 14 | 15 | 16 | def create_fat(stream: typ.BinaryIO) -> FAT: 17 | """ 18 | Detect FAT filesystem type and return an instance of it 19 | :param stream: filedescriptor of a FAT filesystem 20 | :return: FAT filesystem object 21 | :raises: UnsupportedFilesystemError 22 | :rtype: FAT12, FAT16 or FAT32 23 | """ 24 | # get fs_type 25 | fat_type = get_filesystem_type(stream) 26 | # check if it is a FAT12 27 | if fat_type == 'FAT12': 28 | return FAT12(stream) 29 | # check if it is a FAT16 30 | elif fat_type == 'FAT16': 31 | return FAT16(stream) 32 | # check if its FAT32 33 | elif fat_type == 'FAT32': 34 | return FAT32(stream) 35 | -------------------------------------------------------------------------------- /fishy/ext4/ext4_filesystem/parser.py: -------------------------------------------------------------------------------- 1 | import builtins 2 | import time 3 | 4 | class Parser: 5 | @staticmethod 6 | def parse(image, offset, length, structure, byteorder='little'): 7 | data_dict = {} 8 | image.seek(offset) 9 | 10 | data = image.read(length) 11 | for key in structure: 12 | offset = structure[key]['offset'] 13 | size = structure[key]['size'] 14 | 15 | bytes = data[offset:offset+size] 16 | value = int.from_bytes(bytes, byteorder=byteorder) 17 | 18 | if "format" in structure[key]: 19 | if structure[key]["format"] == "ascii": 20 | value = bytes.decode('ascii') 21 | elif structure[key]["format"] == "raw": 22 | value = bytes 23 | elif structure[key]["format"] == "time": 24 | value = time.gmtime(value) 25 | else: 26 | form = getattr(builtins, structure[key]["format"]) 27 | value = form(value) 28 | 29 | data_dict[key] = value 30 | return data_dict -------------------------------------------------------------------------------- /fishy/ntfs/ntfs_filesystem/attribute_header.py: -------------------------------------------------------------------------------- 1 | """ 2 | Declaration of the attribute header for file attributes 3 | """ 4 | 5 | from construct import Struct, Byte, Bytes, Int8ul, Int16ul, Int32ul, Int64ul, Padding, Flag, IfThenElse, Embedded, this 6 | 7 | ATTR_RESIDENT = Struct( 8 | "length" / Int32ul, 9 | "offset" / Int16ul, 10 | "indexed_tag" / Byte, 11 | Padding(1) 12 | ) 13 | 14 | ATTR_NONRESIDENT = Struct( 15 | "start_vcn" / Int64ul, 16 | "end_vcn" / Int64ul, 17 | "datarun_offset" / Int16ul, 18 | "compression_size" / Int16ul, 19 | Padding(4), 20 | "alloc_size" / Int64ul, 21 | "real_size" / Int64ul, 22 | "stream_size" / Int64ul 23 | ) 24 | 25 | ATTRIBUTE_HEADER = Struct( 26 | "type" / Int32ul, 27 | "total_length" / Int32ul, 28 | "nonresident" / Flag, 29 | "name_length" / Int8ul, 30 | "name_offset" / Int16ul, 31 | "flags" / Bytes(2), 32 | "id" / Int16ul, 33 | Embedded(IfThenElse(this.nonresident, ATTR_NONRESIDENT, \ 34 | ATTR_RESIDENT)) 35 | # TODO Construct compatibility update here: Embedded IfThenElse no longer allowed -> EmbeddedSwitch alternative? 36 | ) 37 | -------------------------------------------------------------------------------- /doc/source/resources/module_flowchart.dot: -------------------------------------------------------------------------------- 1 | digraph G { 2 | splines=ortho 3 | rankdir = BT; 4 | 5 | node[fixedsize=true, width=5, shape=component] 6 | edge[fontcolor="#636363", fontsize=10] 7 | 8 | subgraph cluster_0 { 9 | "placeholder2" [style=invisible]; 10 | "cli" [label=CLI]; 11 | "Qtechnique" [label="Hiding Technique", shape=diamond]; 12 | 13 | } 14 | 15 | subgraph cluster_1 { 16 | "techniqueWrapper" [label="Hiding Technique Wrapper"]; 17 | "Qfstype" [label="FS Type", shape=diamond]; 18 | "writeMeta" [label="Write Metadata", shape=box]; 19 | } 20 | 21 | subgraph cluster_2 { 22 | "Write/Read/Clear" [shape=box]; 23 | "hidingTechnique" [label="Hiding Technique"]; 24 | } 25 | 26 | "cli" -> "Qtechnique"; 27 | "Qtechnique" -> "techniqueWrapper"[label="[File Slack]"] 28 | "Qtechnique" -> "techniqueWrapper"[label="[Bad Cluster]"] 29 | "Qtechnique" -> "techniqueWrapper"[label="[...]"] 30 | "techniqueWrapper" -> "Qfstype" 31 | "Qfstype" -> "hidingTechnique"[label="[FAT]"] 32 | "Qfstype" -> "hidingTechnique"[label="[NTFS]"] 33 | "Qfstype" -> "hidingTechnique"[label="[ext4]"] 34 | "hidingTechnique" -> "writeMeta" 35 | "hidingTechnique" -> "Write/Read/Clear" 36 | } 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 da/sec 4 | Copyright (c) 2017 Jonas Plum - github.com/cugu/afro/blob/master/afro/checksum.py 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /fishy/filesystem_detector.py: -------------------------------------------------------------------------------- 1 | """ 2 | Basic filesystem detector for FAT and NTFS 3 | """ 4 | 5 | import typing as typ 6 | 7 | from fishy.APFS.APFS_filesystem import APFS_Detector 8 | from fishy.ext4.ext4_filesystem import ext4_detector 9 | from .fat.fat_filesystem import fat_detector 10 | from .ntfs import ntfs_detector 11 | 12 | 13 | # TODO: This is a duplicate of fat_detector 14 | # we should somehow figure out how to 15 | # deduplicate this 16 | class UnsupportedFilesystemError(Exception): 17 | """ 18 | This exception indicates, that the filesystem type could not be determined 19 | """ 20 | pass 21 | 22 | 23 | def get_filesystem_type(stream: typ.BinaryIO) -> str: 24 | """ 25 | extracts the filesystem type from a given stream 26 | :stream: stream of filesystem 27 | :return: string, 'FAT' 28 | :raises: UnsupportedFilesystemError 29 | """ 30 | if fat_detector.is_fat(stream): 31 | return "FAT" 32 | elif ntfs_detector.is_ntfs(stream): 33 | return "NTFS" 34 | elif ext4_detector.is_ext4(stream): 35 | return "EXT4" 36 | elif APFS_Detector.is_apfs(stream): 37 | return "APFS" 38 | else: 39 | raise UnsupportedFilesystemError() 40 | -------------------------------------------------------------------------------- /tests/test_filesystem_detector.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring 2 | """ 3 | This file contains tests against fishy.filesystem_detector 4 | """ 5 | import pytest 6 | from fishy.filesystem_detector import get_filesystem_type, UnsupportedFilesystemError 7 | 8 | 9 | class TestFileSystemDetector(object): 10 | 11 | def test_fat_images(self, testfs_fat_stable1): 12 | """ Test if FAT images are detected correctly """ 13 | for img in testfs_fat_stable1: 14 | with open(img, 'rb') as fs_stream: 15 | result = get_filesystem_type(fs_stream) 16 | assert result == 'FAT' 17 | 18 | def test_ntfs_images(self, testfs_ntfs_stable1): 19 | """ Test if NTFS images are detected correctly """ 20 | for img in testfs_ntfs_stable1: 21 | with open(img, 'rb') as fs_stream: 22 | result = get_filesystem_type(fs_stream) 23 | assert result == 'NTFS' 24 | 25 | def test_ext4_images(self, testfs_ext4_stable1): 26 | """ Test if ext4 images are detected correctly """ 27 | for img in testfs_ext4_stable1: 28 | with open(img, 'rb') as fs_stream: 29 | result = get_filesystem_type(fs_stream) 30 | assert result == 'EXT4' -------------------------------------------------------------------------------- /tests/test_fat_detector.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains tests against fishy.fat.fat_filesystem.fat_detector 3 | """ 4 | from fishy.fat.fat_filesystem import fat_detector 5 | 6 | def test_get_filesystem(testfs_fat_stable1): 7 | """ Test if specific FAT detection works """ 8 | with open(testfs_fat_stable1[0], 'rb') as img_stream: 9 | result = fat_detector.get_filesystem_type(img_stream) 10 | assert result == 'FAT12' 11 | with open(testfs_fat_stable1[1], 'rb') as img_stream: 12 | result = fat_detector.get_filesystem_type(img_stream) 13 | assert result == 'FAT16' 14 | with open(testfs_fat_stable1[2], 'rb') as img_stream: 15 | result = fat_detector.get_filesystem_type(img_stream) 16 | assert result == 'FAT32' 17 | 18 | def test_is_fat(testfs_fat_stable1): 19 | """ Test if general FAT detection works """ 20 | with open(testfs_fat_stable1[0], 'rb') as img_stream: 21 | result = fat_detector.is_fat(img_stream) 22 | assert result 23 | with open(testfs_fat_stable1[1], 'rb') as img_stream: 24 | result = fat_detector.is_fat(img_stream) 25 | assert result 26 | with open(testfs_fat_stable1[2], 'rb') as img_stream: 27 | result = fat_detector.is_fat(img_stream) 28 | assert result 29 | -------------------------------------------------------------------------------- /tests/test_ntfs_cluster_allocator.py: -------------------------------------------------------------------------------- 1 | 2 | import io 3 | import pytest 4 | from fishy.ntfs.cluster_allocator import ClusterAllocator 5 | 6 | 7 | class TestClusterAllocator(object): 8 | """ Tests the cluster allocator """ 9 | # @pytest.mark.xfail 10 | def test_get_data(self, testfs_ntfs_stable1): 11 | """ 12 | Tests if the correct data is returned 13 | """ 14 | with open(testfs_ntfs_stable1[0], 'rb+') as fs,\ 15 | open('utils/fs-files/another', 'rb') as to_hide1: 16 | allocator = ClusterAllocator(fs) 17 | metadata = allocator.write(to_hide1, 'long_file.txt') 18 | with io.BytesIO() as mem: 19 | allocator.read(mem, metadata) 20 | mem.seek(0) 21 | assert mem.read() == b'222\n' 22 | 23 | to_hide2 = io.BytesIO(b'2'*7000) 24 | metadata = allocator.write(to_hide2, 'long_file.txt') 25 | with io.BytesIO() as mem: 26 | allocator.read(mem, metadata) 27 | mem.seek(0) 28 | assert mem.read() == b'2'*7000 29 | 30 | allocator.clear(metadata) 31 | with io.BytesIO() as mem: 32 | allocator.read(mem, metadata) 33 | mem.seek(0) 34 | assert mem.read() == b'0'*7000 35 | 36 | -------------------------------------------------------------------------------- /fishy/APFS/APFS_filesystem/Object_Header.py: -------------------------------------------------------------------------------- 1 | # Object header Structure 2 | 3 | from fishy.APFS.APFS_filesystem.APFS_Parser import Parser 4 | 5 | 6 | HEADER_SIZE = 32 7 | BYTEORDER = 'little' 8 | 9 | 10 | class ObjectHeader: 11 | 12 | structure = { 13 | "o_chksum": {"offset": 0x0, "size": 8, "format": "hex"}, 14 | # Fletchers Checksum, Input is entire block without first 8 bytes 15 | "oid": {"offset": 0x8, "size": 8}, 16 | # Object ID 17 | "xid": {"offset": 0x10, "size": 8}, 18 | # Version ID 19 | "type": {"offset": 0x18, "size": 2}, 20 | "flags": {"offset": 0x1A, "size": 2, "format": "hex"}, 21 | "subtype": {"offset": 0x1C, "size": 4} 22 | # Alternative size subtype 2 and 2 padding 23 | 24 | } 25 | 26 | def __init__(self, fs_stream, offset): 27 | self.data = self.parse_object_header(fs_stream, offset) 28 | self.offset = offset 29 | 30 | def parse_superblock_object_header(self, fs_stream): 31 | d = Parser.parse(fs_stream, 0, HEADER_SIZE, structure=self.structure) 32 | fs_stream.seek(0) 33 | return d 34 | 35 | def parse_object_header(self, fs_stream, offset): 36 | d = Parser.parse(fs_stream, offset, HEADER_SIZE, structure=self.structure) 37 | fs_stream.seek(0) 38 | 39 | return d 40 | 41 | def getSize(self): 42 | return HEADER_SIZE 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /fishy/APFS/APFS_filesystem/APFS_Parser.py: -------------------------------------------------------------------------------- 1 | # If working with Construct 2.8.2 = temporarily obsolete 2 | # If working without Construct = base on ext4 parser / others 3 | import builtins 4 | import time 5 | 6 | 7 | class Parser: 8 | 9 | @staticmethod 10 | def parser(): 11 | print("This feature is not yet implemented") 12 | 13 | # TODO: parse 1 to 1 taken from ext4; adjust to apfs if needed -> Timer (test on linux) 14 | 15 | @staticmethod 16 | def parse(image, offset, length, structure, byteorder='little'): 17 | data_dict = {} 18 | image.seek(offset) 19 | 20 | data = image.read(length) 21 | for key in structure: 22 | offset = structure[key]['offset'] 23 | size = structure[key]['size'] 24 | 25 | bytes = data[offset:offset + size] 26 | value = int.from_bytes(bytes, byteorder=byteorder) 27 | 28 | if "format" in structure[key]: 29 | if structure[key]["format"] == "ascii": 30 | value = bytes.decode('ascii') 31 | elif structure[key]["format"] == "utf": 32 | value = bytes.decode('utf-8') 33 | elif structure[key]["format"] == "raw": 34 | value = bytes 35 | elif structure[key]["format"] == "time": 36 | value = time.gmtime(value) 37 | else: 38 | form = getattr(builtins, structure[key]["format"]) 39 | value = form(value) 40 | 41 | data_dict[key] = value 42 | return data_dict 43 | -------------------------------------------------------------------------------- /fishy/ntfs/ntfs_filesystem/t_ntfs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Manual testing script for NTFS class 3 | """ 4 | 5 | from .ntfs import NTFS 6 | from .record_header import RECORD_HEADER 7 | from .attribute_header import ATTRIBUTE_HEADER 8 | 9 | with open('ntfs_filesystem/testfs-ntfs.dd', 'rb') as fs: 10 | n = NTFS(fs) 11 | print("Record Size: ", n.record_size) 12 | print("Start Offset: ", n.start_offset) 13 | print("MFT Offset: ", n.mft_offset) 14 | print("MFT Runs: ", n.mft_runs) 15 | record = n.get_record(0) 16 | header = RECORD_HEADER.parse(record) 17 | print("Record Header Signature: ", header.signature) 18 | offset = header.first_attribute_offset 19 | header = ATTRIBUTE_HEADER.parse(record[offset:]) 20 | print("Attribute type: ", header.type) 21 | offset = offset + header.total_length 22 | header = ATTRIBUTE_HEADER.parse(record[offset:]) 23 | print("Attribute type: ", header.type) 24 | 25 | #Print the filenames of the first mft records 26 | for x in range(0, 75): 27 | name: str = n.get_filename_from_record(x) 28 | print("Name of record ", x, ": ", name) 29 | 30 | record_number = n.get_record_of_file("another") 31 | print("Record number of file \"another\": ", record_number) 32 | print("Data of file \"another\": ", n.get_data(record_number)) 33 | print("Dataruns of file \"long_file.txt\": ", n.get_data_runs(n.get_record(66))) 34 | print("Record number of file \"onedirectory/nested_directory/royce.txt\": ", n.get_record_of_file('onedirectory/nested_directory/royce.txt')) 35 | print(n.get_data_runs(n.get_record(6))) 36 | -------------------------------------------------------------------------------- /doc/source/07_2_responsibilities.rst: -------------------------------------------------------------------------------- 1 | Resposibilities 2 | --------------- 3 | 4 | The People who worked on this tool: 5 | 6 | * Deniz Celik: 7 | * ext4: reserved GDT blocks hiding technique 8 | * ext4: filesystem parser 9 | * ext4: filesystem detector 10 | * Tim Christen: 11 | * NTFS filesystem parser 12 | * NTFS Additional Cluster Allocation 13 | * Additional Cluster Allocation Documentation 14 | * Matthias Greune: 15 | * Architecture, overall tool design 16 | * `create_testfs.sh` script 17 | * FAT filesystem parser and corresponding Tests 18 | * FAT and NTFS filesystem detector 19 | * FAT Hiding Technique FileSlack and corresponding Tests 20 | * FAT Hiding Technique Bad Cluster and corresponding Tests 21 | * FAT Hiding Technique Additional Cluster Allocation and corresponding Tests 22 | * Documentation for 23 | * Introduction Stuff 24 | * FAT filesystem explanation 25 | * Fileslack explanation 26 | * Architecture overview 27 | * Christian Hecht: 28 | * Reserved GDT hinding technique 29 | * Ext4 filesystem parser 30 | * Adrian Kailus: 31 | * Ext4 Documentation 32 | * obso_faddr hiding technique for Ext4 33 | * info switch for all Ext4 techniques 34 | * Ext4 filesystem parser 35 |    * Documentation Overview and various other 36 | * Dustin Kern: 37 | * NTFS MFT Slack technique and documentation 38 | * NTFS File Slack technique and documentation 39 | * Metadata encryption 40 | * NTFS Documentation 41 | * Patrick Naili: 42 | * Ext4 File Slack 43 | * Ext4 Superblock Slack 44 | * Ext4 filesystem parser 45 | * Chau Nguyen: 46 | * NTFS Bad Cluster Allocation technique and documentation 47 | * Yannick Mau: 48 | * osd2 Technique 49 | * Ext4 filesystem parser 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore testoutput of unittests 2 | doctests.xml 3 | # ignore images in doc for now 4 | doc/*png 5 | # ignore ide setting foo 6 | .idea 7 | .ropeproject 8 | # ignore create_testfs "config" file 9 | .create_testfs.conf 10 | # ignore fislystem dumps 11 | *.dd 12 | # ignore vim swap files 13 | *.swp 14 | 15 | # Byte-compiled / optimized / DLL files 16 | __pycache__/ 17 | *.py[cod] 18 | *$py.class 19 | 20 | # C extensions 21 | *.so 22 | 23 | # Distribution / packaging 24 | .Python 25 | env/ 26 | build/ 27 | develop-eggs/ 28 | dist/ 29 | downloads/ 30 | eggs/ 31 | .eggs/ 32 | lib/ 33 | lib64/ 34 | parts/ 35 | sdist/ 36 | var/ 37 | *.egg-info/ 38 | .installed.cfg 39 | *.egg 40 | 41 | # PyInstaller 42 | # Usually these files are written by a python script from a template 43 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 44 | *.manifest 45 | *.spec 46 | 47 | # Installer logs 48 | pip-log.txt 49 | pip-delete-this-directory.txt 50 | 51 | # Unit test / coverage reports 52 | htmlcov/ 53 | .tox/ 54 | .coverage 55 | .coverage.* 56 | .cache 57 | nosetests.xml 58 | coverage.xml 59 | *,cover 60 | .hypothesis/ 61 | 62 | # Translations 63 | *.mo 64 | *.pot 65 | 66 | # Django stuff: 67 | *.log 68 | local_settings.py 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | target/ 82 | 83 | # Jupyter Notebook 84 | .ipynb_checkpoints 85 | 86 | # pyenv 87 | .python-version 88 | 89 | # celery beat schedule file 90 | celerybeat-schedule 91 | 92 | # dotenv 93 | .env 94 | 95 | # virtualenv 96 | .venv/ 97 | venv/ 98 | ENV/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | 103 | # Rope project settings 104 | .ropeproject 105 | -------------------------------------------------------------------------------- /doc/source/05_future_work.rst: -------------------------------------------------------------------------------- 1 | Future Work 2 | =========== 3 | 4 | The current state of the project offers enough functionality to hide data in 5 | our chosen filesystems. However there is still room for improvement of some 6 | features. This section gives a brief overview of some potential key points. 7 | 8 | The filesystem auto detection for FAT and NTFS needs improvement. It is 9 | currently performed by checking an ASCII string in the boot sector. In order to 10 | increase the reliability of `fishy`, it could be reimplemented by using the 11 | detection methods, that are already realized in regular filesystem 12 | implementations. 13 | 14 | Likewise, at present the clearing method for all hiding techniques just 15 | overwrites the hidden data with zero bytes. This does not apply to any security 16 | standards for save data removal. This area offers much room for improvements. 17 | 18 | At the current state `fishy` does not provide on the fly data encryption and has 19 | no data integrity methods implemented. As finding unencrypted data is 20 | relatively easy with forensic tools - regardless of the hiding technique - 21 | encrypting all hidden data by default might be a helpful addition. 22 | As most hiding techniques are not stable, if the related files on the 23 | filesystem change often, some data integrity methods would be useful to detect 24 | at least, if the hidden data got corrupted in the meantime. 25 | 26 | Currently `fishy` produces a metadata file with each hiding operation. Although 27 | it can be encrypted, it is visible via conventional data access methods and 28 | hints to hidden data. An idea to tackle this problem might be to hide the 29 | metadata file itself. 30 | 31 | Lastly, the implementation of multiple data support would be a welcome 32 | addition. This includes but is not limited to the implementation of a FUSE 33 | filesystem layer, which can use multiple hiding techniques to store data. This 34 | would drastically lower the burden to use this toolkit on a more regular basis. 35 | -------------------------------------------------------------------------------- /fishy/APFS/APFS_filesystem/APFS.py: -------------------------------------------------------------------------------- 1 | # centerpiece of interface implementation, content tbd 2 | # offset and headersize into parser -> get object type function: parse corresponding object structure with offset+header 3 | # \2 (start after header) and blocksize-header (do not include header in parsing of object) 4 | 5 | from typing import BinaryIO 6 | from fishy.APFS.APFS_filesystem.Container_Superblock import Superblock 7 | from fishy.APFS.APFS_filesystem.Object_Header import ObjectHeader 8 | 9 | # TODO: implement in fishy 10 | # TODO: test on linux 11 | # TODO: Wrapper Functions, CLI implementation, FS Detector implementation 12 | # TODO: macOS VM to test apfs fsck and potential other fs checks 13 | 14 | class APFS: 15 | 16 | def __init__(self, stream: BinaryIO): 17 | self.stream = stream 18 | self.mainSuperblock = Superblock(stream, 0) 19 | self.blocksize = self.getBlockSize() 20 | 21 | def getObjectType(self, offset): 22 | # TODO: move to Object_Header.py 23 | objectType = " \n" 24 | ohead = ObjectHeader(self.stream, offset) 25 | d = ohead.parse_object_header(self.stream, offset) 26 | tempType = d["type"] 27 | if tempType == 0: 28 | objectType = "not found \n" 29 | elif tempType == 1: 30 | objectType = "Container Superblock detected" 31 | elif tempType == 2: 32 | objectType = "Root Node detected" 33 | elif tempType == 3: 34 | objectType = "Node detected" 35 | elif tempType == 5: 36 | objectType = "Space Manager detected" 37 | elif tempType == 7: 38 | objectType = "Space Manager Internal Pool detected" 39 | elif tempType == 11: 40 | objectType = "B-Tree detected" 41 | elif tempType == 12: 42 | objectType = "Checkpoint detected" 43 | elif tempType == 13: 44 | objectType = "Volume Superblock detected" 45 | elif tempType == 17: 46 | objectType = "Reaper detected" 47 | 48 | 49 | return objectType 50 | 51 | def getBlockSize(self): 52 | blocksize = self.mainSuperblock.getBlockSize() 53 | return blocksize 54 | 55 | -------------------------------------------------------------------------------- /fishy/APFS/APFS_filesystem/Object_Map.py: -------------------------------------------------------------------------------- 1 | from fishy.APFS.APFS_filesystem.APFS_Parser import Parser 2 | from fishy.APFS.APFS_filesystem.Volume_Superblock import vSuperblock 3 | from fishy.APFS.APFS_filesystem.Node import Node 4 | 5 | class ObjectMap: 6 | 7 | structure = { 8 | "B-Tree_Type": {"offset": 0x0, "size": 8, "format": "hex"}, 9 | "root": {"offset": 0x10, "size": 8}, 10 | "reserved": {"offset": 0x18, "size": 56} 11 | # There are supposed to be more than just the root Node and tree type in the block type, but none of the images 12 | # show that - so there is a reserved space given just in case the other components show up 13 | 14 | } 15 | 16 | def __init__(self, fs_stream, offset, blocksize): 17 | self.offset = offset 18 | self.blocksize = blocksize 19 | self.data = self.parseObjectMap(fs_stream, offset) 20 | self.root = self.getRootNode(fs_stream) 21 | 22 | def parseObjectMap(self, fs_stream, offset): 23 | d = Parser.parse(fs_stream, offset+32, self.blocksize-32, structure=self.structure) 24 | return d 25 | 26 | def getRootNode(self, fs_stream): 27 | blocksize = self.blocksize 28 | root = self.data["root"] * blocksize 29 | return root 30 | 31 | def map_CObject(self, mapArray): 32 | 33 | # TODO: use mapping parts of this function (receive tuple array, calculate offset of volumes & parse; 34 | # TODO: \2 remove printing parts as soon as other function of mapping is implemented, keep marked as important 35 | 36 | blocksize = self.blocksize 37 | # important 38 | #print("Used Volumes: " + str(len(mapArray)) + "\n") 39 | # for x, y in mapArray: # important 40 | # print("Address: " + str(x) + " | " + " Volume ID: " + str(y)) 41 | 42 | mapArrayCalc = [] 43 | for x,y in mapArray: 44 | singleCalc = ((x*blocksize), y) 45 | mapArrayCalc.append(singleCalc) 46 | 47 | return mapArrayCalc 48 | 49 | def mapCObjectMap(self, fs_stream): 50 | 51 | root = self.root 52 | 53 | rootnode = Node(fs_stream, root, self.blocksize) 54 | 55 | vm = rootnode.getVolumeMapping() 56 | 57 | calcMap = self.map_CObject(vm) 58 | 59 | volumesList = [] 60 | 61 | for x,y in calcMap: 62 | vol_superblock = vSuperblock(fs_stream, x, self.blocksize) 63 | d = (vol_superblock.parseVolumeSuperblock(fs_stream, x), y) 64 | volumesList.append(d) 65 | 66 | return volumesList 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /fishy/fat/fat_filesystem/fat_16.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implementation of a FAT16 filesystem reader 3 | 4 | example usage: 5 | >>> with open('testfs.dd', 'rb') as filesystem: 6 | >>> fs = FAT16(filesystem) 7 | 8 | example to print all entries in root directory: 9 | >>> for i, v in fs.get_root_dir_entries(): 10 | >>> if v != "": 11 | >>> print(v) 12 | 13 | example to print all fat entries 14 | >>> for i in range(fs.entries_per_fat): 15 | >>> print(i,fs.get_cluster_value(i)) 16 | 17 | example to print all root directory entries 18 | >>> for entry in fs.get_root_dir_entries(): 19 | >>> print(v, entry.get_start_cluster()) 20 | 21 | """ 22 | import typing as typ 23 | from construct import Struct, Array, Padding, Embedded, Bytes, this 24 | from .bootsector import FAT12_16_BOOTSECTOR 25 | from .dir_entry import DIR_ENTRY_DEFINITION as DIR_ENTRY 26 | from .fat import FAT 27 | from .fat_entry import FAT16Entry 28 | 29 | 30 | FAT16_PRE_DATA_REGION = Struct( 31 | "bootsector" / Embedded(FAT12_16_BOOTSECTOR), 32 | Padding((this.reserved_sector_count - 1) * this.sector_size), 33 | # FATs 34 | "fats" / Array(this.fat_count, Bytes(this.sectors_per_fat * this.sector_size)), 35 | # RootDir Table 36 | "rootdir" / Bytes(this.rootdir_entry_count * DIR_ENTRY.sizeof()) 37 | ) 38 | 39 | 40 | class FAT16(FAT): 41 | """ 42 | FAT16 filesystem implementation. 43 | """ 44 | def __init__(self, stream: typ.BinaryIO): 45 | """ 46 | :param stream: filedescriptor of a FAT16 filesystem 47 | """ 48 | super().__init__(stream, FAT16_PRE_DATA_REGION) 49 | self.entries_per_fat = int(self.pre.sectors_per_fat 50 | * self.pre.sector_size 51 | / 2) 52 | self._fat_entry = FAT16Entry 53 | self.fat_type = 'FAT16' 54 | 55 | def get_cluster_value(self, cluster_id: int) -> typ.Union[int, str]: 56 | """ 57 | finds the value that is written into fat 58 | for given cluster_id 59 | :param cluster_id: int, cluster that will be looked up 60 | :return: int or string 61 | """ 62 | byte = cluster_id*2 63 | byte_slice = self.pre.fats[0][byte:byte+2] 64 | value = int.from_bytes(byte_slice, byteorder='little') 65 | return self._fat_entry.parse(value.to_bytes(2, 'little')) 66 | 67 | def _root_to_stream(self, stream: typ.BinaryIO) -> None: 68 | """ 69 | write root directory into a given stream 70 | :param stream: stream, where the root directory will be written into 71 | """ 72 | stream.write(self.pre.rootdir) 73 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | This file collects issues, that were still open at the time of our commitment. 2 | 3 | # Checksums 4 | 5 | * We should calculate checksums over input data to detect information loss while reading hidden data 6 | * Maybe [rolling hashes](https://en.wikipedia.org/wiki/Rolling_hash) might be the thing we should use, because with them we don't need to buffer the whole input first, before we can write it to disk. 7 | * Or we refer to archive formats like zip, which have error detection. This way we follow the unix philosophy and don't make things too complicated 8 | 9 | # FileSlack: possible overwrites in directory autoexpand feature 10 | 11 | If a user supplies a directory plus a file in this directory as destinations for fileslack exploitation, the autoexpansion of directories could lead to multiple writes into the slack space of the same file. For instance: 12 | ``` 13 | $ fishy -d testfs_fat12.dd fileslack -w -m "meta.json" -d adir/afile.txt -d adir longfile.txt 14 | ``` 15 | would first write into `adir/afile.txt`, then expand `adir` to `adir/afile.txt` and then write again into the slack space of `adir/afile.txt`. 16 | 17 | This is an issue in FAT fileslack implementation, but I'm not sure if the NTFS implementation is affected. 18 | 19 | # Refine cli interface 20 | 21 | The are some inconsistencies in the cli interface. 22 | 23 | * The `metadata` subcommand does not require a device (`-d`), but all other commands do. 24 | * The info option of `fileslack` subcommand should not require a metadata file 25 | 26 | Also it might be nicer to move the `-d` option behind the subcommand. 27 | 28 | The argparse configuration should require all options which are actually required and should not require options, which are not required... 29 | 30 | * Maybe we should seperate informational functionality (fattools, metadata) from hiding techniques via additional subgroups 31 | * Read/write/clear/info options of a hiding technique should only be used once at a time. We could propably implement this via `add_mutually_exclusive_group` 32 | * for fileslack subcommand, the `-d` option must be required, when writing to fileslack 33 | 34 | Some other things are wrong or need extension in the help output: 35 | * as the mftslack options were copied from fileslack, this keyword occures in the help output, but is wrong there. 36 | * subcommand help output should be more descriptive 37 | 38 | # FAT, NTFS: construct incompatibility 39 | 40 | The construct library changed some things in their recent 2.9.X release so that our code is currently incompatible with their current version. 41 | 42 | Maybe someone has the time to fix those incompatibilities. Meanwhile I will fix our requirements to construct in its pre 2.9 version. 43 | -------------------------------------------------------------------------------- /fishy/fat/fat_filesystem/fat_detector.py: -------------------------------------------------------------------------------- 1 | """ 2 | fat_detector includes is_fat and get_filesystem_type 3 | function, which provide detection of a fat filesystem. 4 | 5 | For the ease of implementation these functions rely on 6 | the fat_version field of the Bootsector (offset 0x36 for 7 | FAT12/16 and offset 0x52 for FAT32). Please note that 8 | this field needs not to be correct, although common tools 9 | set it the right way. 10 | """ 11 | 12 | import typing 13 | 14 | 15 | class UnsupportedFilesystemError(Exception): 16 | """ 17 | This error occures, if a filesystem could not be determined. 18 | """ 19 | pass 20 | 21 | 22 | def is_fat(stream: typing.BinaryIO) -> bool: 23 | """ 24 | checks if a given stream is of type fat or not 25 | :param stream: stream of filesystem 26 | :return: bool, True if it is a FAT filesystem 27 | False if it is not a FAT filesystem 28 | :rtype: bool 29 | """ 30 | try: 31 | fs_type = get_filesystem_type(stream) 32 | if fs_type == 'FAT12' \ 33 | or fs_type == 'FAT16' \ 34 | or fs_type == 'FAT32': 35 | return True 36 | except UnsupportedFilesystemError: 37 | pass 38 | return False 39 | 40 | 41 | def get_filesystem_type(stream: typing.BinaryIO) -> str: 42 | """ 43 | extracts the FAT filesystem type from a given stream 44 | :stream: stream of filesystem 45 | :return: string, 'FAT12', 'FAT16' or 'FAT32' 46 | :raises: UnsupportedFilesystemError 47 | :rtype: str 48 | """ 49 | # save stream offset 50 | offset = stream.tell() 51 | 52 | # check if it is a FAT12 53 | stream.seek(offset + 54) 54 | fat_type = stream.read(8) 55 | if fat_type == b'FAT12 ': 56 | # reapply original stream position 57 | stream.seek(offset) 58 | return "FAT12" 59 | 60 | # check if it is a FAT16 61 | elif fat_type == b'FAT16 ': 62 | # reapply original stream position 63 | stream.seek(offset) 64 | return "FAT16" 65 | 66 | # check if its FAT32 67 | stream.seek(offset + 82) 68 | fat_type = stream.read(8) 69 | if fat_type == b'FAT32 ': 70 | # check if its real FAT32 or fatplus 71 | stream.seek(offset + 42) 72 | fat_version = int.from_bytes(stream.read(2), byteorder='little') 73 | if fat_version == 0: 74 | # yes its fat32 75 | stream.seek(offset) 76 | return "FAT32" 77 | elif fat_version == 1: 78 | # No its fat+ 79 | stream.seek(offset) 80 | raise UnsupportedFilesystemError("FAT+ is currently not supported") 81 | else: 82 | stream.seek(offset) 83 | raise UnsupportedFilesystemError("Could not detect filesystem") 84 | 85 | # reapply original position 86 | stream.seek(offset) 87 | -------------------------------------------------------------------------------- /fishy/ntfs/ntfs_filesystem/attributes.py: -------------------------------------------------------------------------------- 1 | """ 2 | Declaration of the sctructure of attributes 3 | """ 4 | 5 | from construct import Struct, Byte, Bytes, Int8ul, Int16ul, Int32ul, Int64ul, this 6 | from .record_reference import RECORD_REFERENCE 7 | 8 | STANDARD_INFORMATION_ID = 16 9 | 10 | ATTRIBUTE_LIST_ID = 32 11 | ATTRIBUTE_LIST_ENTRY = Struct( 12 | "attribute_type" / Byte, 13 | "record_length" / Int16ul, 14 | "attribute_name_length" / Int8ul, 15 | "attribute_name_offset" / Int8ul, 16 | "lowest_vcn" / Int64ul, 17 | "record_reference" / RECORD_REFERENCE, 18 | "reserved" / Bytes(2), 19 | "attribute_name" / Bytes(2*this.attribute_name_length) 20 | ) 21 | 22 | FILE_NAME_ID = 48 23 | FILE_NAME_ATTRIBUTE = Struct( 24 | "parent_directory" / RECORD_REFERENCE, 25 | "creation_time" / Int64ul, 26 | "last_modified" / Int64ul, 27 | "last_modifed_for_file_record" / Int64ul, 28 | "last_access_time" / Int64ul, 29 | "allocated_size_of_file" / Int64ul, 30 | "real_file_size" / Int64ul, 31 | "file_flags" / Bytes(4), 32 | "reparse_options" / Bytes(4), 33 | "file_name_length" / Int8ul, 34 | "namespace" / Byte, 35 | "file_name" / Bytes(2*this.file_name_length) 36 | ) 37 | 38 | DATA_ID = 128 39 | INDEX_ROOT_ID = 144 40 | INDEX_ALLOCATION_ID = 160 41 | INDEX_ROOT = Struct( 42 | "type" / Bytes(4), 43 | "collation_rule" / Bytes(4), 44 | "index_entry_size" / Int32ul, 45 | "cluster_per_index_record" / Byte, 46 | "padding" / Bytes(3) 47 | ) 48 | 49 | INDEX_HEADER = Struct( 50 | "first_entry_offset" / Int32ul, 51 | "total_size" / Int32ul, 52 | "allocated_size" / Int32ul, 53 | "flags" / Byte, 54 | "padding" / Bytes(3) 55 | ) 56 | 57 | INDEX_RECORD_HEADER = Struct( 58 | "signature" / Bytes(4), 59 | "update_sequence_array_offset" / Int16ul, 60 | "update_sequence_array_size" / Int16ul, 61 | "log_file_sequence" / Int64ul, 62 | "lowest_vcn" / Int64ul, 63 | "index_entry_offset" / Int32ul, 64 | "size_of_entries" / Int32ul, 65 | "size_of_entry_alloc" / Int32ul, 66 | "flags" / Byte, 67 | "reserved" / Bytes(3), 68 | "update_sequence" / Bytes(2*this.update_sequence_array_size) 69 | ) 70 | 71 | INDEX_RECORD_ENTRY = Struct( 72 | "record_reference" / RECORD_REFERENCE, 73 | "size_of_index_entry" / Int16ul, 74 | "file_name_offset" / Int16ul, 75 | "flags" / Bytes(2), 76 | "reserved" / Bytes(2), 77 | "record_reference_of_parent" / RECORD_REFERENCE, 78 | "creation_time" / Int64ul, 79 | "last_modified" / Int64ul, 80 | "last_modifed_for_file_record" / Int64ul, 81 | "last_access_time" / Int64ul, 82 | "allocated_size_of_file" / Int64ul, 83 | "real_file_size" / Int64ul, 84 | "file_flags" / Int64ul, 85 | "file_name_length" / Byte, 86 | "file_name_namespace" / Byte, 87 | "file_name" / Bytes(2*this.file_name_length) 88 | ) 89 | 90 | BITMAP_ID = 176 91 | 92 | -------------------------------------------------------------------------------- /fishy/fat/fat_filesystem/bootsector.py: -------------------------------------------------------------------------------- 1 | """ 2 | Bootsector definition for FAT12, FAT16 and FAT32 3 | This structure is part of the reserved sectors section. 4 | """ 5 | 6 | from construct import Struct, Byte, Bytes, Int16ul, Int32ul, Padding, this, \ 7 | Embedded, BitStruct, Nibble, Flag 8 | from .fs_info import FS_INFORMATION_SECTOR 9 | 10 | 11 | # Core Bootsector, which is the same for all FAT types 12 | FAT_CORE_BOOTSECTOR = Struct( 13 | "jump_instruction" / Bytes(3), 14 | "oem_name" / Bytes(8), 15 | "sector_size" / Int16ul, 16 | "sectors_per_cluster" / Byte, 17 | "reserved_sector_count" / Int16ul, 18 | "fat_count" / Byte, 19 | "rootdir_entry_count" / Int16ul, 20 | "sectorCount_small" / Int16ul, 21 | "media_descriptor_byte" / Byte, 22 | "sectors_per_fat" / Int16ul, 23 | "sectors_per_track" / Int16ul, 24 | "side_count" / Int16ul, 25 | "hidden_sectors_count" / Int32ul, 26 | "sectorCount_large" / Int32ul, 27 | ) 28 | 29 | # FAT12 and FAT16 bootsector extension 30 | FAT12_16_EXTENDED_BOOTSECTOR = Struct( 31 | "physical_drive_number" / Byte, 32 | Padding(1), # reserved 33 | "extended_boot_signature" / Byte, 34 | "volume_id" / Bytes(4), 35 | "volume_label" / Bytes(11), 36 | "fs_type" / Bytes(8), 37 | "boot_code" / Bytes(448), 38 | "boot_sector_signature" / Bytes(2), 39 | ) 40 | 41 | # FAT32 bootsector extension 42 | FAT32_EXTENDED_BOOTSECTOR = Struct( 43 | "sectors_per_fat" / Int32ul, 44 | "flags" / BitStruct( 45 | "active_fat" / Nibble, # only interesting if fat is not mirrored 46 | "mirrored" / Flag, 47 | Padding(3), 48 | Padding(8) 49 | ), 50 | "version" / Int16ul, 51 | "rootdir_cluster" / Int32ul, 52 | "fsinfo_sector" / Int16ul, 53 | "bootsector_copy_sector" / Int16ul, 54 | Padding(12), # reserved 55 | "physical_drive_number" / Byte, 56 | Padding(1), # reserved 57 | "extended_bootsignature" / Byte, 58 | "volume_id" / Int32ul, 59 | "volume_label" / Bytes(11), 60 | "fs_type" / Bytes(8), 61 | "boot_code" / Bytes(420), 62 | "boot_sector_signature" / Bytes(2), 63 | ) 64 | 65 | # ready to use bootsector definition for FAT12 and FAT16 66 | FAT12_16_BOOTSECTOR = Struct( 67 | Embedded(FAT_CORE_BOOTSECTOR), 68 | Embedded(FAT12_16_EXTENDED_BOOTSECTOR), 69 | Padding(this.sector_size - FAT_CORE_BOOTSECTOR.sizeof() - 70 | FAT12_16_EXTENDED_BOOTSECTOR.sizeof()), 71 | ) 72 | 73 | # ready to use bootsector definition for FAT32 74 | FAT32_BOOTSECTOR = Struct( 75 | Embedded(FAT_CORE_BOOTSECTOR), 76 | Embedded(FAT32_EXTENDED_BOOTSECTOR), 77 | Padding(this.sector_size - FAT_CORE_BOOTSECTOR.sizeof() 78 | - FAT32_EXTENDED_BOOTSECTOR.sizeof() 79 | ), 80 | Embedded(FS_INFORMATION_SECTOR), 81 | Padding(this.sector_size 82 | - FS_INFORMATION_SECTOR.sizeof() 83 | ), 84 | ) 85 | -------------------------------------------------------------------------------- /fishy/wrapper/write_gen.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import typing as typ 3 | from os import path 4 | from fishy.filesystem_detector import get_filesystem_type 5 | from fishy.metadata import Metadata 6 | from fishy.APFS.Write_Gen import APFSWriteGen 7 | from fishy.APFS.Write_Gen import APFSWriteGenMetadata 8 | 9 | LOGGER = logging.getLogger("write_gen") 10 | 11 | class write_gen: 12 | 13 | def __init__(self, fs_stream: typ.BinaryIO, metadata: Metadata, dev: str = None): 14 | self.dev = dev 15 | self.metadata = metadata 16 | self.fs_type = get_filesystem_type(fs_stream) 17 | if self.fs_type == 'APFS': 18 | self.metadata.set_module("APFS-Write-Gen") 19 | self.fs = APFSWriteGen(fs_stream) # pylint: disable=invalid-name 20 | else: 21 | raise NotImplementedError() 22 | 23 | def write(self, instream: typ.BinaryIO, 24 | filename: str = None) -> None: 25 | LOGGER.info("Write") 26 | if filename is not None: 27 | filename = path.basename(filename) 28 | if self.fs_type == 'APFS': 29 | LOGGER.info("Write into APFS") 30 | write_gen_metadata = self.fs.write(instream) 31 | self.metadata.add_file(filename, write_gen_metadata) 32 | else: 33 | raise NotImplementedError() 34 | 35 | def read(self, outstream: typ.BinaryIO): 36 | file_metadata = self.metadata.get_file("0")['metadata'] 37 | if self.fs_type == 'APFS': 38 | write_gen_metadata = APFSWriteGenMetadata(file_metadata) 39 | self.fs.read(outstream, write_gen_metadata) 40 | else: 41 | raise NotImplementedError() 42 | 43 | def read_into_file(self, outfilepath: str): 44 | if self.fs_type == 'APFS': 45 | with open(outfilepath, 'wb+') as outfile: 46 | self.read(outfile) 47 | else: 48 | raise NotImplementedError() 49 | 50 | def clear(self): 51 | if self.fs_type == 'APFS': 52 | for file_entry in self.metadata.get_files(): 53 | file_metadata = file_entry['metadata'] 54 | file_metadata = APFSWriteGenMetadata(file_metadata) 55 | self.fs.clear(file_metadata) 56 | else: 57 | raise NotImplementedError() 58 | 59 | 60 | def info(self): 61 | """ 62 | shows info about superblock slack and data hiding space 63 | :param metadata: Metadata, object where metadata is stored in 64 | :raises: NotImplementedError 65 | """ 66 | if self.fs_type == 'APFS': 67 | if len(list(self.metadata.get_files())) > 0: 68 | for file_entry in self.metadata.get_files(): 69 | file_metadata = file_entry['metadata'] 70 | file_metadata = APFSWriteGenMetadata(file_metadata) 71 | self.fs.info(file_metadata) 72 | else: 73 | self.fs.info() 74 | else: 75 | raise NotImplementedError() 76 | -------------------------------------------------------------------------------- /fishy/wrapper/inode_padding.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import typing as typ 3 | from os import path 4 | from fishy.filesystem_detector import get_filesystem_type 5 | from fishy.metadata import Metadata 6 | from fishy.APFS.Inode_Padding import APFSInodePadding 7 | from fishy.APFS.Inode_Padding import APFSInodePaddingMetadata 8 | 9 | LOGGER = logging.getLogger("write_gen") 10 | 11 | class inodePadding: 12 | 13 | def __init__(self, fs_stream: typ.BinaryIO, metadata: Metadata, dev: str = None): 14 | self.dev = dev 15 | self.metadata = metadata 16 | self.fs_type = get_filesystem_type(fs_stream) 17 | if self.fs_type == 'APFS': 18 | self.metadata.set_module("APFS-Inode-Padding") 19 | self.fs = APFSInodePadding(fs_stream) # pylint: disable=invalid-name 20 | else: 21 | raise NotImplementedError() 22 | 23 | def write(self, instream: typ.BinaryIO, 24 | filename: str = None) -> None: 25 | LOGGER.info("Write") 26 | if filename is not None: 27 | filename = path.basename(filename) 28 | if self.fs_type == 'APFS': 29 | LOGGER.info("Write into APFS") 30 | inode_padding_metadata = self.fs.write(instream) 31 | self.metadata.add_file(filename, inode_padding_metadata) 32 | else: 33 | raise NotImplementedError() 34 | 35 | def read(self, outstream: typ.BinaryIO): 36 | file_metadata = self.metadata.get_file("0")['metadata'] 37 | if self.fs_type == 'APFS': 38 | inode_padding_metadata = APFSInodePaddingMetadata(file_metadata) 39 | self.fs.read(outstream, inode_padding_metadata) 40 | else: 41 | raise NotImplementedError() 42 | 43 | def read_into_file(self, outfilepath: str): 44 | if self.fs_type == 'APFS': 45 | with open(outfilepath, 'wb+') as outfile: 46 | self.read(outfile) 47 | else: 48 | raise NotImplementedError() 49 | 50 | def clear(self): 51 | if self.fs_type == 'APFS': 52 | for file_entry in self.metadata.get_files(): 53 | file_metadata = file_entry['metadata'] 54 | file_metadata = APFSInodePaddingMetadata(file_metadata) 55 | self.fs.clear(file_metadata) 56 | else: 57 | raise NotImplementedError() 58 | 59 | def info(self): 60 | """ 61 | shows info about superblock slack and data hiding space 62 | :param metadata: Metadata, object where metadata is stored in 63 | :raises: NotImplementedError 64 | """ 65 | if self.fs_type == 'APFS': 66 | if len(list(self.metadata.get_files())) > 0: 67 | for file_entry in self.metadata.get_files(): 68 | file_metadata = file_entry['metadata'] 69 | file_metadata = APFSInodePaddingMetadata(file_metadata) 70 | self.fs.info(file_metadata) 71 | else: 72 | self.fs.info() 73 | else: 74 | raise NotImplementedError() 75 | -------------------------------------------------------------------------------- /fishy/wrapper/xfield_padding.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import typing as typ 3 | from os import path 4 | from fishy.filesystem_detector import get_filesystem_type 5 | from fishy.metadata import Metadata 6 | from fishy.APFS.Xfield_Padding import APFSXfieldPadding 7 | from fishy.APFS.Xfield_Padding import APFSXfieldPaddingMetadata 8 | 9 | LOGGER = logging.getLogger("xfield_padding") 10 | 11 | class xfieldPadding: 12 | 13 | def __init__(self, fs_stream: typ.BinaryIO, metadata: Metadata, dev: str = None): 14 | self.dev = dev 15 | self.metadata = metadata 16 | self.fs_type = get_filesystem_type(fs_stream) 17 | if self.fs_type == 'APFS': 18 | self.metadata.set_module("APFS-Xfield-Padding") 19 | self.fs = APFSXfieldPadding(fs_stream) # pylint: disable=invalid-name 20 | else: 21 | raise NotImplementedError() 22 | 23 | def write(self, instream: typ.BinaryIO, 24 | filename: str = None) -> None: 25 | LOGGER.info("Write") 26 | if filename is not None: 27 | filename = path.basename(filename) 28 | if self.fs_type == 'APFS': 29 | LOGGER.info("Write into APFS") 30 | xfield_metadata = self.fs.write(instream) 31 | self.metadata.add_file(filename, xfield_metadata) 32 | else: 33 | raise NotImplementedError() 34 | 35 | def read(self, outstream: typ.BinaryIO): 36 | file_metadata = self.metadata.get_file("0")['metadata'] 37 | if self.fs_type == 'APFS': 38 | xfield_metadata = APFSXfieldPaddingMetadata(file_metadata) 39 | self.fs.read(outstream, xfield_metadata) 40 | else: 41 | raise NotImplementedError() 42 | 43 | def read_into_file(self, outfilepath: str): 44 | if self.fs_type == 'APFS': 45 | with open(outfilepath, 'wb+') as outfile: 46 | self.read(outfile) 47 | else: 48 | raise NotImplementedError() 49 | 50 | def clear(self): 51 | if self.fs_type == 'APFS': 52 | for file_entry in self.metadata.get_files(): 53 | file_metadata = file_entry['metadata'] 54 | file_metadata = APFSXfieldPaddingMetadata(file_metadata) 55 | self.fs.clear(file_metadata) 56 | else: 57 | raise NotImplementedError() 58 | 59 | 60 | 61 | def info(self): 62 | """ 63 | shows info about superblock slack and data hiding space 64 | :param metadata: Metadata, object where metadata is stored in 65 | :raises: NotImplementedError 66 | """ 67 | if self.fs_type == 'APFS': 68 | if len(list(self.metadata.get_files())) > 0: 69 | for file_entry in self.metadata.get_files(): 70 | file_metadata = file_entry['metadata'] 71 | file_metadata = APFSXfieldPaddingMetadata(file_metadata) 72 | self.fs.info(file_metadata) 73 | else: 74 | self.fs.info() 75 | else: 76 | raise NotImplementedError() 77 | -------------------------------------------------------------------------------- /fishy/wrapper/timestamp_hiding.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import typing as typ 3 | from os import path 4 | from fishy.filesystem_detector import get_filesystem_type 5 | from fishy.metadata import Metadata 6 | from fishy.APFS.Timestamp_Hiding import APFSTimestampHiding 7 | from fishy.APFS.Timestamp_Hiding import APFSTimestampHidingMetadata 8 | 9 | LOGGER = logging.getLogger("write_gen") 10 | 11 | class timestampHiding: 12 | 13 | def __init__(self, fs_stream: typ.BinaryIO, metadata: Metadata, dev: str = None): 14 | self.dev = dev 15 | self.metadata = metadata 16 | self.fs_type = get_filesystem_type(fs_stream) 17 | if self.fs_type == 'APFS': 18 | self.metadata.set_module("APFS-Timestamp-Hiding") 19 | self.fs = APFSTimestampHiding(fs_stream) # pylint: disable=invalid-name 20 | else: 21 | raise NotImplementedError() 22 | 23 | def write(self, instream: typ.BinaryIO, 24 | filename: str = None) -> None: 25 | LOGGER.info("Write") 26 | if filename is not None: 27 | filename = path.basename(filename) 28 | if self.fs_type == 'APFS': 29 | LOGGER.info("Write into APFS") 30 | timestamp_metadata = self.fs.write(instream) 31 | self.metadata.add_file(filename, timestamp_metadata) 32 | else: 33 | raise NotImplementedError() 34 | 35 | def read(self, outstream: typ.BinaryIO): 36 | file_metadata = self.metadata.get_file("0")['metadata'] 37 | if self.fs_type == 'APFS': 38 | timestamp_metadata = APFSTimestampHidingMetadata(file_metadata) 39 | self.fs.read(outstream, timestamp_metadata) 40 | else: 41 | raise NotImplementedError() 42 | 43 | def read_into_file(self, outfilepath: str): 44 | if self.fs_type == 'APFS': 45 | with open(outfilepath, 'wb+') as outfile: 46 | self.read(outfile) 47 | else: 48 | raise NotImplementedError() 49 | 50 | def clear(self): 51 | if self.fs_type == 'APFS': 52 | for file_entry in self.metadata.get_files(): 53 | file_metadata = file_entry['metadata'] 54 | file_metadata = APFSTimestampHidingMetadata(file_metadata) 55 | self.fs.clear(file_metadata) 56 | else: 57 | raise NotImplementedError() 58 | 59 | 60 | def info(self): 61 | """ 62 | shows info about superblock slack and data hiding space 63 | :param metadata: Metadata, object where metadata is stored in 64 | :raises: NotImplementedError 65 | """ 66 | if self.fs_type == 'APFS': 67 | if len(list(self.metadata.get_files())) > 0: 68 | for file_entry in self.metadata.get_files(): 69 | file_metadata = file_entry['metadata'] 70 | file_metadata = APFSTimestampHidingMetadata(file_metadata) 71 | self.fs.info(file_metadata) 72 | else: 73 | self.fs.info() 74 | else: 75 | raise NotImplementedError() 76 | -------------------------------------------------------------------------------- /fishy/fat/fat_filesystem/fattools.py: -------------------------------------------------------------------------------- 1 | """ 2 | FATtools implements some common operations on 3 | FAT filesystems. 4 | """ 5 | 6 | 7 | class FATtools: 8 | """ 9 | FATtools implements some common inspection operations on 10 | FAT filesystems. 11 | """ 12 | def __init__(self, fat): 13 | """ 14 | :param fat: FAT filesystem object 15 | """ 16 | self.fat = fat 17 | 18 | def list_directory(self, cluster_id=0): 19 | """ 20 | list directory entries 21 | :param cluster_id: id of cluster that contains the 22 | directory. If cluster_id == 0 23 | then the root directory is listed 24 | """ 25 | if cluster_id == 0: 26 | dir_iterator = self.fat.get_root_dir_entries() 27 | else: 28 | dir_iterator = self.fat.get_dir_entries(cluster_id) 29 | for entry in dir_iterator: 30 | # get correct filetype 31 | if entry.is_dir(): 32 | filetype = 'd' 33 | else: 34 | filetype = 'f' 35 | # check if file is marked as deleted 36 | if entry.is_deleted(): 37 | deleted = 'd' 38 | else: 39 | deleted = ' ' 40 | # check if it is a dot entry 41 | if entry.is_dot_entry(): 42 | dot = '.' 43 | else: 44 | dot = ' ' 45 | print(filetype, deleted, dot, 46 | str(entry.get_start_cluster()).ljust(8), 47 | str(entry.get_filesize()).ljust(8), 48 | entry.get_name() 49 | ) 50 | 51 | def list_fat(self): 52 | """ 53 | list all fat entries 54 | """ 55 | for i in range(self.fat.entries_per_fat): 56 | print(i, self.fat.get_cluster_value(i)) 57 | 58 | def list_info(self): 59 | """ 60 | shows some info about the FAT filesystem 61 | """ 62 | # get FAT Type: 63 | fat_type = self.fat.pre.fs_type.decode('ascii').strip() 64 | print('FAT Type:'.ljust(42), 65 | fat_type) 66 | # Sector Size 67 | print('Sector Size:'.ljust(42), 68 | self.fat.pre.sector_size) 69 | # secors per cluster 70 | print('Sectors per Cluster:'.ljust(42), 71 | self.fat.pre.sectors_per_cluster) 72 | # sectors per fat 73 | print('Sectors per FAT:'.ljust(42), 74 | self.fat.pre.sectors_per_fat) 75 | # fat count 76 | print('FAT Count:'.ljust(42), 77 | self.fat.pre.fat_count) 78 | # Start of dataregion 79 | print('Dataregion Start Byte:'.ljust(42), 80 | self.fat.start_dataregion) 81 | # FAT32 specific 82 | if fat_type == 'FAT32': 83 | print("Free Data Clusters (FS Info):".ljust(42), 84 | self.fat.pre.free_data_cluster_count) 85 | print("Recently Allocated Data Cluster (FS Info):".ljust(42), 86 | self.fat.pre.last_allocated_data_cluster) 87 | print("Root Directory Cluster:".ljust(42), 88 | self.fat.pre.rootdir_cluster) 89 | print("FAT Mirrored:".ljust(42), 90 | self.fat.pre.flags.mirrored) 91 | print("Active FAT:".ljust(42), 92 | self.fat.pre.flags.active_fat) 93 | print("Sector of Bootsector Copy:".ljust(42), 94 | self.fat.pre.bootsector_copy_sector) 95 | -------------------------------------------------------------------------------- /doc/source/07_1_create_testfs.rst: -------------------------------------------------------------------------------- 1 | Create test images with `create_testfs.sh` 2 | ------------------------------------------ 3 | 4 | FAT and NTFS 5 | ............ 6 | 7 | With `create_testfs.sh` you can create prepared filesystem images. These 8 | already include files, which get copied from `utils/fs-files/`. 9 | These file systems are intended to be used by unit tests and for developing 10 | a new hiding technique. 11 | 12 | The script requires: 13 | 14 | * `sudo` to get mount permissions while creating images 15 | * `mount` and `umount` executables 16 | * `mkfs.vfat`, `mkfs.ntfs`, `mkfs.ext4` 17 | * `dd` for creating an empty image file 18 | 19 | To create a set of test images, simply run 20 | 21 | .. code:: bash 22 | 23 | $ ./create_testfs.sh 24 | 25 | 26 | The script is capable of handling "branches" to generate multiple images with a 27 | different filestructure. These are especially useful for writing unit tests 28 | that expect a certain file structure on the tested filesystem. 29 | 30 | If you would like to use existing test images while running unit tests, create 31 | a file called `.create_testfs.conf` under `utils`. Here you can define the 32 | variable `copyfrom` to provide a directory, where your existing test images are 33 | located. For instance: 34 | 35 | .. code:: 36 | 37 | copyfrom="/my/image/folder" 38 | 39 | 40 | To build all images that might be necessary for unittests, run 41 | 42 | ... code bash 43 | $ ./create_testfs.sh -t all 44 | 45 | 46 | 47 | These files are currently included in the stable1 branch: 48 | 49 | .. code:: 50 | 51 | . 52 | regular file ├── another 53 | parse longfilenames in fat parser ├── areallylongfilenamethatiwanttoreadcorrectly.txt 54 | parse files greater than one cluster ├── long_file.txt 55 | test fail of writes into empty file slack ├── no_free_slack.txt 56 | regular directory ├── onedirectory 57 | test reading files in sub directories │   ├── afileinadirectory.txt 58 | parse long filenames in directories │   ├── areallylongfilenamethatialsowanttoreadassoonaspossible.txt 59 | test parsing files in nested sub dirs │   └── nested_directory 60 | test parsing files in nested sub dirs │   └── royce.txt 61 | test if recursive directory parsing works └── testfile.txt 62 | 63 | Ext4 64 | .... 65 | 66 | Currently, you have to generate the ext4 filesystem by hand. 67 | 68 | .. code:: 69 | 70 | dd if=/dev/zero of=file.img bs=4M count=250 71 | mkfs ext4 -F file.img 72 | sudo mkdir -p /tmp/mount_tmp/ && sudo mount -o loop,rw,sync file.img /tmp/mount_tmp 73 | sudo chmod -R ug+rw /tmp/mount_tmp 74 | sudo mv /tmp/mount_tmp/ 75 | 76 | 77 | APFS 78 | .... 79 | 80 | As of right now, you have manually create an APFS image using a macOS machine. This can be achieved through multiple means, 81 | though it might be the most comfortable to use an external tool like AutoDMG. An official guide to create .dmg images can be found here. 82 | Once you have acquired a .dmg image file, you need to convert it to a .dd raw image. This can be achieved following these steps: 83 | 84 | * Use sleuthkit's mmls command to find the starting point of the container. 85 | 86 | * Follow up by using sleuthkit's mmcat command. An example would be: mmcat apfs_image.dmg 4 > apfs_volume.dd In this example "apfs_image.dmg" would represent the name of the extracted image, "4" is the starting point found through mmls and "apfs_volume.dd" would be the name of the extracted image. -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | this file contains test fixtures which can be used by unittests 3 | """ 4 | import os 5 | import tempfile 6 | import subprocess 7 | import shutil 8 | import pytest 9 | 10 | THIS_DIR = os.path.dirname(os.path.abspath(__file__)) 11 | UTILSDIR = os.path.join(THIS_DIR, os.pardir, 'utils') 12 | 13 | 14 | 15 | @pytest.fixture(scope="module") 16 | def testfs_fat_stable1(): 17 | """ 18 | creates FAT filesystem test images 19 | :return: list of strings, containing paths to fat test images 20 | """ 21 | image_dir = tempfile.mkdtemp() 22 | 23 | image_paths = [ 24 | os.path.join(image_dir, 'testfs-fat12-stable1.dd'), 25 | os.path.join(image_dir, 'testfs-fat16-stable1.dd'), 26 | os.path.join(image_dir, 'testfs-fat32-stable1.dd'), 27 | ] 28 | 29 | # create test filesystems 30 | cmd = os.path.join(UTILSDIR, "create_testfs.sh") + " -w " + UTILSDIR \ 31 | + " -d " + image_dir + " -t " + "fat" + " -u -s '-stable1'" 32 | subprocess.call(cmd, stdout=subprocess.PIPE, 33 | stderr=subprocess.PIPE, 34 | shell=True) 35 | 36 | yield image_paths 37 | 38 | # remove created filesystem images 39 | shutil.rmtree(image_dir) 40 | 41 | @pytest.fixture(scope="module") 42 | def testfs_ntfs_stable1(): 43 | """ 44 | creates NFTS filesystem test images 45 | :return: list of strings, containing paths to ntfs test images 46 | """ 47 | image_dir = tempfile.mkdtemp() 48 | 49 | image_paths = [ 50 | os.path.join(image_dir, 'testfs-ntfs-stable1.dd'), 51 | ] 52 | 53 | # create test filesystems 54 | cmd = os.path.join(UTILSDIR, "create_testfs.sh") + " -w " + UTILSDIR \ 55 | + " -d " + image_dir + " -t " + "ntfs" + " -u -s '-stable1'" 56 | subprocess.call(cmd, stdout=subprocess.PIPE, 57 | stderr=subprocess.PIPE, 58 | shell=True) 59 | 60 | yield image_paths 61 | 62 | # remove created filesystem images 63 | shutil.rmtree(image_dir) 64 | 65 | @pytest.fixture(scope="module") 66 | def testfs_ntfs_stable2(): 67 | """ 68 | creates NFTS filesystem test images 69 | :return: list of strings, containing paths to ntfs test images 70 | """ 71 | image_dir = tempfile.mkdtemp() 72 | 73 | image_paths = [ 74 | os.path.join(image_dir, 'testfs-ntfs-stable2.dd'), 75 | ] 76 | 77 | # create test filesystems 78 | cmd = os.path.join(UTILSDIR, "create_testfs.sh") + " -w " + UTILSDIR \ 79 | + " -d " + image_dir + " -t " + "ntfs" + " -u -s '-stable2'" 80 | subprocess.call(cmd, stdout=subprocess.PIPE, 81 | stderr=subprocess.PIPE, 82 | shell=True) 83 | 84 | yield image_paths 85 | 86 | # remove created filesystem images 87 | shutil.rmtree(image_dir) 88 | 89 | @pytest.fixture(scope="module") 90 | def testfs_ext4_stable1(): 91 | """ 92 | creates ext4 filesystem test images 93 | :return: list of strings, containing paths to fat test images 94 | """ 95 | image_dir = tempfile.mkdtemp() 96 | 97 | image_paths = [ 98 | os.path.join(image_dir, 'testfs-ext4-stable1.dd'), 99 | ] 100 | 101 | # create test filesystems 102 | cmd = os.path.join(UTILSDIR, "create_testfs.sh") + " -w " + UTILSDIR \ 103 | + " -d " + image_dir + " -t " + "ext4" + " -u -s '-stable1'" 104 | subprocess.call(cmd, stdout=subprocess.PIPE, 105 | stderr=subprocess.PIPE, 106 | shell=True) 107 | 108 | yield image_paths 109 | 110 | # remove created filesystem images 111 | shutil.rmtree(image_dir) 112 | -------------------------------------------------------------------------------- /doc/source/01_introduction.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | `fishy` is a toolkit for filesystem based data hiding techniques implemented in 5 | Python. It collects various common exploitation methods that make use of 6 | existing data structures on the filesystem layer, for hiding data from 7 | conventional file access methods. This toolkit is intended to introduce people 8 | to the concept of established anti-forensic methods associated with data 9 | hiding. 10 | 11 | In our research regarding existing tools for filesystem based data hinding 12 | techniques we only came up with a hand full of tools. None of these provide a 13 | consistent interface for multiple filesystems and various hiding techniques. 14 | For most of them it seemed that development has been stopped. 15 | 16 | With this background, there is no currently active framework for filesystem 17 | based data hinding techniques, other than `fishy`. As `fishy` aimes to provide an 18 | easy to use framework for creating new hiding techniques, this project might be 19 | useful for all security researchers, which are concerned with data hiding. 20 | 21 | This toolkit provides a cli interface for hiding data via the command line. Also 22 | the implemented hiding techniques can be used in other projects by importing 23 | `fishy` as a library. Besides that, `fishy` can also act as a framework to easily 24 | implement custom hiding techniques. 25 | 26 | Limitations 27 | ----------- 28 | 29 | `fishy` is currently only tested to run under linux. Other operating systems may 30 | provide different functions to access low level devices. 31 | 32 | Although it is possible to hide multiple different files on the filesystem, 33 | `fishy` is currently not capable of managing them. So, it is up to the user to avoid 34 | overwritten data. 35 | 36 | `fishy` does not encrypt the data it hides. If the user needs encryption, it is 37 | up to him to apply the encryption before he hides the data with this tool. The same 38 | applies to data integrity functionality. 39 | 40 | See also 41 | -------- 42 | 43 | During our research we mainly found two tools that implement filesystem based 44 | data hiding techniques and that seemed to be in a broader use. First there is 45 | `bmap `_, a linux tool for hiding 46 | data in ntfs slack space. This project seems to have no active website and 47 | downloads of this tool can only found on some shady tool collection sites. 48 | 49 | The second tool we found was `slacker.exe 50 | `_, a windows 51 | tool for hiding data in ntfs slack space. This tool was developed by Bishop Fox 52 | and seems to be included into the metasploit framework, at some time. The actual 53 | download on their website is disabled. 54 | 55 | Documentation Overview 56 | ---------------------- 57 | 58 | This paragraph will provide a brief overview of this documentation, giving a short summary of its structure. 59 | You will be introduced to the paper by its abstract and introductory sections. 60 | The `getting started` section gives beginners a fast start with the tool. 61 | After this basic introduction on how this toolkit can be used, we give some background 62 | information about `filesystem datastructures` and a brief explanation of each implemented `hiding technique`. 63 | The following `architecture overview` gives an introduction to fishy's core design principles and structures. 64 | The `module reference` documents the most important modules and classes, which you 65 | might use, if you want to integrate fishy into your own projects. 66 | In the `evaluation` section, all implemented hiding techniques are shortly rated for 67 | their gained capacity, stability and their propability of detection. 68 | The `future work` section ends this documentation. 69 | -------------------------------------------------------------------------------- /doc/source/03_reference_api.rst: -------------------------------------------------------------------------------- 1 | Module Reference 2 | ================ 3 | 4 | Filesystem Implementations 5 | -------------------------- 6 | 7 | FAT Filesystem 8 | ************** 9 | 10 | This toolkit uses its own FAT implementation, supporting FAT12, FAT16 and 11 | FAT32. It implements most parts of parsing the filesystem and some basic write 12 | operations, which can be used by hiding techniques. 13 | 14 | To parse a filesystem image use the wrapper function `create_fat`. 15 | This parses a file stream and returns the appropriate FAT instance. 16 | 17 | .. code-block:: python 18 | 19 | from fishy.fat.fat_filesystem.fat_wrapper import create_fat 20 | 21 | f = open('testfs.dd', 'rb') 22 | fs = create_fat(f) 23 | 24 | 25 | This filesystem instance provides some important methods. 26 | 27 | .. autoclass:: fishy.fat.fat_filesystem.fat.FAT 28 | :members: 29 | 30 | NTFS Filesystem 31 | *************** 32 | 33 | This is an implementation of a parser for NTFS, wich can give various 34 | information about the low level structure and content of the filesystem. 35 | 36 | To parse a filesystem image you just need to provide the NTFS class with a stream 37 | of the image. 38 | 39 | .. code-block:: python 40 | 41 | from fishy.ntfs.ntfs_filesystem.ntfs import NTFS 42 | 43 | f = open('testfs.dd', 'rb') 44 | fs = NTFS(f) 45 | 46 | 47 | This instance of the NTFS class provides functions to get all parsed information 48 | about the filesystem 49 | 50 | .. autoclass:: fishy.ntfs.ntfs_filesystem.ntfs.NTFS 51 | :members: 52 | 53 | Metadata 54 | -------- 55 | 56 | .. automodule:: fishy.metadata 57 | :members: 58 | 59 | Hiding Techniques 60 | ----------------- 61 | 62 | FAT 63 | *** 64 | 65 | File Slack 66 | .......... 67 | 68 | .. automodule:: fishy.fat.file_slack 69 | :members: 70 | 71 | Additional Cluster Allocation 72 | ............................. 73 | 74 | .. automodule:: fishy.fat.cluster_allocator 75 | :members: 76 | 77 | Bad Cluster Allocation 78 | ...................... 79 | 80 | .. automodule:: fishy.fat.bad_cluster 81 | :members: 82 | 83 | NTFS 84 | **** 85 | 86 | File Slack 87 | .......... 88 | 89 | .. automodule:: fishy.ntfs.ntfs_file_slack 90 | :members: 91 | 92 | MFT Slack 93 | ......... 94 | 95 | .. automodule:: fishy.ntfs.ntfs_mft_slack 96 | :members: 97 | 98 | Bad Cluster Allocation 99 | ...................... 100 | 101 | .. automodule:: fishy.ntfs.bad_cluster 102 | :members: 103 | 104 | Ext4 105 | **** 106 | 107 | Reserved GDT Blocks 108 | ................... 109 | 110 | .. automodule:: fishy.ext4.reserved_gdt_blocks 111 | :members: 112 | 113 | Superblock Slack 114 | ................ 115 | 116 | .. automodule:: fishy.ext4.superblock_slack 117 | :members: 118 | 119 | File Slack 120 | .......... 121 | 122 | .. automodule:: fishy.ext4.ext4_file_slack 123 | :members: 124 | 125 | osd2 126 | .... 127 | 128 | .. automodule:: fishy.ext4.osd2 129 | :members: 130 | 131 | obso_faddr 132 | .......... 133 | 134 | .. automodule:: fishy.ext4.obso_faddr 135 | :members: 136 | 137 | APFS 138 | **** 139 | 140 | Superblock Slack 141 | ................ 142 | 143 | .. automodule:: fishy.APFS.Superblock_Slack 144 | :members: 145 | 146 | Inode Padding 147 | ............. 148 | 149 | .. automodule:: fishy.APFS.Inode_Padding 150 | :members: 151 | 152 | Write-Gen-Counter 153 | ................. 154 | 155 | .. automodule:: fishy.APFS.Write_Gen 156 | :members: 157 | 158 | Timestamp Hiding 159 | ................ 160 | 161 | .. automodule:: fishy.APFS.Timestamp_Hiding 162 | :members: 163 | 164 | Extended Field Padding 165 | ...................... 166 | 167 | .. automodule:: fishy.APFS.Xfield_Padding 168 | :members: 169 | 170 | -------------------------------------------------------------------------------- /fishy/ext4/ext4_filesystem/gdt.py: -------------------------------------------------------------------------------- 1 | import math 2 | 3 | from fishy.ext4.ext4_filesystem.parser import Parser 4 | 5 | 6 | class GDT: 7 | 8 | # represents a 32 bit group descriptor 9 | structure32 = { 10 | "block_bitmap_lo": {"offset": 0x0, "size": 4}, 11 | "inode_bitmap_lo": {"offset": 0x4, "size": 4}, 12 | "inode_table_lo": {"offset": 0x8, "size": 4}, 13 | "free_blocks_count_lo": {"offset": 0xC, "size": 2}, 14 | "free_inodes_count_lo": {"offset": 0xE, "size": 2}, 15 | "used_dirs_count_lo": {"offset": 0x10, "size": 2}, 16 | "flags": {"offset": 0x12, "size": 2}, 17 | "exclude_bitmap_lo": {"offset": 0x14, "size": 4}, 18 | "block_bitmap_csum_lo": {"offset": 0x18, "size": 2}, 19 | "inode_bitmap_csum_lo": {"offset": 0x1A, "size": 2}, 20 | "itable_unused_lo": {"offset": 0x1C, "size": 2}, 21 | "checksum": {"offset": 0x1E, "size": 2} 22 | } 23 | 24 | # represents a 64 bit group descriptor 25 | structure64 = { 26 | "block_bitmap_lo": {"offset": 0x0, "size": 4}, 27 | "inode_bitmap_lo": {"offset": 0x4, "size": 4}, 28 | "inode_table_lo": {"offset": 0x8, "size": 4}, 29 | "free_blocks_count_lo": {"offset": 0xC, "size": 2}, 30 | "free_inodes_count_lo": {"offset": 0xE, "size": 2}, 31 | "used_dirs_count_lo": {"offset": 0x10, "size": 2}, 32 | "flags": {"offset": 0x12, "size": 2}, 33 | "exclude_bitmap_lo": {"offset": 0x14, "size": 4}, 34 | "block_bitmap_csum_lo": {"offset": 0x18, "size": 2}, 35 | "inode_bitmap_csum_lo": {"offset": 0x1A, "size": 2}, 36 | "itable_unused_lo": {"offset": 0x1C, "size": 2}, 37 | "checksum": {"offset": 0x1E, "size": 2}, 38 | # these fields only exist if the 64bit feature is enabled and s_desc_size > 32. 39 | "block_bitmap_hi": {"offset": 0x20, "size": 4}, 40 | "inode_bitmap_hi": {"offset": 0x24, "size": 4}, 41 | "inode_table_hi": {"offset": 0x28, "size": 4}, 42 | "free_blocks_count_hi": {"offset": 0x2C, "size": 2}, 43 | "free_inodes_count_hi": {"offset": 0x2E, "size": 2}, 44 | "used_dirs_count_hi": {"offset": 0x30, "size": 2}, 45 | "itable_unused_hi": {"offset": 0x32, "size": 2}, 46 | "exclude_bitmap_hi": {"offset": 0x34, "size": 4}, 47 | "block_bitmap_csum_hi": {"offset": 0x38, "size": 2}, 48 | "inode_bitmap_csum_hi": {"offset": 0x3A, "size": 2}, 49 | "reserved": {"offset": 0x3C, "size": 4} 50 | } 51 | 52 | def __init__(self, fs_stream, superblock, blocksize): 53 | self.blocksize = blocksize 54 | if (int(superblock.data['feature_incompat'], 0) & 0x80) == 0x80: 55 | self.is_64bit = True 56 | else: 57 | self.is_64bit = False 58 | # contains a list of all group descriptors 59 | self.data = self.parse_gdt(fs_stream, superblock) 60 | 61 | def parse_gdt(self, fs_stream, superblock): 62 | total_block_count = superblock.data['total_block_count'] 63 | blocks_per_group = superblock.data['blocks_per_group'] 64 | total_block_group_count = int(math.ceil(total_block_count / blocks_per_group)) 65 | data = [] 66 | if self.blocksize == 1024: 67 | offset = 2048 68 | else: 69 | offset = self.blocksize 70 | 71 | if self.is_64bit: 72 | for i in range(0, total_block_group_count): 73 | data.append(Parser.parse(fs_stream, offset, 64, structure=self.structure64)) 74 | offset += 64 75 | else: 76 | for i in range(0, total_block_group_count): 77 | data.append(Parser.parse(fs_stream, offset, 32, structure=self.structure32)) 78 | offset += 32 79 | 80 | return data -------------------------------------------------------------------------------- /fishy/fat/fat_filesystem/fat_entry.py: -------------------------------------------------------------------------------- 1 | """ 2 | FAT12, FAT16 and FAT32 Cluster address definition. 3 | """ 4 | from construct import Pass, Mapping, Int16ul, Int32ul 5 | 6 | # As construct.Enum does not allow multiple integers mapping to one value, 7 | # these cluster address definitions are implemented by imitation the 8 | # construct.Enum function and returning a valid construct.Mapping instance. 9 | 10 | 11 | def get_12_bit_cluster_address() -> Mapping: 12 | """ 13 | Mapping of a FAT12 File Allocation Table Entry 14 | :note: don't use this Mapping to generate the actual bytes to store on 15 | filesystem, because they might depend on the next or previous value 16 | :rtype: construct.Mapping 17 | """ 18 | # subcon = Bitwise(Int12ul()) 19 | subcon = Int16ul 20 | default = Pass 21 | mapping = dict() 22 | mapping[0x0] = 'free_cluster' 23 | mapping[0x1] = 'last_cluster' 24 | mapping[0xff7] = 'bad_cluster' 25 | for i in range(0xff8, 0xfff + 1): 26 | mapping[i] = 'last_cluster' 27 | 28 | encmapping = mapping.copy() 29 | encmapping['free_cluster'] = 0x000 30 | encmapping['bad_cluster'] = 0xff7 31 | encmapping['last_cluster'] = 0xfff 32 | 33 | return Mapping(subcon, 34 | encoding=encmapping, 35 | decoding=mapping, 36 | encdefault=default, 37 | decdefault=default, 38 | ) 39 | # TODO: Construct compatibility update here: Mapping syntax/function changed (original functionality removed and Mapping merged with Symmetric Mapping function) -> alternative? also: decoders before encoders 40 | 41 | 42 | def get_16_bit_cluster_address() -> Mapping: 43 | """ 44 | Mapping of a FAT16 File Allocation Table Entry 45 | :rtype: construct.Mapping 46 | """ 47 | subcon = Int16ul 48 | default = Pass 49 | mapping = dict() 50 | mapping[0x0] = 'free_cluster' 51 | mapping[0x1] = 'last_cluster' 52 | mapping[0xfff7] = 'bad_cluster' 53 | for i in range(0xfff8, 0xffff + 1): 54 | mapping[i] = 'last_cluster' 55 | 56 | encmapping = mapping.copy() 57 | encmapping['free_cluster'] = 0x0000 58 | encmapping['bad_cluster'] = 0xfff7 59 | encmapping['last_cluster'] = 0xffff 60 | 61 | return Mapping(subcon, 62 | encoding=encmapping, 63 | decoding=mapping, 64 | encdefault=default, 65 | decdefault=default, 66 | ) 67 | # TODO: Construct compatibility update here: Mapping syntax/function changed (original functionality removed and Mapping merged with Symmetric Mapping function) -> alternative? also: decoders before encoders 68 | 69 | 70 | def get_32_bit_cluster_address() -> Mapping: 71 | """ 72 | Mapping of a FAT32 File Allocation Table Entry 73 | :rtype: construct.Mapping 74 | """ 75 | subcon = Int32ul 76 | default = Pass 77 | mapping = dict() 78 | mapping[0x0] = 'free_cluster' 79 | mapping[0x1] = 'last_cluster' 80 | mapping[0xffffff7] = 'bad_cluster' 81 | for i in range(0xffffff8, 0xfffffff + 1): 82 | mapping[i] = 'last_cluster' 83 | 84 | encmapping = mapping.copy() 85 | encmapping['free_cluster'] = 0x0000000 86 | encmapping['bad_cluster'] = 0xffffff7 87 | encmapping['last_cluster'] = 0xfffffff 88 | 89 | return Mapping(subcon, 90 | encoding=encmapping, 91 | decoding=mapping, 92 | encdefault=default, 93 | decdefault=default, 94 | ) 95 | # TODO: Construct compatibility update here: Mapping syntax/function changed (original functionality removed and Mapping merged with Symmetric Mapping function) -> alternative? also: decoders before encoders 96 | 97 | 98 | FAT12Entry = get_12_bit_cluster_address() # pylint: disable=invalid-name 99 | 100 | FAT16Entry = get_16_bit_cluster_address() # pylint: disable=invalid-name 101 | 102 | FAT32Entry = get_32_bit_cluster_address() # pylint: disable=invalid-name 103 | -------------------------------------------------------------------------------- /fishy/APFS/APFS_filesystem/Container_Superblock.py: -------------------------------------------------------------------------------- 1 | from fishy.APFS.APFS_filesystem.APFS_Parser import Parser 2 | 3 | BLOCK_SIZE = 4096 4 | # Assumed starting block size 5 | 6 | class Superblock: 7 | 8 | structure = { 9 | "magic": {"offset": 0x0, "size": 4, "format": "raw"}, 10 | "block_size": {"offset": 0x4, "size": 4}, 11 | "block_count": {"offset": 0x8, "size": 8}, 12 | "features": {"offset": 0x10, "size": 8}, 13 | "rom_features": {"offset": 0x18, "size": 8}, 14 | "incompatible_features": {"offset": 0x20, "size": 8}, 15 | "uuid": {"offset": 0x28, "size": 16, "format": "raw"}, 16 | "next_oid": {"offset": 0x38, "size": 8}, 17 | "next_xid": {"offset": 0x40, "size": 8}, 18 | "xp_desc_blocks": {"offset": 0x48, "size": 4}, 19 | "xp_data_blocks": {"offset": 0x4C, "size": 4}, 20 | "xp_desc_base": {"offset": 0x50, "size": 8}, 21 | "xp_data_base": {"offset": 0x58, "size": 8}, 22 | "xp_desc_len": {"offset": 0x60, "size": 4}, 23 | "xp_data_len": {"offset": 0x64, "size": 4}, 24 | "xp_desc_index": {"offset": 0x68, "size": 4}, 25 | "xp_desc_index_len": {"offset": 0x6C, "size": 4}, 26 | "xp_data_index": {"offset": 0x70, "size": 4}, 27 | "xp_data_index_len": {"offset": 0x74, "size": 4}, 28 | "spaceman_oid": {"offset": 0x78, "size": 8}, 29 | "omap_oid": {"offset": 0x80, "size": 8}, 30 | "reaper_oid": {"offset": 0x88, "size": 8}, 31 | "test_type": {"offset": 0x90, "size": 4}, 32 | "max_file_systems": {"offset": 0x94, "size": 4}, 33 | } 34 | 35 | def __init__(self, fs_stream, offset): 36 | self.data = self.parse_superblock(fs_stream, offset) 37 | self.block_size = self.data["block_size"] 38 | self.offset = offset 39 | 40 | def parse_superblock(self, fs_stream, offset): 41 | c = Parser.parse(fs_stream, offset+32, BLOCK_SIZE-32, structure=self.structure) 42 | i = 0 43 | v = c["max_file_systems"] 44 | blocksize = c["block_size"] 45 | 46 | structureLower = { 47 | "fs_oid "+str(i+1): {"offset": 0x98+(i*8), "size": 8} for i in range(0, v) 48 | } 49 | 50 | structureLowerTwo = { 51 | "nx_counter": {"offset": 0x98+(v*8), "size": 32*8}, # TODO Hier size nicht richtig? 52 | # "Array" of counters in reality, here just allocated space 53 | "nx_blocked_range": {"offset": 0x98+(v*8)+(32*8), "size": 16}, 54 | "nx_evict_mapping_tree": {"offset": 0x108+(v*8)+(32*8), "size": 8}, 55 | "nx_flags": {"offset": 0x110+(v*8)+(32*8), "size": 8}, 56 | "nx_efi_jumpstart": {"offset": 0x118+(v*8)+(32*8), "size": 8}, 57 | "nx_fusion_uuid": {"offset": 0x120+(v*8)+(32*8), "size": 16}, 58 | "nx_keylocker": {"offset": 0x130+(v*8)+(32*8), "size": 16}, 59 | "nx_ephemeral_info": {"offset": 0x140+(v*8)+(32*8), "size": 32}, 60 | # "Array" of Ephemeral info in reality, here just allocated space 61 | "test_oid": {"offset": 0x160+(v*8)+(32*8), "size": 8}, 62 | "nx_fusion_mt_oid": {"offset": 0x168+(v*8)+(32*8), "size": 8}, 63 | "nx_fusion_wbc_oid": {"offset": 0x170+(v*8)+(32*8), "size": 8}, 64 | "nx_fusion_wbc": {"offset": 0x178+(v*8)+(32*8), "size": 16} 65 | 66 | 67 | 68 | 69 | } 70 | 71 | struct = {**structureLower, **structureLowerTwo} 72 | 73 | e = Parser.parse(fs_stream, offset+32, blocksize-32, structure=struct) 74 | 75 | d = {**c, **e} 76 | 77 | fs_stream.seek(0) 78 | 79 | return d 80 | 81 | def getBlockSize(self): 82 | return self.block_size 83 | 84 | def getVolumes(self): 85 | i = 0 86 | v = self.data["max_file_systems"] 87 | volumes = [] 88 | for i in range(0, v): 89 | volumes.append(self.data["fs_oid "+str(i+1)]) 90 | 91 | return volumes 92 | 93 | def getObjectMapAdr(self): 94 | offset_blocks = self.data["omap_oid"] 95 | # offset in blocks 96 | offset = offset_blocks * self.block_size 97 | # offset in bytes 98 | return offset 99 | -------------------------------------------------------------------------------- /fishy/ext4/ext4_filesystem/inode.py: -------------------------------------------------------------------------------- 1 | from fishy.ext4.ext4_filesystem.parser import Parser 2 | 3 | class Inode: 4 | """ 5 | This class represents a single inode. 6 | """ 7 | 8 | # Structure of Superblock: 9 | # name: {offset: hex, size: in bytes, [format: python builtin function (e.g. hex, bin, etc...] } 10 | structure = { 11 | "mode": {"offset": 0x0, "size": 2, "format": "hex"}, 12 | "uid": {"offset": 0x2, "size": 2}, 13 | "size": {"offset": 0x4, "size": 4}, 14 | "atime": {"offset": 0x8, "size": 4, "format": "time"}, 15 | "ctime": {"offset": 0xC, "size": 4, "format": "time"}, 16 | "mtime": {"offset": 0x10, "size": 4, "format": "time"}, 17 | "dtime": {"offset": 0x14, "size": 4, "format": "time"}, 18 | "gid": {"offset": 0x18, "size": 2}, 19 | "links_count": {"offset": 0x1A, "size": 2}, 20 | "blocks": {"offset": 0x1C, "size": 4}, 21 | "flags": {"offset": 0x20, "size": 4, "format": "hex"}, 22 | "osd1": {"offset": 0x24, "size": 4, "format": "raw"}, 23 | "extent_tree": {"offset": 0x28, "size": 60, "format": "raw"}, 24 | "generation": {"offset": 0x64, "size": 4}, 25 | "file_acl": {"offset": 0x68, "size": 4}, 26 | "dir_acl": {"offset": 0x6C, "size": 4}, 27 | "obso_faddr": {"offset": 0x70, "size": 4}, 28 | "osd2": {"offset": 0x74, "size": 12, "format": "raw"}, 29 | "extra_isize": {"offset": 0x80, "size": 2}, 30 | "checksum_hi": {"offset": 0x82, "size": 2}, 31 | "ctime_extra": {"offset": 0x84, "size": 4}, 32 | "mtime_extra": {"offset": 0x88, "size": 4}, 33 | "atime_extra": {"offset": 0x8C, "size": 4}, 34 | "crtime": {"offset": 0x90, "size": 4}, 35 | "crtime_extra": {"offset": 0x94, "size": 4}, 36 | "version_hi": {"offset": 0x98, "size": 4}, 37 | "projid": {"offset": 0x9C, "size": 4}, 38 | } 39 | 40 | extent_tree_header = { 41 | "magic": {"offset": 0x0, "size": 2}, 42 | "entries": {"offset": 0x2, "size": 2}, 43 | "max": {"offset": 0x4, "size": 2}, 44 | "depth": {"offset": 0x6, "size": 2}, 45 | "generation": {"offset": 0x8, "size": 4}, 46 | } 47 | 48 | extent_internal_nodes = { 49 | "block": {"offset": 0x0, "size": 4}, 50 | "leaf_lo": {"offset": 0x4, "size": 4}, 51 | "leaf_hi": {"offset": 0x8, "size": 2}, 52 | "unused": {"offset": 0xA, "size": 2}, 53 | } 54 | 55 | extent_leaf_nodes = { 56 | "block": {"offset": 0x0, "size": 4}, 57 | "len": {"offset": 0x4, "size": 2}, 58 | "start_hi": {"offset": 0x6, "size": 2}, 59 | "start_lo": {"offset": 0x8, "size": 4}, 60 | } 61 | 62 | def __init__(self, fs_stream, offset, lenght, blocksize): 63 | self.blocksize = blocksize 64 | self.offset = offset 65 | self.length = lenght 66 | self.data = self.parse_inode(fs_stream) 67 | 68 | self.extents = self.parse_extents(self.data['extent_tree']) 69 | 70 | def parse_inode(self, filename): 71 | d = Parser.parse(filename, offset=self.offset, length=self.length, structure=self.structure) 72 | return d 73 | 74 | def parse_extents(self, tree): 75 | extent_data = {} 76 | 77 | header_part = tree[:12] 78 | 79 | header = {} 80 | for key, value in self.extent_tree_header.items(): 81 | field_offset = value["offset"] 82 | field_size = value["size"] 83 | bytes = header_part[field_offset:field_offset+field_size] 84 | header[key] = int.from_bytes(bytes, byteorder='little') 85 | 86 | extent_data["header"] = header 87 | 88 | if header["depth"] == 0: 89 | leaf_part = tree[12:24] 90 | data = {} 91 | for key, value in self.extent_leaf_nodes.items(): 92 | field_offset = value["offset"] 93 | field_size = value["size"] 94 | bytes = leaf_part[field_offset:field_offset+field_size] 95 | data[key] = int.from_bytes(bytes, byteorder='little') 96 | 97 | extent_data["data"] = data 98 | 99 | return extent_data 100 | -------------------------------------------------------------------------------- /fishy/APFS/APFS_filesystem/InodeTable.py: -------------------------------------------------------------------------------- 1 | # This table is NOT a real APFS structure; it is included so finding inodes for various tasks is done at a central point 2 | # this table also only finds current inodes; no possible previous checkpoint inodes are found yet, but with the volume 3 | # and volume map list from the checkpoints class this can be implemented in the future 4 | from fishy.APFS.APFS_filesystem.Container_Superblock import Superblock 5 | from fishy.APFS.APFS_filesystem.Node import Node 6 | #from fishy.APFS.APFS_filesystem.Volume_Superblock import vSuperblock 7 | from fishy.APFS.APFS_filesystem.Object_Map import ObjectMap 8 | 9 | 10 | class InodeTable: 11 | 12 | def __init__(self, fs_stream): 13 | self.mainContainerBlock = Superblock(fs_stream, 0) 14 | self.data = Superblock(fs_stream, 0).parse_superblock(fs_stream, 0) 15 | self.blocksize = self.data["block_size"] 16 | self.CMAP = ObjectMap(fs_stream, self.mainContainerBlock.getObjectMapAdr(), self.blocksize) 17 | self.cmapDATA = self.CMAP.parseObjectMap(fs_stream, self.mainContainerBlock.getObjectMapAdr()) 18 | self.cmapRootNode = self.cmapDATA["root"] 19 | 20 | 21 | def getAllInodes(self, fs_stream): 22 | # gets all Inodes from volumes 23 | inodeAddressList = [] 24 | # inodeAddressList contains address to block as well as offset to inode in tuple) 25 | listOfVolumes = self.CMAP.mapCObjectMap(fs_stream) 26 | # list of all volumes with their oids attached 27 | volumeMapList = [] 28 | rootNodeList = [] 29 | # lists of all volume maps and rootnodes 30 | for d, y in listOfVolumes: 31 | volumeMapList.append(d["omap_oid"]) 32 | rootNodeList.append(d["root_tree_oid"]) 33 | 34 | vmapRootNode = [] 35 | vmapList = [] 36 | for i in range (0, len(volumeMapList)): 37 | vmap = ObjectMap(fs_stream, volumeMapList[i]*self.blocksize, self.blocksize).parseObjectMap(fs_stream, 38 | volumeMapList[i] 39 | *self.blocksize) 40 | vmapList.append(vmap) 41 | #Liste alles Volume Object Maps 42 | 43 | for i in range (0, len(vmapList)): 44 | vrootnode = Node(fs_stream, vmapList[i]["root"]*self.blocksize, self.blocksize).parseNode(fs_stream, 45 | vmapList[i][ 46 | "root"] * self 47 | .blocksize) 48 | vmapRootNode.append(vrootnode) 49 | #Liste aller Volume Map Root Nodes 50 | 51 | 52 | 53 | oidList = [] 54 | # list of oid addresses linked to by volume omap root nodes 55 | for i in range(0, len(vmapRootNode)): 56 | entries = vmapRootNode[i]["entry_count"] 57 | for j in range(0, entries): 58 | oidList.append((vmapRootNode[i]["oid " + str(j)], vmapRootNode[i]["omv_paddr " + str(j)])) 59 | if oidList[0][0] == rootNodeList[i]: 60 | notNeeded = oidList.pop(0) 61 | #get rid of rootnode since it has nothing of value for inode table 62 | #TODO pop other root nodes 63 | 64 | for j in range(0, len(rootNodeList)): 65 | oidList = [i for i in oidList if i[0] != rootNodeList[j]] 66 | 67 | for i in range(0, len(oidList)): 68 | temp = Node(fs_stream, oidList[i][1]*self.blocksize, self.blocksize).parseNode(fs_stream, oidList[i][1]* 69 | self.blocksize) 70 | #Überprüft ob ein Entry eine Inode ist und fügt diese dann der Tupelliste als Tupel Node Adresse|Inode Adresse 71 | #hinzu 72 | for j in range(0, temp["entry_count"]): 73 | if temp["kind " + str(j)] >> 28 == 3: 74 | inodeAddressList.append((oidList[i][1]*self.blocksize, self.blocksize - 75 | temp["data_offset " + str(j)] - 40 * (temp["node_type"] & 1))) 76 | 77 | return inodeAddressList -------------------------------------------------------------------------------- /tests/test_ntfs.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file contains tests for the NTFS class 3 | """ 4 | 5 | import pytest 6 | from fishy.ntfs.ntfs_filesystem.ntfs import NTFS 7 | 8 | class TestGetBootsector(object): 9 | """ 10 | Tests if the bootsectors are parsed correctly 11 | """ 12 | def test_get_bootsector(self, testfs_ntfs_stable1): 13 | """ Tests for the main bootsector """ 14 | with open(testfs_ntfs_stable1[0], 'rb') as fs: 15 | ntfs = NTFS(fs) 16 | bootsector = ntfs.get_bootsector() 17 | assert bootsector.oem_name == b'NTFS ' 18 | assert bootsector.eos_marker == b'\x55\xaa' 19 | 20 | def test_get_bootsector_copy(self, testfs_ntfs_stable1): 21 | """ Tests for the bootsector copy """ 22 | with open(testfs_ntfs_stable1[0], 'rb') as fs: 23 | ntfs = NTFS(fs) 24 | bootsector = ntfs.get_bootsector_copy() 25 | assert bootsector.oem_name == b'NTFS ' 26 | assert bootsector.eos_marker == b'\x55\xaa' 27 | 28 | 29 | class TestBasicInformation(object): 30 | """ 31 | Tests if the basic information of the filesystem 32 | is parsed correctly 33 | """ 34 | def test_parse_bootsector(self, testfs_ntfs_stable1): 35 | """ 36 | Tests if the necessary information from the bootsector 37 | is parsed correctly 38 | """ 39 | with open(testfs_ntfs_stable1[0], 'rb') as fs: 40 | ntfs = NTFS(fs) 41 | assert ntfs.cluster_size == 4096 42 | assert ntfs.record_size == 1024 43 | assert ntfs.mft_offset == 16384 44 | 45 | 46 | def test_mft_info(self, testfs_ntfs_stable1): 47 | """ 48 | Tests the information about the mft itself 49 | """ 50 | with open(testfs_ntfs_stable1[0], 'rb') as fs: 51 | ntfs = NTFS(fs) 52 | assert ntfs.mft_runs == [{'length': 77824, 'offset': 16384}] 53 | 54 | class TestGetRecord(object): 55 | """ Tests if getting records works correctly """ 56 | def test_record_alignment(self, testfs_ntfs_stable1): 57 | """ 58 | Tests if the records start with the correct value 59 | """ 60 | with open(testfs_ntfs_stable1[0], 'rb') as fs: 61 | ntfs = NTFS(fs) 62 | assert ntfs.get_record(0)[0:4] == b'FILE' 63 | assert ntfs.get_record(25)[0:4] == b'FILE' 64 | assert ntfs.get_record(50)[0:4] == b'FILE' 65 | assert ntfs.get_record(100)[0:4] == b'FILE' 66 | 67 | def test_record_position(self, testfs_ntfs_stable1): 68 | """ 69 | Tests if the record returned is of the requested position 70 | """ 71 | with open(testfs_ntfs_stable1[0], 'rb') as fs: 72 | ntfs = NTFS(fs) 73 | assert ntfs.get_record(0)[0xf2:0xfa].decode('utf-16') == '$MFT' 74 | assert ntfs.get_record(1)[0xf2:0x0102].decode('utf-16') == '$MFTMirr' 75 | assert ntfs.get_record(8)[0xf2:0x0102].decode('utf-16') == '$BadClus' 76 | 77 | 78 | class TestGetRecordOfFile(object): 79 | """ Tests if getting records by filename/-path works correctly """ 80 | def test_get_record_of_file(self, testfs_ntfs_stable1): 81 | """ 82 | Tests if the correct record is returned for 83 | the supplied name 84 | """ 85 | with open(testfs_ntfs_stable1[0], 'rb') as fs: 86 | ntfs = NTFS(fs) 87 | assert ntfs.get_record_of_file('$MFT') == 0 88 | assert ntfs.get_record_of_file('$MFTMirr') == 1 89 | assert ntfs.get_record_of_file('$BadClus') == 8 90 | assert ntfs.get_record_of_file('$Extend/$Reparse') == 26 91 | assert ntfs.get_record_of_file('onedirectory/nested_directory/royce.txt') == 72 92 | assert ntfs.get_record_of_file('notexisting') == None 93 | 94 | 95 | class TestGetData(object): 96 | """ Tests if getting the data of a file works correctly """ 97 | def test_get_data(self, testfs_ntfs_stable1): 98 | """ 99 | Tests if the correct data is returned 100 | """ 101 | with open(testfs_ntfs_stable1[0], 'rb') as fs: 102 | ntfs = NTFS(fs) 103 | record_another = ntfs.get_record_of_file('another') 104 | record_long_file = ntfs.get_record_of_file('long_file.txt') 105 | assert ntfs.get_data(record_another) == b'222\n' 106 | assert ntfs.get_data(record_long_file) == b'1'*8000 + b'\n' 107 | -------------------------------------------------------------------------------- /utils/fs-files-stable2/other_file.txt: -------------------------------------------------------------------------------- 1 | 11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111 2 | -------------------------------------------------------------------------------- /fishy/wrapper/osd2.py: -------------------------------------------------------------------------------- 1 | """ 2 | Wrapper for filesystem specific implementations of hiding data inside reserved GDT blocks 3 | """ 4 | import logging 5 | import typing as typ 6 | from os import path 7 | from fishy.filesystem_detector import get_filesystem_type 8 | from fishy.metadata import Metadata 9 | from fishy.ext4.osd2 import EXT4OSD2 10 | from fishy.ext4.osd2 import EXT4OSD2Metadata 11 | 12 | LOGGER = logging.getLogger("ReservedGDTBlocks") 13 | 14 | class OSD2: 15 | """ 16 | This class wraps the filesystem specific implementation of the osd2 hiding technique 17 | """ 18 | def __init__(self, fs_stream: typ.BinaryIO, metadata: Metadata, 19 | dev: str = None): 20 | """ 21 | :param dev: Path to filesystem 22 | :param fs_stream: Stream of filesystem 23 | :param metadata: Metadata object 24 | """ 25 | self.dev = dev 26 | self.metadata = metadata 27 | self.fs_type = get_filesystem_type(fs_stream) 28 | if self.fs_type == 'EXT4': 29 | self.metadata.set_module("ext4-osd2") 30 | self.fs = EXT4OSD2(fs_stream, dev) # pylint: disable=invalid-name 31 | else: 32 | raise NotImplementedError() 33 | 34 | def write(self, instream: typ.BinaryIO, 35 | filename: str = None) -> None: 36 | """ 37 | writes data from instream into inodes' osd2 field. Metadata of 38 | those files will be stored in Metadata object 39 | 40 | :param instream: stream to read data from 41 | :param filename: name that will be used, when file gets written into inodes' osd2 field. 42 | If none, a random name will be generated. 43 | :raises: IOError 44 | """ 45 | LOGGER.info("Write") 46 | if filename is not None: 47 | filename = path.basename(filename) 48 | if self.fs_type == 'EXT4': 49 | LOGGER.info("Write into ext4") 50 | osd2_metadata = self.fs.write(instream) 51 | self.metadata.add_file(filename, osd2_metadata) 52 | else: 53 | raise NotImplementedError() 54 | 55 | def read(self, outstream: typ.BinaryIO): 56 | """ 57 | writes hidden data from inodes' osd2 field into stream. 58 | 59 | :param outstream: stream to write hidden data into osd2 fields 60 | :raises: IOError 61 | """ 62 | file_metadata = self.metadata.get_file("0")['metadata'] 63 | if self.fs_type == 'EXT4': 64 | osd2_metadata = EXT4OSD2Metadata(file_metadata) 65 | self.fs.read(outstream, osd2_metadata) 66 | else: 67 | raise NotImplementedError() 68 | 69 | def read_into_file(self, outfilepath: str): 70 | """ 71 | reads hidden data from inodes' osd2 field into files 72 | :note: If provided filepath already exists, this file will be 73 | overwritten without a warning. 74 | :param outfilepath: filepath to file, where hidden data will be 75 | restored into 76 | """ 77 | if self.fs_type == 'EXT4': 78 | with open(outfilepath, 'wb+') as outfile: 79 | self.read(outfile) 80 | else: 81 | raise NotImplementedError() 82 | 83 | def clear(self): 84 | """ 85 | clears inodes' osd2 field in which data has been hidden 86 | :param metadata: Metadata, object where metadata is stored in 87 | :raises: NotImplementedError 88 | """ 89 | if self.fs_type == 'EXT4': 90 | for file_entry in self.metadata.get_files(): 91 | file_metadata = file_entry['metadata'] 92 | file_metadata = EXT4OSD2Metadata(file_metadata) 93 | self.fs.clear(file_metadata) 94 | 95 | else: 96 | raise NotImplementedError() 97 | 98 | def info(self): 99 | """ 100 | shows info about inode osd2 fields and data hiding space 101 | :param metadata: Metadata, object where metadata is stored in 102 | :raises: NotImplementedError 103 | """ 104 | if self.fs_type == 'EXT4': 105 | if len(list(self.metadata.get_files())) > 0: 106 | for file_entry in self.metadata.get_files(): 107 | file_metadata = file_entry['metadata'] 108 | file_metadata = EXT4OSD2Metadata(file_metadata) 109 | self.fs.info(file_metadata) 110 | else: 111 | self.fs.info() 112 | else: 113 | raise NotImplementedError() 114 | -------------------------------------------------------------------------------- /fishy/wrapper/obso_faddr.py: -------------------------------------------------------------------------------- 1 | """ 2 | Wrapper for filesystem specific implementations of hiding data inside inodes' obso_faddr fields 3 | """ 4 | import logging 5 | import typing as typ 6 | from os import path 7 | from fishy.filesystem_detector import get_filesystem_type 8 | from fishy.metadata import Metadata 9 | from fishy.ext4.obso_faddr import EXT4FADDR 10 | from fishy.ext4.obso_faddr import EXT4FADDRMetadata 11 | 12 | LOGGER = logging.getLogger("obso_faddr") 13 | 14 | class FADDR: 15 | """ 16 | This class wraps the filesystem specific implementation of the obso_faddr hiding technique, 17 | which is an extension to the osd2 technique 18 | """ 19 | def __init__(self, fs_stream: typ.BinaryIO, metadata: Metadata, 20 | dev: str = None): 21 | """ 22 | :param dev: Path to filesystem 23 | :param fs_stream: Stream of filesystem 24 | :param metadata: Metadata object 25 | """ 26 | self.dev = dev 27 | self.metadata = metadata 28 | self.fs_type = get_filesystem_type(fs_stream) 29 | if self.fs_type == 'EXT4': 30 | self.metadata.set_module("ext4-faddr") 31 | self.fs = EXT4FADDR(fs_stream, dev) # pylint: disable=invalid-name 32 | else: 33 | raise NotImplementedError() 34 | 35 | def write(self, instream: typ.BinaryIO, 36 | filename: str = None) -> None: 37 | """ 38 | writes data from instream into obso_faddr fields. Metadata of 39 | those files will be stored in Metadata object 40 | 41 | :param instream: stream to read data from 42 | :param filename: name that will be used, when file gets written into obso_faddr fields. 43 | If none, a random name will be generated. 44 | :raises: IOError 45 | """ 46 | LOGGER.info("Write") 47 | if filename is not None: 48 | filename = path.basename(filename) 49 | if self.fs_type == 'EXT4': 50 | LOGGER.info("Write into ext4") 51 | obso_faddr_metadata = self.fs.write(instream) 52 | self.metadata.add_file(filename, obso_faddr_metadata) 53 | else: 54 | raise NotImplementedError() 55 | 56 | def read(self, outstream: typ.BinaryIO): 57 | """ 58 | writes hidden data from obso_faddr fields into stream. 59 | 60 | :param outstream: stream to write hidden data into 61 | :raises: IOError 62 | """ 63 | file_metadata = self.metadata.get_file("0")['metadata'] 64 | if self.fs_type == 'EXT4': 65 | obso_faddr_metadata = EXT4FADDRMetadata(file_metadata) 66 | self.fs.read(outstream, obso_faddr_metadata) 67 | else: 68 | raise NotImplementedError() 69 | 70 | def read_into_file(self, outfilepath: str): 71 | """ 72 | reads hidden data from obso_faddr fields into files 73 | :note: If provided filepath already exists, this file will be 74 | overwritten without a warning. 75 | :param outfilepath: filepath to file, where hidden data will be 76 | restored into 77 | """ 78 | if self.fs_type == 'EXT4': 79 | with open(outfilepath, 'wb+') as outfile: 80 | self.read(outfile) 81 | else: 82 | raise NotImplementedError() 83 | 84 | def clear(self): 85 | """ 86 | clears obso_faddr fields in which data has been hidden 87 | :param metadata: Metadata, object where metadata is stored in 88 | :raises: IOError 89 | """ 90 | if self.fs_type == 'EXT4': 91 | for file_entry in self.metadata.get_files(): 92 | file_metadata = file_entry['metadata'] 93 | file_metadata = EXT4FADDRMetadata(file_metadata) 94 | self.fs.clear(file_metadata) 95 | 96 | else: 97 | raise NotImplementedError() 98 | 99 | def info(self): 100 | """ 101 | shows info about inode obso_faddr fields and data hiding space 102 | :param metadata: Metadata, object where metadata is stored in 103 | :raises: NotImplementedError 104 | """ 105 | if self.fs_type == 'EXT4': 106 | if len(list(self.metadata.get_files())) > 0: 107 | for file_entry in self.metadata.get_files(): 108 | file_metadata = file_entry['metadata'] 109 | file_metadata = EXT4FADDRMetadata(file_metadata) 110 | self.fs.info(file_metadata) 111 | else: 112 | self.fs.info() 113 | else: 114 | raise NotImplementedError() 115 | -------------------------------------------------------------------------------- /fishy/fat/fat_filesystem/fat_32.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implementation of a FAT32 filesystem reader 3 | 4 | example usage: 5 | >>> with open('testfs.dd', 'rb') as filesystem: 6 | >>> fs = FAT32(filesystem) 7 | 8 | example to print all entries in root directory: 9 | >>> for i, v in fs.get_root_dir_entries(): 10 | >>> if v != "": 11 | >>> print(v) 12 | 13 | example to print all fat entries 14 | >>> for i in range(fs.entries_per_fat): 15 | >>> print(i,fs.get_cluster_value(i)) 16 | 17 | example to print all root directory entries 18 | >>> for entry in fs.get_root_dir_entries(): 19 | >>> print(entry.get_start_cluster()) 20 | 21 | """ 22 | import typing as typ 23 | from construct import Struct, Array, Padding, Embedded, Bytes, this 24 | from .bootsector import FAT32_BOOTSECTOR, FAT_CORE_BOOTSECTOR, \ 25 | FAT32_EXTENDED_BOOTSECTOR 26 | from .fat import FAT 27 | from .fat_entry import FAT32Entry 28 | 29 | 30 | FAT32_PRE_DATA_REGION = Struct( 31 | "bootsector" / Embedded(FAT32_BOOTSECTOR), 32 | Padding((this.reserved_sector_count - 2) * this.sector_size), 33 | # FATs 34 | "fats" / Array(this.fat_count, Bytes(this.sectors_per_fat * this.sector_size)), 35 | ) 36 | 37 | 38 | class FAT32(FAT): 39 | """ 40 | FAT32 filesystem implementation. 41 | """ 42 | def __init__(self, stream: typ.BinaryIO): 43 | """ 44 | :param stream: filedescriptor of a FAT32 filesystem 45 | """ 46 | super().__init__(stream, FAT32_PRE_DATA_REGION) 47 | self.entries_per_fat = int(self.pre.sectors_per_fat 48 | * self.pre.sector_size 49 | / 4) 50 | self._fat_entry = FAT32Entry 51 | self.fat_type = 'FAT32' 52 | 53 | def get_cluster_value(self, cluster_id: int) -> typ.Union[int, str]: 54 | """ 55 | finds the value that is written into fat 56 | for given cluster_id 57 | :param cluster_id: int, cluster that will be looked up 58 | :return: int or string 59 | """ 60 | byte = cluster_id*4 61 | # TODO: Use active FAT 62 | byte_slice = self.pre.fats[0][byte:byte+4] 63 | value = int.from_bytes(byte_slice, byteorder='little') 64 | # TODO: Remove highest 4 Bits as FAT32 uses only 28Bit 65 | # long addresses. 66 | return self._fat_entry.parse(value.to_bytes(4, 'little')) 67 | 68 | def _root_to_stream(self, stream: typ.BinaryIO) -> None: 69 | """ 70 | write root directory into a given stream 71 | :param stream: stream, where the root directory will be written into 72 | """ 73 | raise NotImplementedError 74 | 75 | def get_root_dir_entries(self) \ 76 | -> typ.Generator[typ.Tuple[Struct, str], None, None]: 77 | return self.get_dir_entries(self.pre.rootdir_cluster) 78 | 79 | def write_free_clusters(self, new_value: int) -> None: 80 | """ 81 | Write a new value to free_cluster_count field of FAT32 FS INFO sector 82 | :param new_value: int, new value for free_cluster_count 83 | """ 84 | # calculate start address of FS_INFO sector 85 | bootsector_size = FAT_CORE_BOOTSECTOR.sizeof() \ 86 | + FAT32_EXTENDED_BOOTSECTOR.sizeof() 87 | bootsector_padd = self.pre.sector_size - bootsector_size 88 | fs_info_start = bootsector_size + bootsector_padd 89 | free_cluster_offset = 488 90 | # write new value to disk 91 | self.stream.seek(self.offset + fs_info_start + free_cluster_offset) 92 | self.stream.write(int(new_value).to_bytes(4, 'little')) 93 | # re-read pre_data_region 94 | self.stream.seek(self.offset) 95 | self.pre = FAT32_PRE_DATA_REGION.parse_stream(self.stream) 96 | 97 | def write_last_allocated(self, new_value: int) -> None: 98 | """ 99 | Write a new value to 'last_allocated_data_cluster' field of FAT32 100 | FS INFO sector 101 | :param new_value: int, new value for free_cluster_count 102 | """ 103 | # calculate start address of FS_INFO sector 104 | bootsector_size = FAT_CORE_BOOTSECTOR.sizeof() \ 105 | + FAT32_EXTENDED_BOOTSECTOR.sizeof() 106 | bootsector_padd = self.pre.sector_size - bootsector_size 107 | fs_info_start = bootsector_size + bootsector_padd 108 | free_cluster_offset = 492 109 | # write new value to disk 110 | self.stream.seek(self.offset + fs_info_start + free_cluster_offset) 111 | self.stream.write(int(new_value).to_bytes(4, 'little')) 112 | # re-read pre_data_region 113 | self.stream.seek(self.offset) 114 | self.pre = FAT32_PRE_DATA_REGION.parse_stream(self.stream) 115 | -------------------------------------------------------------------------------- /fishy/wrapper/reserved_gdt_blocks.py: -------------------------------------------------------------------------------- 1 | """ 2 | Wrapper for filesystem specific implementations of hiding data inside reserved GDT blocks 3 | """ 4 | import logging 5 | import typing as typ 6 | from os import path 7 | from fishy.filesystem_detector import get_filesystem_type 8 | from fishy.metadata import Metadata 9 | from fishy.ext4.reserved_gdt_blocks import EXT4ReservedGDTBlocks 10 | from fishy.ext4.reserved_gdt_blocks import EXT4ReservedGDTBlocksMetadata 11 | 12 | LOGGER = logging.getLogger("ReservedGDTBlocks") 13 | 14 | class ReservedGDTBlocks: 15 | """ 16 | This class wraps the filesystem specific implementation of the reserved GDT blocks hiding technique 17 | """ 18 | def __init__(self, fs_stream: typ.BinaryIO, metadata: Metadata, 19 | dev: str = None): 20 | """ 21 | :param dev: Path to filesystem 22 | :param fs_stream: Stream of filesystem 23 | :param metadata: Metadata object 24 | """ 25 | self.dev = dev 26 | self.metadata = metadata 27 | self.fs_type = get_filesystem_type(fs_stream) 28 | if self.fs_type == 'EXT4': 29 | self.metadata.set_module("ext4-reserved-gdt-blocks") 30 | self.fs = EXT4ReservedGDTBlocks(fs_stream, dev) # pylint: disable=invalid-name 31 | else: 32 | raise NotImplementedError() 33 | 34 | def write(self, instream: typ.BinaryIO, 35 | filename: str = None) -> None: 36 | """ 37 | writes data from instream into reserved GDT blocks. Metadata of 38 | those files will be stored in Metadata object 39 | 40 | :param instream: stream to read data from 41 | :param filename: name that will be used, when file gets written into reserved GDT blocks. 42 | If none, a random name will be generated. 43 | :raises: IOError 44 | """ 45 | LOGGER.info("Write") 46 | if filename is not None: 47 | filename = path.basename(filename) 48 | if self.fs_type == 'EXT4': 49 | LOGGER.info("Write into ext4") 50 | reserved_gdt_block_metadata = self.fs.write(instream) 51 | self.metadata.add_file(filename, reserved_gdt_block_metadata) 52 | else: 53 | raise NotImplementedError() 54 | 55 | def read(self, outstream: typ.BinaryIO): 56 | """ 57 | writes hidden data from reserved GDT blocks into stream. 58 | 59 | :param outstream: stream to write hidden data into 60 | :raises: IOError 61 | """ 62 | file_metadata = self.metadata.get_file("0")['metadata'] 63 | if self.fs_type == 'EXT4': 64 | reserved_gdt_blocks_metadata = EXT4ReservedGDTBlocksMetadata(file_metadata) 65 | self.fs.read(outstream, reserved_gdt_blocks_metadata) 66 | else: 67 | raise NotImplementedError() 68 | 69 | def read_into_file(self, outfilepath: str): 70 | """ 71 | reads hidden data from reserved GDT blocks into files 72 | :note: If provided filepath already exists, this file will be 73 | overwritten without a warning. 74 | :param outfilepath: filepath to file, where hidden data will be 75 | restored into 76 | """ 77 | if self.fs_type == 'EXT4': 78 | with open(outfilepath, 'wb+') as outfile: 79 | self.read(outfile) 80 | else: 81 | raise NotImplementedError() 82 | 83 | def clear(self): 84 | """ 85 | clears reserved GDT blocks in which data has been hidden 86 | :param metadata: Metadata, object where metadata is stored in 87 | :raises: IOError 88 | """ 89 | if self.fs_type == 'EXT4': 90 | for file_entry in self.metadata.get_files(): 91 | file_metadata = file_entry['metadata'] 92 | file_metadata = EXT4ReservedGDTBlocksMetadata(file_metadata) 93 | self.fs.clear(file_metadata) 94 | else: 95 | raise NotImplementedError() 96 | 97 | def info(self): 98 | """ 99 | shows info about reserved GDT blocks and data hiding space 100 | :param metadata: Metadata, object where metadata is stored in 101 | :raises: NotImplementedError 102 | """ 103 | if self.fs_type == 'EXT4': 104 | if len(list(self.metadata.get_files())) > 0: 105 | for file_entry in self.metadata.get_files(): 106 | file_metadata = file_entry['metadata'] 107 | file_metadata = EXT4ReservedGDTBlocksMetadata(file_metadata) 108 | self.fs.info(file_metadata) 109 | else: 110 | self.fs.info() 111 | else: 112 | raise NotImplementedError() 113 | -------------------------------------------------------------------------------- /fishy/APFS/APFS_filesystem/Volume_Superblock.py: -------------------------------------------------------------------------------- 1 | # structure of volume superblock; at first only for potential links in container 2 | 3 | from fishy.APFS.APFS_filesystem.APFS_Parser import Parser 4 | 5 | class vSuperblock: 6 | 7 | structure = { 8 | "magic": {"offset": 0x0, "size": 4, "format": "raw"}, 9 | "fs_index": {"offset": 0x4, "size": 4}, 10 | "features": {"offset": 0x8, "size": 8}, 11 | "readonly_features": {"offset": 0x10, "size": 8}, 12 | "incompatible_features": {"offset": 0x18, "size": 8}, 13 | "unmount_time": {"offset": 0x20, "size": 8, "format": "hex"}, 14 | "reserve_block_count": {"offset": 0x28, "size": 8}, 15 | "quota_block_count": {"offset": 0x30, "size": 8}, 16 | "alloc_block_count": {"offset": 0x38, "size": 8}, 17 | "crypto_meta": {"offset": 0x40, "size": 32, "format": "raw"}, 18 | # single values unimportant; easily implementable via official ref 19 | # "root_tree_type": {"offset": 0x60, "size": 4}, in doc but not in image 20 | # "extentref_tree_type": {"offset": 0x64, "size": 4}, in doc but not in image 21 | # "snapmeta_tree_type": {"offset": 0x68, "size": 4}, in doc but not in image 22 | "omap_oid": {"offset": 0x60, "size": 8}, 23 | "root_tree_oid": {"offset": 0x68, "size": 8}, 24 | "extentref_tree_oid": {"offset": 0x70, "size": 8}, 25 | "snapmeta_tree_oid": {"offset": 0x78, "size": 8}, 26 | "revert_to_xid": {"offset": 0x80, "size": 8}, 27 | "revert_to_sblock_oid": {"offset": 0x88, "size": 8}, 28 | "next_obj_id": {"offset": 0x90, "size": 8}, 29 | "num_files": {"offset": 0x98, "size": 8}, 30 | "num_dir": {"offset": 0xA0, "size": 8}, 31 | "num_symlinks": {"offset": 0xA8, "size": 8}, 32 | "num_other_fs_obj": {"offset": 0xB0, "size": 8}, 33 | "num_snapshots": {"offset": 0xB8, "size": 8}, 34 | "total_blocks_alloc": {"offset": 0xC0, "size": 8}, 35 | "total_block_freed": {"offset": 0xC8, "size": 8}, 36 | "vol_uuid": {"offset": 0xD0, "size": 16}, 37 | "last_mod_time": {"offset": 0xE0, "size": 8, "format": "hex"}, 38 | "fs_flags": {"offset": 0xE8, "size": 8, "format": "hex"}, 39 | "apfs_formatted_by_id": {"offset": 0xF0, "size": 32, "format": "utf"}, 40 | "apfs_formatted_by_timestamp": {"offset": 0xF0+32, "size": 8, "format": "hex"}, 41 | "apfs_formatted_by_xid": {"offset": 0xF0+40, "size": 8} 42 | } 43 | 44 | 45 | def __init__(self, fs_stream, offset, blocksize): 46 | self.fs_stream = fs_stream 47 | self.blocksize = blocksize 48 | self.data = self.parseVolumeSuperblock(fs_stream, offset) 49 | self.offset = offset 50 | 51 | def parseVolumeSuperblock(self, fs_stream, offset): 52 | blocksize = self.blocksize 53 | d = Parser.parse(fs_stream, offset+32, blocksize-32, structure=self.structure) 54 | 55 | 56 | 57 | structLowerId = { 58 | "apfs_modified_by_id " + str(i): {"offset": 0xF0+48+(i*48), "size": 32, "format": "utf"} for i in range(0,8) 59 | } 60 | 61 | structLowerTs = { 62 | "apfs_modified_by_timestamp " + str(i): {"offset": 0xF0+80+(i*48), "size": 8, "format": "hex"} for i in range(0,8) 63 | } 64 | 65 | s1 = {**structLowerId, **structLowerTs} 66 | 67 | structLowerXid = { 68 | "apfs_modified_by_xid " + str(i): {"offset": 0xF0+88+(i*48), "size": 8} for i in range(0,8) 69 | } 70 | 71 | s2 = {**s1, **structLowerXid} 72 | 73 | 74 | structLowerToo = { 75 | "volname": {"offset": 0xF0 + (48 * 9), "size": 256, "format": "utf"}, 76 | "next_doc_id": {"offset": 0xF0 + (48 * 9) + 256, "size": 4}, 77 | "role": {"offset": 0xF4 + (48 * 9) + 256, "size": 2}, 78 | "reserved1": {"offset": 0xF6 + (48 * 9) + 256, "size": 2}, 79 | "root_to_xid": {"offset": 0xF8 + (48 * 9) + 256, "size": 8}, 80 | "er_state_oid": {"offset": 0x100 + (48 * 9) + 256, "size": 8}, 81 | "reserved2": {"offset": 0x100 + (48 * 9) + 256, "size": 60} 82 | 83 | # Timestamps are not implemented completely due to their unique structure 84 | } 85 | 86 | s3 = {**s2, **structLowerToo} 87 | 88 | e = Parser.parse(fs_stream, offset+32, blocksize-32, structure=s3) 89 | 90 | f = {**e, **d} 91 | 92 | return f 93 | 94 | def getOmapRootNode(self): 95 | blocksize = self.blocksize 96 | 97 | return self.data["omap_oid"]*blocksize 98 | 99 | def getExtentRefTree(self): 100 | blocksize = self.blocksize 101 | 102 | return self.data["extentref_tree_oid"]*blocksize 103 | 104 | def getrootTreeOid(self): 105 | 106 | return self.data["root_tree_oid"] 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /tests/test_fat_bad_cluster.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring, protected-access 2 | """ 3 | These tests run against fishy.fat.bad_cluster which implements the 4 | bad cluster allocation hiding technique for FAT filesystems 5 | """ 6 | import io 7 | import pytest 8 | from fishy.fat.bad_cluster import BadCluster 9 | 10 | 11 | class TestWrite(object): 12 | """ Test write method """ 13 | def test_write_single_cluster(self, testfs_fat_stable1): 14 | """ Test if writing to a single bad cluster works """ 15 | for img_path in testfs_fat_stable1: 16 | with open(img_path, 'rb+') as img_stream: 17 | # create Allocator object 18 | fatfs = BadCluster(img_stream) 19 | expected_start_cluster = fatfs.fatfs.get_free_cluster() 20 | # setup raw stream and write testmessage 21 | with io.BytesIO() as mem: 22 | teststring = "This is a simple write test." 23 | mem.write(teststring.encode('utf-8')) 24 | mem.seek(0) 25 | # write testmessage to disk 26 | with io.BufferedReader(mem) as reader: 27 | result = fatfs.write(reader) 28 | assert result.get_clusters()[0] \ 29 | == expected_start_cluster 30 | 31 | 32 | class TestRead(object): 33 | """ Test read method """ 34 | def test_read(self, testfs_fat_stable1): 35 | """ Test if reading from a single bad cluster works """ 36 | for img_path in testfs_fat_stable1: 37 | with open(img_path, 'rb+') as img_stream: 38 | # create Allocator object 39 | fatfs = BadCluster(img_stream) 40 | teststring = "This is a simple write test." 41 | # write content that we want to read 42 | with io.BytesIO() as mem: 43 | mem.write(teststring.encode('utf-8')) 44 | mem.seek(0) 45 | with io.BufferedReader(mem) as reader: 46 | write_res = fatfs.write(reader) 47 | # read content we wrote and compare result with 48 | # our initial test message 49 | with io.BytesIO() as mem: 50 | fatfs.read(mem, write_res) 51 | mem.seek(0) 52 | result = mem.read() 53 | assert result.decode('utf-8') == teststring 54 | 55 | def test_read_multi_cluster(self, testfs_fat_stable1): 56 | """ Test if reading from multiple bad clusters works """ 57 | for img_path in testfs_fat_stable1: 58 | with open(img_path, 'rb+') as img_stream: 59 | # create Allocator object 60 | fatfs = BadCluster(img_stream) 61 | teststring = "This is a simple write test."*80 62 | # write content that we want to read 63 | with io.BytesIO() as mem: 64 | mem.write(teststring.encode('utf-8')) 65 | mem.seek(0) 66 | with io.BufferedReader(mem) as reader: 67 | write_res = fatfs.write(reader) 68 | # read clusters 69 | result = io.BytesIO() 70 | fatfs.read(result, write_res) 71 | result.seek(0) 72 | # compare cluster content 73 | assert result.read() == teststring.encode('utf-8') 74 | 75 | class TestClean(object): 76 | def test_clean(self, testfs_fat_stable1): 77 | """ Test if cleaning bad clusters works """ 78 | for img_path in testfs_fat_stable1: 79 | with open(img_path, 'rb+') as img_stream: 80 | # create Allocator object 81 | fatfs = BadCluster(img_stream) 82 | teststring = "This is a simple write test." 83 | # write content that we want to read 84 | with io.BytesIO() as mem: 85 | mem.write(teststring.encode('utf-8')) 86 | mem.seek(0) 87 | with io.BufferedReader(mem) as reader: 88 | write_res = fatfs.write(reader) 89 | # save written bytes 90 | resulting_bytes = io.BytesIO() 91 | fatfs.read(resulting_bytes, write_res) 92 | resulting_bytes.seek(0) 93 | # save used clusters 94 | used_clusters = write_res.clusters 95 | fatfs.clear(write_res) 96 | # read overwritten clusters 97 | resulting = io.BytesIO() 98 | for cluster_id in used_clusters: 99 | fatfs.fatfs.cluster_to_stream(cluster_id, resulting) 100 | resulting.seek(0) 101 | # compare cluster content after write and after clear 102 | assert resulting_bytes.read() != resulting.read() 103 | -------------------------------------------------------------------------------- /fishy/wrapper/mft_slack.py: -------------------------------------------------------------------------------- 1 | """ 2 | MftSlack a wrapper to comply with README - How to implement a hiding technique 3 | """ 4 | import logging 5 | import typing as typ 6 | from os import path 7 | from ..filesystem_detector import get_filesystem_type 8 | from ..metadata import Metadata 9 | from ..ntfs.ntfs_mft_slack import NtfsMftSlack as NTFSMftSlack 10 | from ..ntfs.ntfs_mft_slack import MftSlackMetadata as NTFSMftSlackMetadata 11 | 12 | LOGGER = logging.getLogger("MftSlack") 13 | 14 | class MftSlack: 15 | """ 16 | This class wrapps the mft slack implementations 17 | 18 | usage examples: 19 | 20 | >>> f = open('/dev/sdb1', 'rb+') 21 | >>> ms = MftSlack(f) 22 | >>> m = Metadata("MftSlack") 23 | 24 | to write something from stdin into slack: 25 | >>> fs.write(sys.stdin.buffer, m) 26 | 27 | to write something from stdin into slack with offset: 28 | >>> fs.write(sys.stdin.buffer, m, 36) 29 | 30 | to read something from slack to stdout: 31 | >>> fs.read(sys.stdout.buffer, m) 32 | 33 | to wipe slackspace via metadata file: 34 | >>> fs.clear_with_metadata(m) 35 | """ 36 | def __init__(self, fs_stream: typ.BinaryIO, metadata: Metadata, 37 | dev: str = None, domirr = False): 38 | """ 39 | :param fs_stream: Stream of filesystem 40 | :param metadata: Metadata object 41 | :param dev: device to use 42 | :param domirr: write copy of data to $MFTMirr 43 | """ 44 | self.dev = dev 45 | self.metadata = metadata 46 | self.fs_type = get_filesystem_type(fs_stream) 47 | if self.fs_type == 'NTFS': 48 | self.fs = NTFSMftSlack(dev, fs_stream) 49 | self.fs.domirr = domirr 50 | self.metadata.set_module("ntfs-mft-slack") 51 | else: 52 | raise NotImplementedError() 53 | 54 | def write(self, instream: typ.BinaryIO, filename: str=None, offset=0) -> None: 55 | """ 56 | writes data from instream into slackspace of mft entires starting at the offset. 57 | Metadata of those files will be stored in Metadata object 58 | 59 | :param instream: stream to read data from 60 | :param filename: name that will be used, when file gets written into a 61 | directory (while reading fileslack). if none, a random 62 | name will be generated 63 | :param offset: first sector of mft entry to start with 64 | :raises: IOError 65 | """ 66 | LOGGER.info("Write") 67 | if filename is not None: 68 | filename = path.basename(filename) 69 | if self.fs_type == 'NTFS': 70 | slack_metadata = self.fs.write(instream, offset) 71 | self.metadata.add_file(filename, slack_metadata) 72 | else: 73 | raise NotImplementedError() 74 | 75 | def read(self, outstream: typ.BinaryIO): 76 | """ 77 | writes hidden data from slackspace into stream. The examined slack 78 | space information is taken from metadata. 79 | 80 | :param outstream: stream to write hidden data into 81 | :raises: IOError 82 | """ 83 | file_metadata = self.metadata.get_file("0")['metadata'] 84 | if self.fs_type == 'NTFS': 85 | slack_metadata = NTFSMftSlackMetadata(file_metadata) 86 | self.fs.read(outstream, slack_metadata) 87 | else: 88 | raise NotImplementedError() 89 | 90 | def read_into_file(self, outfilepath: str): 91 | """ 92 | reads hidden data from slack into files 93 | :note: If provided filepath already exists, this file will be 94 | overwritten without a warning. 95 | :param outfilepath: filepath to file, where hidden data will be 96 | restored into 97 | """ 98 | if self.fs_type == 'NTFS': 99 | with open(outfilepath, 'wb+') as outfile: 100 | self.read(outfile) 101 | else: 102 | raise NotImplementedError() 103 | 104 | def clear(self): 105 | """ 106 | clears the slackspace of mft entires. Information of them is stored in 107 | metadata. 108 | :param metadata: Metadata, object where metadata is stored in 109 | :raises: IOError 110 | """ 111 | if self.fs_type == 'NTFS': 112 | for file_entry in self.metadata.get_files(): 113 | file_metadata = file_entry['metadata'] 114 | file_metadata = NTFSMftSlackMetadata(file_metadata) 115 | self.fs.clear(file_metadata) 116 | else: 117 | raise NotImplementedError() 118 | 119 | def info(self, offset=0, limit=-1) -> None: 120 | """ 121 | prints info about available file slack of mft entries 122 | 123 | :param offset: First sector of mft entry to start with. 124 | :param limit: Amount of mft entries to display info for. Unlimited if -1. 125 | """ 126 | if self.fs_type == 'NTFS': 127 | self.fs.print_info(offset, limit) 128 | else: 129 | raise NotImplementedError() 130 | -------------------------------------------------------------------------------- /fishy/wrapper/bad_cluster.py: -------------------------------------------------------------------------------- 1 | """ 2 | BadCluster wrapper for filesystem specific implementations 3 | """ 4 | import logging 5 | import typing as typ 6 | from os import path 7 | from ..fat.bad_cluster import BadCluster as FATBadCluster 8 | from ..fat.bad_cluster import BadClusterMetadata as FATBadClusterMetadata 9 | from ..ntfs.bad_cluster import NtfsBadCluster as NTFSBadCluster 10 | from ..ntfs.bad_cluster import BadClusterMetadata as NTFSBadClusterMetadata 11 | from ..filesystem_detector import get_filesystem_type 12 | from ..metadata import Metadata 13 | 14 | LOGGER = logging.getLogger("ClusterAllocation") 15 | 16 | class BadClusterWrapper: 17 | """ 18 | This class wrapps the filesystem specific file cluster allocation 19 | implementations 20 | 21 | usage examples: 22 | 23 | >>> f = open('/dev/sdb1', 'rb+') 24 | >>> fs = BadClusterWrapper(f) 25 | >>> m = Metadata("BadCluster") 26 | 27 | to write something from stdin into slack: 28 | >>> fs.write(sys.stdin.buffer, m) 29 | 30 | to read something from slack to stdout: 31 | >>> fs.read(sys.stdout.buffer, m) 32 | 33 | to wipe slackspace via metadata file: 34 | >>> fs.clear_with_metadata(m) 35 | """ 36 | def __init__(self, fs_stream: typ.BinaryIO, metadata: Metadata, 37 | dev: str = None): 38 | """ 39 | :param fs_stream: Stream of filesystem 40 | :param metadata: Metadata object 41 | """ 42 | self.dev = dev 43 | self.metadata = metadata 44 | self.fs_type = get_filesystem_type(fs_stream) 45 | if self.fs_type == 'FAT': 46 | self.metadata.set_module("fat-bad-cluster") 47 | self.fs = FATBadCluster(fs_stream) # pylint: disable=invalid-name 48 | elif self.fs_type == 'NTFS': 49 | self.metadata.set_module("ntfs-bad-cluster") 50 | #self.fs = NTFSBadCluster(dev) 51 | self.fs = NTFSBadCluster(dev, fs_stream) 52 | else: 53 | raise NotImplementedError() 54 | 55 | def write(self, instream: typ.BinaryIO, filename: str = None) -> None: 56 | """ 57 | writes data from instream into bad cluster. 58 | Metadata of this file will be stored in Metadata object 59 | 60 | :param instream: stream to read data from 61 | :param filename: name that will be used, when file gets written into a 62 | directory (while reading bad clusters). if none, a 63 | random name will be generated 64 | :raises: IOError 65 | """ 66 | if filename is not None: 67 | filename = path.basename(filename) 68 | if self.fs_type == 'FAT': 69 | bad_cluster_metadata = self.fs.write(instream) 70 | self.metadata.add_file(filename, bad_cluster_metadata) 71 | elif self.fs_type == 'NTFS': 72 | bad_cluster_metadata = self.fs.write(instream) 73 | self.metadata.add_file(filename, bad_cluster_metadata) 74 | else: 75 | raise NotImplementedError() 76 | 77 | def read(self, outstream: typ.BinaryIO): 78 | """ 79 | writes hidden data from bad clusters into stream. 80 | 81 | :param outstream: stream to write hidden data into 82 | :raises: IOError 83 | """ 84 | file_metadata = self.metadata.get_file("0")['metadata'] 85 | if self.fs_type == 'FAT': 86 | bad_cluster_metadata = FATBadClusterMetadata(file_metadata) 87 | self.fs.read(outstream, bad_cluster_metadata) 88 | elif self.fs_type == 'NTFS': 89 | bad_cluster_metadata = NTFSBadClusterMetadata(file_metadata) 90 | self.fs.read(outstream, bad_cluster_metadata) 91 | else: 92 | raise NotImplementedError() 93 | 94 | def read_into_file(self, outfilepath: str): 95 | """ 96 | reads hidden data from bad clusters into file 97 | :note: If provided filepath already exists, this file will be 98 | overwritten without a warning. 99 | :param outfilepath: filepath to file, where hidden data will be 100 | restored into 101 | """ 102 | if self.fs_type == 'FAT': 103 | with open(outfilepath, 'wb+') as outfile: 104 | self.read(outfile) 105 | elif self.fs_type == 'NTFS': 106 | with open(outfilepath, 'wb+') as outfile: 107 | self.read(outfile) 108 | else: 109 | raise NotImplementedError() 110 | 111 | def clear(self): 112 | """ 113 | clears the allocated bad clusters. Information of them is stored in 114 | metadata. 115 | :param metadata: Metadata, object where metadata is stored in 116 | :raises: IOError 117 | """ 118 | if self.fs_type == 'FAT': 119 | for file_entry in self.metadata.get_files(): 120 | file_metadata = file_entry['metadata'] 121 | file_metadata = FATBadClusterMetadata(file_metadata) 122 | self.fs.clear(file_metadata) 123 | elif self.fs_type == 'NTFS': 124 | for file_entry in self.metadata.get_files(): 125 | file_metadata = file_entry['metadata'] 126 | file_metadata = NTFSBadClusterMetadata(file_metadata) 127 | self.fs.clear(file_metadata) 128 | else: 129 | raise NotImplementedError() 130 | -------------------------------------------------------------------------------- /tests/test_ntfs_file_slack.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring, protected-access 2 | """ 3 | This file contains tests against fishy.ntfs.ntfs_file_slack, which implements the 4 | fileslack hiding technique for NTFS filesystems 5 | """ 6 | import io 7 | import pytest 8 | from fishy.ntfs.ntfs_file_slack import NtfsSlack as FileSlack 9 | 10 | 11 | class TestWrite: 12 | """ Test writing into the slack space """ 13 | def test_write_file(self, testfs_ntfs_stable2): 14 | """" Test writing a file into root directory """ 15 | for img_path in testfs_ntfs_stable2: 16 | with open(img_path, 'rb+') as img: 17 | # create FileSlack object 18 | ntfs = FileSlack(img_path, img) 19 | # setup raw stream and write testmessage 20 | with io.BytesIO() as mem: 21 | teststring = "This is a simple write test." 22 | mem.write(teststring.encode('utf-8')) 23 | mem.seek(0) 24 | # write testmessage to disk 25 | with io.BufferedReader(mem) as reader: 26 | result = ntfs.write(reader, ['other_file.txt']) 27 | assert result.addrs == [(1733632, 28)] 28 | with pytest.raises(IOError): 29 | mem.seek(0) 30 | ntfs = FileSlack(img_path, img) 31 | ntfs.write(reader, ['no_free_slack.txt']) 32 | 33 | def test_write_file_autoexpand_subdir(self, testfs_ntfs_stable2): # pylint: disable=invalid-name 34 | """ Test if autoexpansion for directories as input filepath works """ 35 | # if user supplies a directory instead of a file path, all files under 36 | # this directory will recusively added 37 | for img_path in testfs_ntfs_stable2: 38 | with open(img_path, 'rb+') as img: 39 | # create FileSlack object 40 | ntfs = FileSlack(img_path, img) 41 | # setup raw stream and write testmessage 42 | with io.BytesIO() as mem: 43 | teststring = "This is a simple write test."*100 44 | mem.write(teststring.encode('utf-8')) 45 | mem.seek(0) 46 | # write testmessage to disk 47 | with io.BufferedReader(mem) as reader: 48 | result = ntfs.write(reader, ['/']) 49 | assert sorted(result.addrs) == sorted([(1733632, 2800)]) 50 | 51 | class TestRead: 52 | """ Test reading slackspace """ 53 | def test_read_slack(self, testfs_ntfs_stable2): 54 | """ Test if reading content of slackspace in a simple case works """ 55 | for img_path in testfs_ntfs_stable2: 56 | with open(img_path, 'rb+') as img: 57 | # create FileSlack object 58 | ntfs = FileSlack(img_path, img) 59 | teststring = "This is a simple write test." 60 | # write content that we want to read 61 | with io.BytesIO() as mem: 62 | mem.write(teststring.encode('utf-8')) 63 | mem.seek(0) 64 | with io.BufferedReader(mem) as reader: 65 | write_res = ntfs.write(reader, ['other_file.txt']) 66 | # read content we wrote and compare result with 67 | # our initial test message 68 | ntfs = FileSlack(img_path, img) 69 | with io.BytesIO() as mem: 70 | ntfs.read(mem, write_res) 71 | mem.seek(0) 72 | result = mem.read() 73 | assert result.decode('utf-8') == teststring 74 | 75 | class TestInfo: 76 | def test_info_slack(self, testfs_ntfs_stable2): 77 | """ Test if info works """ 78 | for img_path in testfs_ntfs_stable2: 79 | with open(img_path, 'rb+') as img: 80 | # create FileSlack object 81 | ntfs = FileSlack(img_path, img) 82 | slack = ntfs.print_info(['/']) 83 | assert slack == 3072 84 | 85 | class TestClear: 86 | def test_clear_slack(self, testfs_ntfs_stable2): 87 | """ Test if clearing slackspace of a file works """ 88 | for img_path in testfs_ntfs_stable2: 89 | with open(img_path, 'rb+') as img: 90 | # create FileSlack object 91 | ntfs = FileSlack(img_path, img) 92 | teststring = "This is a simple write test." 93 | # write content that we want to clear 94 | with io.BytesIO() as mem: 95 | mem.write(teststring.encode('utf-8')) 96 | mem.seek(0) 97 | with io.BufferedReader(mem) as reader: 98 | write_res = ntfs.write(reader, ['other_file.txt']) 99 | ntfs = FileSlack(img_path, img) 100 | ntfs.clear(write_res) 101 | ntfs = FileSlack(img_path, img) 102 | # clear content we wrote, then read the cleared part again. 103 | # As it should be overwritten we expect a stream of \x00 104 | with io.BytesIO() as mem: 105 | ntfs.read(mem, write_res) 106 | mem.seek(0) 107 | result = mem.read() 108 | expected = len(teststring.encode('utf-8')) * b'\x00' 109 | assert result == expected 110 | -------------------------------------------------------------------------------- /fishy/wrapper/cluster_allocation.py: -------------------------------------------------------------------------------- 1 | """ 2 | ClusterAllocator wrapper for filesystem specific implementations 3 | """ 4 | import logging 5 | import typing as typ 6 | from os import path 7 | from ..ntfs.cluster_allocator import ClusterAllocator as NTFSAllocator 8 | from ..ntfs.cluster_allocator import AllocatorMetadata as NTFSAllocatorMeta 9 | from ..fat.cluster_allocator import ClusterAllocator as FATAllocator 10 | from ..fat.cluster_allocator import AllocatorMetadata as FATAllocatorMeta 11 | from ..filesystem_detector import get_filesystem_type 12 | from ..metadata import Metadata 13 | 14 | LOGGER = logging.getLogger("ClusterAllocation") 15 | 16 | class ClusterAllocation: 17 | """ 18 | This class wrapps the filesystem specific file cluster allocation 19 | implementations 20 | 21 | usage examples: 22 | 23 | >>> f = open('/dev/sdb1', 'rb+') 24 | >>> fs = ClusterAllocation(f) 25 | >>> m = Metadata("FileSlack") 26 | >>> filename = 'path/to/file/on/fs' 27 | 28 | to write something from stdin into slack: 29 | >>> fs.write(sys.stdin.buffer, m, filename) 30 | 31 | to read something from slack to stdout: 32 | >>> fs.read(sys.stdout.buffer, m) 33 | 34 | to wipe slackspace via metadata file: 35 | >>> fs.clear_with_metadata(m) 36 | """ 37 | def __init__(self, fs_stream: typ.BinaryIO, metadata: Metadata, 38 | dev: str = None): 39 | """ 40 | :param fs_stream: Stream of filesystem 41 | :param metadata: Metadata object 42 | """ 43 | self.dev = dev 44 | self.metadata = metadata 45 | self.fs_type = get_filesystem_type(fs_stream) 46 | if self.fs_type == 'FAT': 47 | self.metadata.set_module("fat-cluster-allocator") 48 | self.fs = FATAllocator(fs_stream) # pylint: disable=invalid-name 49 | elif self.fs_type == 'NTFS': 50 | self.metadata.set_module("ntfs-cluster-allocator") 51 | self.fs = NTFSAllocator(fs_stream) # pylint: disable=invalid-name 52 | else: 53 | raise NotImplementedError() 54 | 55 | def write(self, instream: typ.BinaryIO, filepath: str, 56 | filename: str = None) -> None: 57 | """ 58 | writes data from instream into additional allocated clusters of given 59 | file. Metadata of this file will be stored in Metadata object 60 | 61 | :param instream: stream to read data from 62 | :param filepath: string, path to file, for which additional clusters 63 | will be allocated to hide data in 64 | :param filename: name that will be used, when file gets written into a 65 | directory (while reading fileslack). if none, a random 66 | name will be generated 67 | :raises: IOError 68 | """ 69 | if filename is not None: 70 | filename = path.basename(filename) 71 | if self.fs_type == 'FAT': 72 | allocator_metadata = self.fs.write(instream, filepath) 73 | self.metadata.add_file(filename, allocator_metadata) 74 | elif self.fs_type == 'NTFS': 75 | allocator_metadata = self.fs.write(instream, filepath) 76 | self.metadata.add_file(filename, allocator_metadata) 77 | else: 78 | raise NotImplementedError() 79 | 80 | def read(self, outstream: typ.BinaryIO): 81 | """ 82 | writes hidden data from slackspace into stream. The examined slack 83 | space information is taken from metadata. 84 | 85 | :param outstream: stream to write hidden data into 86 | :raises: IOError 87 | """ 88 | file_metadata = self.metadata.get_file("0")['metadata'] 89 | if self.fs_type == 'FAT': 90 | allocator_metadata = FATAllocatorMeta(file_metadata) 91 | self.fs.read(outstream, allocator_metadata) 92 | elif self.fs_type == 'NTFS': 93 | allocator_metadata = NTFSAllocatorMeta(file_metadata) 94 | self.fs.read(outstream, allocator_metadata) 95 | else: 96 | raise NotImplementedError() 97 | 98 | def read_into_file(self, outfilepath: str): 99 | """ 100 | reads hidden data from slack into files 101 | :note: If provided filepath already exists, this file will be 102 | overwritten without a warning. 103 | :param outfilepath: filepath to file, where hidden data will be 104 | restored into 105 | """ 106 | if self.fs_type == 'FAT': 107 | with open(outfilepath, 'wb+') as outfile: 108 | self.read(outfile) 109 | elif self.fs_type == 'NTFS': 110 | with open(outfilepath, 'wb+') as outfile: 111 | self.read(outfile) 112 | else: 113 | raise NotImplementedError() 114 | 115 | def clear(self): 116 | """ 117 | clears the slackspace of files. Information of them is stored in 118 | metadata. 119 | :param metadata: Metadata, object where metadata is stored in 120 | :raises: IOError 121 | """ 122 | if self.fs_type == 'FAT': 123 | for file_entry in self.metadata.get_files(): 124 | file_metadata = file_entry['metadata'] 125 | file_metadata = FATAllocatorMeta(file_metadata) 126 | self.fs.clear(file_metadata) 127 | elif self.fs_type == 'NTFS': 128 | for file_entry in self.metadata.get_files(): 129 | file_metadata = file_entry['metadata'] 130 | file_metadata = NTFSAllocatorMeta(file_metadata) 131 | self.fs.clear(file_metadata) 132 | else: 133 | raise NotImplementedError() 134 | -------------------------------------------------------------------------------- /tests/test_ntfs_mft_slack.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-docstring, protected-access 2 | """ 3 | This file contains tests against fishy.ntfs.ntfs_mft_slack, which implements the 4 | mftslack hiding technique for NTFS filesystems 5 | """ 6 | import io 7 | import pytest 8 | from fishy.ntfs.ntfs_mft_slack import NtfsMftSlack as MftSlack 9 | 10 | 11 | class TestWrite: 12 | """ Test writing into the slack space """ 13 | def test_write(self, testfs_ntfs_stable1): 14 | """" Test hiding data starting at mft root """ 15 | for img_path in testfs_ntfs_stable1: 16 | with open(img_path, 'rb+') as img: 17 | # create MftSlack object 18 | ntfs = MftSlack(img_path, img) 19 | # setup raw stream and write testmessage 20 | with io.BytesIO() as mem: 21 | teststring = "This is a simple write test."*30 22 | mem.write(teststring.encode('utf-8')) 23 | mem.seek(0) 24 | # write testmessage to disk 25 | with io.BufferedReader(mem) as reader: 26 | result = ntfs.write(reader) 27 | assert result.addrs == [(16792, 102, 0), (16896, 510, 0), 28 | (17752, 166, 0), (17920, 62, 0)] 29 | with pytest.raises(IOError): 30 | mem.seek(0) 31 | ntfs = MftSlack(img_path, img) 32 | ntfs.write(reader, 178) 33 | 34 | def test_write_offset(self, testfs_ntfs_stable1): 35 | """ Test hiding data with offset """ 36 | for img_path in testfs_ntfs_stable1: 37 | with open(img_path, 'rb+') as img: 38 | # create MftSlack object 39 | ntfs = MftSlack(img_path, img) 40 | # setup raw stream and write testmessage 41 | with io.BytesIO() as mem: 42 | teststring = "This is a simple write test." 43 | mem.write(teststring.encode('utf-8')) 44 | mem.seek(0) 45 | # write testmessage to disk 46 | with io.BufferedReader(mem) as reader: 47 | result = ntfs.write(reader,36) 48 | assert result.addrs == [(18776, 28, 0)] 49 | 50 | class TestRead: 51 | """ Test reading slackspace """ 52 | def test_read_slack(self, testfs_ntfs_stable1): 53 | """ Test if reading content of slackspace in a simple case works """ 54 | for img_path in testfs_ntfs_stable1: 55 | with open(img_path, 'rb+') as img: 56 | # create MftSlack object 57 | ntfs = MftSlack(img_path, img) 58 | teststring = "This is a simple write test." 59 | # write content that we want to read 60 | with io.BytesIO() as mem: 61 | mem.write(teststring.encode('utf-8')) 62 | mem.seek(0) 63 | with io.BufferedReader(mem) as reader: 64 | write_res = ntfs.write(reader) 65 | # read content we wrote and compare result with 66 | # our initial test message 67 | ntfs = MftSlack(img_path, img) 68 | with io.BytesIO() as mem: 69 | ntfs.read(mem, write_res) 70 | mem.seek(0) 71 | result = mem.read() 72 | assert result.decode('utf-8') == teststring 73 | 74 | class TestInfo: 75 | def test_info_slack(self, testfs_ntfs_stable1): 76 | """ Test if info works """ 77 | for img_path in testfs_ntfs_stable1: 78 | with open(img_path, 'rb+') as img: 79 | # create MftSlack object 80 | ntfs = MftSlack(img_path, img) 81 | slack = ntfs.print_info() 82 | assert slack == 59372 83 | 84 | def test_limit_info_slack(self, testfs_ntfs_stable1): 85 | """ Test if info limited works """ 86 | for img_path in testfs_ntfs_stable1: 87 | with open(img_path, 'rb+') as img: 88 | # create MftSlack object 89 | ntfs = MftSlack(img_path, img) 90 | slack = ntfs.print_info(0, 5) 91 | assert slack == 3100 92 | 93 | class TestClear: 94 | def test_clear_slack(self, testfs_ntfs_stable1): 95 | """ Test if clearing slackspace works """ 96 | for img_path in testfs_ntfs_stable1: 97 | with open(img_path, 'rb+') as img: 98 | # create MftSlack object 99 | ntfs = MftSlack(img_path, img) 100 | teststring = "This is a simple write test." 101 | # write content that we want to clear 102 | with io.BytesIO() as mem: 103 | mem.write(teststring.encode('utf-8')) 104 | mem.seek(0) 105 | with io.BufferedReader(mem) as reader: 106 | write_res = ntfs.write(reader) 107 | ntfs = MftSlack(img_path, img) 108 | ntfs.clear(write_res) 109 | ntfs = MftSlack(img_path, img) 110 | # clear content we wrote, then read the cleared part again. 111 | # As it should be overwritten we expect a stream of \x00 112 | with io.BytesIO() as mem: 113 | ntfs.read(mem, write_res) 114 | mem.seek(0) 115 | result = mem.read() 116 | expected = len(teststring.encode('utf-8')) * b'\x00' 117 | assert result == expected 118 | -------------------------------------------------------------------------------- /fishy/APFS/APFS_filesystem/Checkpoints.py: -------------------------------------------------------------------------------- 1 | # this is a simple checkpoint class that only iterates through all container and volume superblocks (and comaps) 2 | # 1/2 So far, this checkpoint parsing system only supports apfs images that have contiguous checkpoint descriptor areas 3 | # 2/2 since it has proven impossible to artificially create a non-contiguous checkpoint descriptor area 4 | 5 | 6 | from fishy.APFS.APFS_filesystem.Container_Superblock import Superblock 7 | from fishy.APFS.APFS_filesystem.Object_Header import ObjectHeader 8 | from fishy.APFS.APFS_filesystem.Node import Node 9 | from fishy.APFS.APFS_filesystem.Volume_Superblock import vSuperblock 10 | from fishy.APFS.APFS_filesystem.Object_Map import ObjectMap 11 | 12 | 13 | class Checkpoints: 14 | 15 | def __init__(self, fs_stream): 16 | self.mainContainerBlock = Superblock(fs_stream, 0) 17 | self.data = Superblock(fs_stream, 0).parse_superblock(fs_stream, 0) 18 | self.blocksize = self.data["block_size"] 19 | 20 | def getCheckpointSuperblocks(self, fs_stream): 21 | # this function only parses contiguous checkpoint descriptor areas; 22 | mainInfo = self.mainContainerBlock.parse_superblock(fs_stream, 0) 23 | csdblocks = mainInfo["xp_desc_blocks"] 24 | addresslist = [] 25 | addressXidlist = [] 26 | 27 | for i in range(1, csdblocks+1): 28 | fs_stream.seek(i*self.blocksize) 29 | offset = fs_stream.tell() 30 | fs_stream.seek(offset + 32) 31 | fs_type = fs_stream.read(4) 32 | fs_stream.seek(offset) 33 | if fs_type == b'NXSB': 34 | addresslist.append(i*self.blocksize) 35 | fs_stream.seek(0) 36 | 37 | a = addresslist 38 | addresslist = list(set(a)) 39 | 40 | for i in range(0, len(addresslist)): 41 | temp = ObjectHeader(fs_stream, addresslist[i]).parse_object_header(fs_stream, addresslist[i]) 42 | xid = temp["xid"] 43 | addressXidlist.append((addresslist[i], xid)) 44 | 45 | addressXidlist=sorted(addressXidlist, key = lambda x:x[1], reverse=True) 46 | 47 | return addressXidlist 48 | 49 | # return tuple list of superblockaddresses and corresponding xid 50 | 51 | def getCheckpointCMAP(self, fs_stream, contxidlist): 52 | maplist = [] 53 | mapxidlist = [] 54 | 55 | for address, xid in contxidlist: 56 | temp = self.mainContainerBlock.parse_superblock(fs_stream, address) 57 | maplist.append(temp["omap_oid"]*self.blocksize) 58 | 59 | e = maplist 60 | maplist = list(set(e)) 61 | 62 | for i in range(0, len(maplist)): 63 | temp = ObjectHeader(fs_stream, maplist[i]).parse_object_header(fs_stream, maplist[i]) 64 | xid = temp["xid"] 65 | mapxidlist.append((maplist[i], xid)) 66 | 67 | mapxidlist=sorted(mapxidlist, key = lambda x:x[1], reverse=True) 68 | 69 | return mapxidlist 70 | 71 | #use parsed checkpoints for maps; returns map addresses 72 | 73 | def getCheckpointVolumes(self, fs_stream, cmapxidlist): 74 | vollist = [] 75 | volxidlist = [] 76 | for address, xid in cmapxidlist: 77 | temp = ObjectMap(fs_stream, address, self.blocksize).parseObjectMap(fs_stream, address) 78 | temproot = temp["root"]*self.blocksize 79 | rootnode = Node(fs_stream, temproot, self.blocksize) 80 | vm = rootnode.getVolumeMapping() 81 | add = [x for x, _ in vm] 82 | vollist.append(add) 83 | 84 | a = [i[0] for i in vollist] 85 | b = [i[1] for i in vollist] 86 | c = [i[2] for i in vollist] 87 | d = [i[3] for i in vollist] 88 | 89 | e = a + b + c + d 90 | 91 | # Turn list of tuples into multiple lists and make them one list without duplicates 92 | 93 | tempvollist = list(set(e)) 94 | 95 | for i in range(0, len(tempvollist)): 96 | temp = ObjectHeader(fs_stream, tempvollist[i]*self.blocksize)\ 97 | .parse_object_header(fs_stream, tempvollist[i]*self.blocksize) 98 | xid = temp["xid"] 99 | volxidlist.append((tempvollist[i]*self.blocksize, xid)) 100 | 101 | volxidlist=sorted(volxidlist, key = lambda x:x[1], reverse=True) 102 | 103 | return volxidlist 104 | 105 | # use parsed maps for volume addresses; returns volume addresses 106 | 107 | def getCheckpointVMAP(self, fs_stream, vxidlist): 108 | vmaplist = [] 109 | vmapxidlist = [] 110 | 111 | for address, xid in vxidlist: 112 | temp = vSuperblock(fs_stream, address, self.blocksize).parseVolumeSuperblock(fs_stream, address) 113 | vmaplist.append(temp["omap_oid"]*self.blocksize) 114 | 115 | f = vmaplist 116 | 117 | vmaplist = list(set(f)) 118 | for i in range(0, len(vmaplist)): 119 | temp = ObjectHeader(fs_stream, vmaplist[i]).parse_object_header(fs_stream, vmaplist[i]) 120 | xid = temp["xid"] 121 | vmapxidlist.append((vmaplist[i], xid)) 122 | 123 | vmapxidlist=sorted(vmapxidlist, key = lambda x:x[1], reverse=True) 124 | 125 | return vmapxidlist 126 | 127 | # use parsed volumes for maps; returns map addresses 128 | 129 | def getAllCheckpoints(self, fs_stream): 130 | contxidlist = self.getCheckpointSuperblocks(fs_stream) 131 | cmapxidlist = self.getCheckpointCMAP(fs_stream, contxidlist) 132 | vxidlist = self.getCheckpointVolumes(fs_stream, cmapxidlist) 133 | vmapxidlist = self.getCheckpointVMAP(fs_stream, vxidlist) 134 | 135 | completelist = contxidlist + cmapxidlist + vxidlist + vmapxidlist 136 | 137 | return completelist 138 | 139 | # put all blockaddresses and tuples in one list; remove duplicates (maps and volumes) ? 140 | 141 | -------------------------------------------------------------------------------- /fishy/ext4/osd2.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import math 3 | import typing as typ 4 | 5 | from fishy.ext4.ext4_filesystem.EXT4 import EXT4 6 | 7 | LOGGER = logging.getLogger("ext4-osd2") 8 | 9 | 10 | class EXT4OSD2Metadata: 11 | """ 12 | holds inode numbers which hold the hidden data in. 13 | """ 14 | def __init__(self, d: dict = None): 15 | """ 16 | :param d: dict, dictionary representation of a EXT4OSD2Metadata 17 | object 18 | """ 19 | if d is None: 20 | # self.inode_table = None 21 | self.inode_numbers = [] 22 | else: 23 | # self.inode_table = d["inode_table"] 24 | self.inode_numbers = d["inode_numbers"] 25 | 26 | def add_inode_number(self, inode_number: int) -> None: 27 | """ 28 | adds a block to the list of blocks 29 | 30 | :param block_id: int, id of the block 31 | """ 32 | self.inode_numbers.append(inode_number) 33 | 34 | def get_inode_numbers(self) \ 35 | -> []: 36 | """ 37 | returns list of inode_numbers 38 | 39 | :returns: list of inode_numbers 40 | """ 41 | return self.inode_numbers 42 | 43 | class EXT4OSD2: 44 | """ 45 | Hides data in osd2 field of inodes in the first inode_table. 46 | """ 47 | def __init__(self, stream: typ.BinaryIO, dev: str): 48 | """ 49 | :param dev: path to an ext4 filesystem 50 | 51 | :param stream: filedescriptor of an ext4 filesystem 52 | """ 53 | self.dev = dev 54 | self.stream = stream 55 | self.ext4fs = EXT4(stream, dev) 56 | self.inode_table = self.ext4fs.inode_tables[0] 57 | 58 | def write(self, instream: typ.BinaryIO) -> EXT4OSD2Metadata: 59 | """ 60 | writes from instream into the last two bytes of inodes osd2 field. 61 | This method currently supports only data sizes less than 4000 bytes. 62 | 63 | :param instream: stream to read from 64 | 65 | :return: EXT4OSD2Metadata 66 | """ 67 | metadata = EXT4OSD2Metadata() 68 | instream = instream.read() 69 | 70 | if not self._check_if_supported(instream): 71 | raise IOError("The hiding data size is currently not supported") 72 | 73 | 74 | instream_chunks = [instream[i:i+2] for i in range(0, len(instream), 2)] 75 | # print(instream_chunks) 76 | inode_number = 1 77 | hidden_chunks = 0 78 | 79 | while hidden_chunks < len(instream_chunks): 80 | chunk = instream_chunks[hidden_chunks] 81 | 82 | if self._write_to_osd2(chunk, inode_number): 83 | metadata.add_inode_number(inode_number) 84 | hidden_chunks += 1 85 | 86 | inode_number += 1 87 | 88 | return metadata 89 | 90 | def read(self, outstream: typ.BinaryIO, metadata: EXT4OSD2Metadata) \ 91 | -> None: 92 | """ 93 | writes data hidden in osd2 blocks into outstream 94 | 95 | :param outstream: stream to write into 96 | 97 | :param metadata: EXT4OSD2Metadata object 98 | """ 99 | inode_numbers = metadata.get_inode_numbers() 100 | # print(inode_numbers) 101 | for nr in inode_numbers: 102 | outstream.write(self._read_from_osd2(nr)) 103 | 104 | def clear(self, metadata: EXT4OSD2Metadata) -> None: 105 | """ 106 | clears the osd2 field in which data has been hidden 107 | 108 | :param metadata: EXT4OSD2Metadata object 109 | """ 110 | inode_numbers = metadata.get_inode_numbers() 111 | for nr in inode_numbers: 112 | self._clear_osd2(nr) 113 | 114 | def info(self, metadata: EXT4OSD2Metadata = None) -> None: 115 | """ 116 | shows info about inode osd2 fields and data hiding space 117 | 118 | :param metadata: EXT4OSD2Metadata object 119 | """ 120 | print("Inodes: " + str(self.ext4fs.superblock.data["inode_count"])) 121 | print("Total hiding space in osd2 fields: " + str((self.ext4fs.superblock.data["inode_count"]) * 2) + " Bytes") 122 | if metadata != None: 123 | filled_inode_numbers = metadata.get_inode_numbers() 124 | print('Used: ' + str(len(filled_inode_numbers) * 2) + ' Bytes') 125 | 126 | def _write_to_osd2(self, instream_chunk, inode_nr) -> bool: 127 | # print(instream_chunk) 128 | self.stream.seek(0) 129 | total_osd2_offset = self._get_total_osd2_offset(inode_nr) 130 | # print(total_osd2_offset) 131 | self.stream.seek(total_osd2_offset) 132 | if self.stream.read(2) == b'\x00\x00': 133 | self.stream.seek(total_osd2_offset) 134 | # print(self.stream.read(12)) 135 | self.stream.write(instream_chunk) 136 | return True 137 | else: 138 | return False 139 | 140 | def _clear_osd2(self, inode_nr: int): 141 | total_osd2_offset = self._get_total_osd2_offset(inode_nr) 142 | self.stream.seek(total_osd2_offset) 143 | self.stream.write(b"\x00\x00") 144 | 145 | def _read_from_osd2(self, inode_nr: int): 146 | self.stream.seek(0) 147 | total_osd2_offset = self._get_total_osd2_offset(inode_nr) 148 | self.stream.seek(total_osd2_offset) 149 | data = self.stream.read(2) 150 | # print(data) 151 | return data 152 | 153 | def _get_total_osd2_offset(self, inode_nr: int) -> int: 154 | inode_size = self.ext4fs.superblock.data["inode_size"] 155 | # print("table start", self.inode_table.table_start) 156 | 157 | return self.inode_table.inodes[inode_nr].offset + 0x74 + 0xA 158 | 159 | def _check_if_supported(self, instream) -> bool: 160 | if len(instream) >= ((self.ext4fs.superblock.data["inode_count"]) * 2): 161 | return False 162 | else: 163 | return True 164 | -------------------------------------------------------------------------------- /fishy/ext4/obso_faddr.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import math 3 | import typing as typ 4 | 5 | from fishy.ext4.ext4_filesystem.EXT4 import EXT4 6 | 7 | LOGGER = logging.getLogger("ext4-obso_faddr") 8 | 9 | 10 | class EXT4FADDRMetadata: 11 | """ 12 | holds inode numbers which hold the hidden data in. 13 | """ 14 | def __init__(self, d: dict = None): 15 | """ 16 | :param d: dict, dictionary representation of a EXT4FADDRMetadata 17 | object 18 | """ 19 | if d is None: 20 | # self.inode_table = None 21 | self.inode_numbers = [] 22 | else: 23 | # self.inode_table = d["inode_table"] 24 | self.inode_numbers = d["inode_numbers"] 25 | 26 | def add_inode_number(self, inode_number: int) -> None: 27 | """ 28 | adds a block to the list of blocks 29 | 30 | :param block_id: int, id of the block 31 | """ 32 | self.inode_numbers.append(inode_number) 33 | 34 | def get_inode_numbers(self) \ 35 | -> []: 36 | """ 37 | returns list of inode_numbers 38 | 39 | :returns: list of inode_numbers 40 | """ 41 | return self.inode_numbers 42 | 43 | class EXT4FADDR: 44 | """ 45 | Hides data in obso_faddr field of inodes in the first inode_table. 46 | """ 47 | def __init__(self, stream: typ.BinaryIO, dev: str): 48 | """ 49 | :param dev: path to an ext4 filesystem 50 | 51 | :param stream: filedescriptor of an ext4 filesystem 52 | """ 53 | self.dev = dev 54 | self.stream = stream 55 | self.ext4fs = EXT4(stream, dev) 56 | self.inode_table = self.ext4fs.inode_tables[0] 57 | 58 | def write(self, instream: typ.BinaryIO) -> EXT4FADDRMetadata: 59 | """ 60 | writes from instream into the last two bytes of inodes obso_faddr field. 61 | This method currently supports only data sizes less than 4000 bytes. 62 | 63 | :param instream: stream to read from 64 | 65 | :return: EXT4FADDRMetadata 66 | """ 67 | metadata = EXT4FADDRMetadata() 68 | instream = instream.read() 69 | 70 | if not self._check_if_supported(instream): 71 | raise IOError("The hiding data size is currently not supported") 72 | 73 | 74 | instream_chunks = [instream[i:i+4] for i in range(0, len(instream), 4)] 75 | # print(instream_chunks) 76 | inode_number = 1 77 | hidden_chunks = 0 78 | 79 | while hidden_chunks < len(instream_chunks): 80 | chunk = instream_chunks[hidden_chunks] 81 | 82 | if self._write_to_obso_faddr(chunk, inode_number): 83 | metadata.add_inode_number(inode_number) 84 | hidden_chunks += 1 85 | 86 | inode_number += 1 87 | 88 | return metadata 89 | 90 | def read(self, outstream: typ.BinaryIO, metadata: EXT4FADDRMetadata) \ 91 | -> None: 92 | """ 93 | writes data hidden in obso_faddr blocks into outstream 94 | 95 | :param outstream: stream to write into 96 | 97 | :param metadata: EXT4FADDRMetadata object 98 | """ 99 | inode_numbers = metadata.get_inode_numbers() 100 | # print(inode_numbers) 101 | for nr in inode_numbers: 102 | outstream.write(self._read_from_obso_faddr(nr)) 103 | 104 | def clear(self, metadata: EXT4FADDRMetadata) -> None: 105 | """ 106 | clears the obso_faddr field in which data has been hidden 107 | 108 | :param metadata: EXT4FADDRMetadata object 109 | """ 110 | inode_numbers = metadata.get_inode_numbers() 111 | for nr in inode_numbers: 112 | self._clear_obso_faddr(nr) 113 | 114 | def info(self, metadata: EXT4FADDRMetadata = None) -> None: 115 | """ 116 | shows info about inode obso_faddr fields and data hiding space 117 | 118 | :param metadata: EXT4FADDRMetadata object 119 | """ 120 | print("Inodes: " + str(self.ext4fs.superblock.data["inode_count"])) 121 | print("Total hiding space in obso_faddr fields: " + str((self.ext4fs.superblock.data["inode_count"]) * 4) + " Bytes") 122 | if metadata != None: 123 | filled_inode_numbers = metadata.get_inode_numbers() 124 | print('Used: ' + str(len(filled_inode_numbers) * 4) + ' Bytes') 125 | 126 | def _write_to_obso_faddr(self, instream_chunk, inode_nr) -> bool: 127 | # print(instream_chunk) 128 | self.stream.seek(0) 129 | total_obso_faddr_offset = self._get_total_obso_faddr_offset(inode_nr) 130 | # print(total_obso_faddr_offset) 131 | self.stream.seek(total_obso_faddr_offset) 132 | if self.stream.read(4) == b'\x00\x00\x00\x00': #\x00\x00 133 | self.stream.seek(total_obso_faddr_offset) 134 | # print(self.stream.read(12)) 135 | self.stream.write(instream_chunk) 136 | return True 137 | else: 138 | return False 139 | 140 | def _clear_obso_faddr(self, inode_nr: int): 141 | total_obso_faddr_offset = self._get_total_obso_faddr_offset(inode_nr) 142 | self.stream.seek(total_obso_faddr_offset) 143 | self.stream.write(b"\x00\x00\x00\x00") #\x00\x00 144 | 145 | def _read_from_obso_faddr(self, inode_nr: int): 146 | self.stream.seek(0) 147 | total_obso_faddr_offset = self._get_total_obso_faddr_offset(inode_nr) 148 | self.stream.seek(total_obso_faddr_offset) 149 | data = self.stream.read(4) 150 | # print(data) 151 | return data 152 | 153 | def _get_total_obso_faddr_offset(self, inode_nr: int) -> int: 154 | inode_size = self.ext4fs.superblock.data["inode_size"] 155 | # print("table start", self.inode_table.table_start) 156 | 157 | return self.inode_table.inodes[inode_nr].offset + 0x70 158 | 159 | def _check_if_supported(self, instream) -> bool: 160 | if len(instream) >= ((self.ext4fs.superblock.data["inode_count"]) * 4): 161 | return False 162 | else: 163 | return True 164 | -------------------------------------------------------------------------------- /doc/source/02_module_overview.rst: -------------------------------------------------------------------------------- 1 | Architecture Overview 2 | ===================== 3 | 4 | General module structure 5 | ------------------------ 6 | 7 | The following flowchart diagram represents the logical procedure of using a 8 | hiding technique. 9 | 10 | .. image:: _static/module_flowchart.png 11 | 12 | The `CLI` evaluates the command line parameters and calls the appropriate `hiding 13 | technique wrapper`. 14 | The `hiding technique wrapper` then checks for the filesystem type of the given 15 | filesystem image and calls the specific `hiding technique` implementation for 16 | this filesystem. 17 | In case of calling the write method, the `hiding technique` implementation 18 | returns metadata, which it needs to restore the hidden data later. This metadata 19 | is written to disk, using a simple json datastructure. 20 | 21 | The command line argument parsing part is implemented in the `cli.py` module. 22 | `Hiding techniques wrapper` are located in the root module. 23 | They adopt converting input data into streams, casting/reading/writing hiding 24 | technique specific metadata and calling the appropriate methods of those hiding 25 | technique specific implementations. 26 | To detect the filesystem type of a given image, the `Hiding techniques wrapper` 27 | use the `filesystem_detector`, which uses filesystem detection methods, implemented 28 | in the particular filesystem module. 29 | Filesystem specific `hiding technique` implementations provide at least a write, 30 | read and a clear method to hide/read/delete data. 31 | `Hiding technique` implementation use either `pytsk3` to gather information of 32 | the given filesystem or use custom filesystem parsers, which then are located 33 | under the particular filesystem package. 34 | 35 | CLI 36 | --- 37 | 38 | The cli forms the user interface for this toolkit. Each hiding technique is 39 | accessible via a subcommand, which itself defines further options. The CLI 40 | must be able to read data, that the user wants to hide either from stdin or 41 | from a file. Hidden data which the user wants to read from a filesystem are 42 | returned to stdout or a given file. 43 | 44 | The cli module takes up the task of parsing the command line arguments and calls, 45 | depending on the given subcommand, the appropriate `Hiding technique wrapper`. 46 | 47 | If reading data from a file, the cli is in charge of turning it into a buffered 48 | stream, on which the hiding technique operates. 49 | 50 | .. note:: The `CLI` is limited to store only one file per call and won't honour 51 | other files already written to disk. 52 | 53 | Hiding technique wrapper 54 | ------------------------ 55 | 56 | Each type of hiding technique has its own wrapper. This hiding technique wrapper 57 | gets called by the CLI and calls the filesystem specific `hiding technique 58 | implementation`, based on the filesystem type. To detect the filesystem type, the 59 | `filesystem detector` function is called. 60 | 61 | Read and clear methods of the hiding techniques require some metadata which 62 | are gathered during a write operation. So the hiding technique wrapper is also 63 | responsible for reading and writing metadata files and providing hiding technique 64 | specific metadata objects for read and write methods. 65 | 66 | If the user wants to read hidden data into a file instead of stdout, the hiding 67 | technique wrapper is in charge of opening and writing this file. 68 | 69 | Hiding technique 70 | ---------------- 71 | 72 | The hiding technique implementations do the real work of this toolkit. Every 73 | hiding technique must at least offer a read, write and a clear method. They 74 | must operate on streams only to ensure high reusability and reduce boilerplate 75 | code. 76 | 77 | All hiding techniques are called by a `hiding technique wrapper`. 78 | 79 | To get required information about the filesystem the hiding techniques use 80 | either the `pytsk3` library or use a filesystem parser implementation located 81 | in the aproppriate filesystem package. 82 | 83 | The clear method must overwrite all hidden data with zeros and leave the filesystem 84 | in a consistent state. 85 | 86 | .. warning:: The clear method does not perform erasure of data in terms of any 87 | regulatory compliance. It does not ensure that all possible traces 88 | are removed from the filesystem. So, don't rely on this method to 89 | securely wipe hidden data from disk. 90 | 91 | If a hiding technique needs some metadata to restore hidden data, it must 92 | implement a hiding technique specific metadata class. This is used during the 93 | write process to store those necessary information. The write method must return 94 | this metadata instance, so that the `hiding technique wrapper` can serialize it 95 | and pass it to the read and clear methods. 96 | 97 | If a write method failes, already written data must be cleared before exiting. 98 | 99 | Hiding techniques may implement further methods. 100 | 101 | Filesystem detector 102 | ------------------- 103 | 104 | The filesystem detector is a simple wrapper function to unify calls to the 105 | filesystem specific detection functions, which are implemented in the 106 | corresponding filesystem package. 107 | 108 | Metadata handling 109 | ----------------- 110 | 111 | To be able to restore hidden data, most hiding techniques will need some 112 | additional information. These information will be stored in a json metadata file. 113 | The `fishy.metadata` class provides functions to read and write metadata files 114 | and automatically de-/encrypting the metadata if a password is provided. The 115 | purpose of this class is to ensure, that all metadata files have a similar 116 | datastructure. Though the program can detect at an early point, that for 117 | example a user uses the wrong hiding technique to restore hidden data. This 118 | metadata class we can call the 'main-metadata' class. 119 | 120 | When implementing a hiding technique, this technique must implement its own, 121 | hiding technique specific, metadata class. So the hiding technique itself defines 122 | which data will be stored. The write method then returns this technique specific 123 | metadata class which then gets serialized and stored in the main-metadata. 124 | -------------------------------------------------------------------------------- /fishy/wrapper/superblock_slack.py: -------------------------------------------------------------------------------- 1 | """ 2 | Wrapper for filesystem specific implementations of hiding data inside superblock slackspace 3 | """ 4 | import logging 5 | import typing as typ 6 | from os import path 7 | from fishy.filesystem_detector import get_filesystem_type 8 | from fishy.metadata import Metadata 9 | from fishy.ext4.superblock_slack import EXT4SuperblockSlack 10 | from fishy.ext4.superblock_slack import EXT4SuperblockSlackMetadata 11 | from fishy.APFS.Superblock_Slack import APFSSuperblockSlack 12 | from fishy.APFS.Superblock_Slack import APFSSuperblockSlackMetaData 13 | 14 | 15 | LOGGER = logging.getLogger("SuperblockSlack") 16 | 17 | class SuperblockSlack: 18 | """ 19 | This class wraps the filesystem specific implementation of the superblock slack hiding technique 20 | """ 21 | def __init__(self, fs_stream: typ.BinaryIO, metadata: Metadata, 22 | dev: str = None): 23 | """ 24 | :param dev: Path to filesystem 25 | :param fs_stream: Stream of filesystem 26 | :param metadata: Metadata object 27 | """ 28 | self.dev = dev 29 | self.metadata = metadata 30 | self.fs_type = get_filesystem_type(fs_stream) 31 | if self.fs_type == 'EXT4': 32 | self.metadata.set_module("ext4-superblock-slack") 33 | self.fs = EXT4SuperblockSlack(fs_stream, dev) # pylint: disable=invalid-name 34 | elif self.fs_type == 'APFS': 35 | self.metadata.set_module("apfs-superblock-slack") 36 | self.fs = APFSSuperblockSlack(fs_stream) 37 | else: 38 | raise NotImplementedError() 39 | 40 | def write(self, instream: typ.BinaryIO, 41 | filename: str = None) -> None: 42 | """ 43 | writes data from instream into superblock slack. Metadata of 44 | those files will be stored in Metadata object 45 | 46 | :param instream: stream to read data from 47 | :param filename: name that will be used, when file gets written into superblock slack. 48 | If none, a random name will be generated. 49 | :raises: IOError 50 | """ 51 | LOGGER.info("Write") 52 | if filename is not None: 53 | filename = path.basename(filename) 54 | if self.fs_type == 'EXT4': 55 | LOGGER.info("Write into ext4") 56 | superblock_slack_metadata = self.fs.write(instream) 57 | self.metadata.add_file(filename, superblock_slack_metadata) 58 | elif self.fs_type == 'APFS': 59 | LOGGER.info("Write into APFS") 60 | superblock_slack_metadata = self.fs.write(instream) 61 | self.metadata.add_file(filename, superblock_slack_metadata) 62 | else: 63 | raise NotImplementedError() 64 | 65 | def read(self, outstream: typ.BinaryIO): 66 | """ 67 | writes hidden data from superblock slack into stream. 68 | 69 | :param outstream: stream to write hidden data into 70 | :raises: IOError 71 | """ 72 | file_metadata = self.metadata.get_file("0")['metadata'] 73 | if self.fs_type == 'EXT4': 74 | superblock_slack_metadata = EXT4SuperblockSlackMetadata(file_metadata) 75 | self.fs.read(outstream, superblock_slack_metadata) 76 | elif self.fs_type == 'APFS': 77 | superblock_slack_metadata = APFSSuperblockSlackMetaData(file_metadata) 78 | self.fs.read(outstream, superblock_slack_metadata) 79 | else: 80 | raise NotImplementedError() 81 | 82 | def read_into_file(self, outfilepath: str): 83 | """ 84 | reads hidden data from superblock slack into files 85 | :note: If provided filepath already exists, this file will be 86 | overwritten without a warning. 87 | :param outfilepath: filepath to file, where hidden data will be 88 | restored into 89 | """ 90 | if self.fs_type == 'EXT4': 91 | with open(outfilepath, 'wb+') as outfile: 92 | self.read(outfile) 93 | if self.fs_type == 'APFS': 94 | with open(outfilepath, 'wb+') as outfile: 95 | self.read(outfile) 96 | else: 97 | raise NotImplementedError() 98 | 99 | def clear(self): 100 | """ 101 | clears superblock slack in which data has been hidden 102 | :param metadata: Metadata, object where metadata is stored in 103 | :raises: IOError 104 | """ 105 | if self.fs_type == 'EXT4': 106 | for file_entry in self.metadata.get_files(): 107 | file_metadata = file_entry['metadata'] 108 | file_metadata = EXT4SuperblockSlackMetadata(file_metadata) 109 | self.fs.clear(file_metadata) 110 | elif self.fs_type == 'APFS': 111 | for file_entry in self.metadata.get_files(): 112 | file_metadata = file_entry['metadata'] 113 | file_metadata = APFSSuperblockSlackMetaData(file_metadata) 114 | self.fs.clear(file_metadata) 115 | else: 116 | raise NotImplementedError() 117 | 118 | def info(self): 119 | """ 120 | shows info about superblock slack and data hiding space 121 | :param metadata: Metadata, object where metadata is stored in 122 | :raises: NotImplementedError 123 | """ 124 | if self.fs_type == 'EXT4': 125 | if len(list(self.metadata.get_files())) > 0: 126 | for file_entry in self.metadata.get_files(): 127 | file_metadata = file_entry['metadata'] 128 | file_metadata = EXT4SuperblockSlackMetadata(file_metadata) 129 | self.fs.info(file_metadata) 130 | else: 131 | self.fs.info() 132 | if self.fs_type == 'APFS': 133 | if len(list(self.metadata.get_files())) > 0: 134 | for file_entry in self.metadata.get_files(): 135 | file_metadata = file_entry['metadata'] 136 | file_metadata = APFSSuperblockSlackMetaData(file_metadata) 137 | self.fs.info(file_metadata) 138 | else: 139 | self.fs.info() 140 | else: 141 | raise NotImplementedError() 142 | -------------------------------------------------------------------------------- /doc/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # fishy documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Dec 5 12:49:58 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | import os 21 | import sys 22 | sys.path.insert(0, os.path.abspath('../fishy')) 23 | autoclass_content = 'both' 24 | 25 | 26 | # -- General configuration ------------------------------------------------ 27 | 28 | # If your documentation needs a minimal Sphinx version, state it here. 29 | # 30 | # needs_sphinx = '1.0' 31 | 32 | # Add any Sphinx extension module names here, as strings. They can be 33 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 34 | # ones. 35 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.graphviz', 'sphinxarg.ext'] 36 | 37 | # Add any paths that contain templates here, relative to this directory. 38 | templates_path = ['_templates'] 39 | 40 | # The suffix(es) of source filenames. 41 | # You can specify multiple suffix as a list of string: 42 | # 43 | # source_suffix = ['.rst', '.md'] 44 | source_suffix = '.rst' 45 | 46 | # The master toctree document. 47 | master_doc = 'toc' 48 | 49 | # General information about the project. 50 | project = 'fishy' 51 | copyright = '2018, Deniz Celik, Tim Christen, Matthias Greune, Christian Hecht, Adrian Kailus, Dustin Kern, Patrick Naili, Chau Nguyen, Yannick Mau' 52 | author = 'Deniz Celik, Tim Christen, Matthias Greune, Christian Hecht, Adrian Kailus, Dustin Kern, Patrick Naili, Chau Nguyen, Yannick Mau' 53 | 54 | # The version info for the project you're documenting, acts as replacement for 55 | # |version| and |release|, also used in various other places throughout the 56 | # built documents. 57 | # 58 | # The short X.Y version. 59 | version = '0.2' 60 | # The full version, including alpha/beta/rc tags. 61 | release = '0.2' 62 | 63 | # The language for content autogenerated by Sphinx. Refer to documentation 64 | # for a list of supported languages. 65 | # 66 | # This is also used if you do content translation via gettext catalogs. 67 | # Usually you set "language" from the command line for these cases. 68 | language = None 69 | 70 | # List of patterns, relative to source directory, that match files and 71 | # directories to ignore when looking for source files. 72 | # This patterns also effect to html_static_path and html_extra_path 73 | exclude_patterns = [] 74 | 75 | # The name of the Pygments (syntax highlighting) style to use. 76 | pygments_style = 'sphinx' 77 | 78 | # If true, `todo` and `todoList` produce output, else they produce nothing. 79 | todo_include_todos = False 80 | 81 | 82 | # -- Options for HTML output ---------------------------------------------- 83 | 84 | # The theme to use for HTML and HTML Help pages. See the documentation for 85 | # a list of builtin themes. 86 | # 87 | html_theme = 'alabaster' 88 | 89 | # Theme options are theme-specific and customize the look and feel of a theme 90 | # further. For a list of options available for each theme, see the 91 | # documentation. 92 | # 93 | # html_theme_options = {} 94 | 95 | # Add any paths that contain custom static files (such as style sheets) here, 96 | # relative to this directory. They are copied after the builtin static files, 97 | # so a file named "default.css" will overwrite the builtin "default.css". 98 | html_static_path = ['_static'] 99 | 100 | # Custom sidebar templates, must be a dictionary that maps document names 101 | # to template names. 102 | # 103 | # This is required for the alabaster theme 104 | # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars 105 | html_sidebars = { 106 | '**': [ 107 | 'globaltoc.html', 108 | 'relations.html', # needs 'show_related': True theme option to display 109 | 'searchbox.html', 110 | ] 111 | } 112 | 113 | 114 | # -- Options for HTMLHelp output ------------------------------------------ 115 | 116 | # Output file base name for HTML help builder. 117 | htmlhelp_basename = 'fishydoc' 118 | 119 | 120 | # -- Options for LaTeX output --------------------------------------------- 121 | 122 | latex_elements = { 123 | # The paper size ('letterpaper' or 'a4paper'). 124 | # 125 | # 'papersize': 'letterpaper', 126 | 127 | # The font size ('10pt', '11pt' or '12pt'). 128 | # 129 | # 'pointsize': '10pt', 130 | 131 | # Additional stuff for the LaTeX preamble. 132 | # 133 | # 'preamble': '', 134 | 135 | # Latex figure (float) alignment 136 | # 137 | # 'figure_align': 'htbp', 138 | } 139 | 140 | # Grouping the document tree into LaTeX files. List of tuples 141 | # (source start file, target name, title, 142 | # author, documentclass [howto, manual, or own class]). 143 | latex_documents = [ 144 | (master_doc, 'fishy.tex', 'fishy Documentation', 145 | 'Deniz Celik, Tim Christen, Matthias Greune, \\\\\n Christian Hecht, Adrian Kailus, Dustin Kern, \\\\\n Patrick Naili, Chau Nguyen, Yannick Mau', 'manual'), 146 | ] 147 | 148 | latex_elements = { 149 | 'classoptions': ',openany,oneside' # remove blank pages between chapters 150 | } 151 | 152 | latex_appendices = ['07_appendix'] 153 | 154 | # -- Options for manual page output --------------------------------------- 155 | 156 | # One entry per manual page. List of tuples 157 | # (source start file, name, description, authors, manual section). 158 | man_pages = [ 159 | (master_doc, 'fishy', 'fishy Documentation', 160 | [author], 1) 161 | ] 162 | 163 | 164 | # -- Options for Texinfo output ------------------------------------------- 165 | 166 | # Grouping the document tree into Texinfo files. List of tuples 167 | # (source start file, target name, title, author, 168 | # dir menu entry, description, category) 169 | texinfo_documents = [ 170 | (master_doc, 'fishy', 'fishy Documentation', 171 | author, 'fishy', 'Toolkit for filesystem based hiding techniques', 172 | 'Miscellaneous'), 173 | ] 174 | 175 | 176 | 177 | -------------------------------------------------------------------------------- /fishy/fat/fat_filesystem/fat_12.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implementation of a FAT12 filesystem reader 3 | 4 | example usage: 5 | >>> with open('testfs.dd', 'rb') as filesystem: 6 | >>> fs = FAT12(filesystem) 7 | 8 | example to print all entries in root directory: 9 | >>> for i, v in fs.get_root_dir_entries(): 10 | >>> if v != "": 11 | >>> print(v) 12 | 13 | example to print all fat entries 14 | >>> for i in range(fs.entries_per_fat): 15 | >>> print(i,fs.get_cluster_value(i)) 16 | 17 | example to print all root directory entries 18 | >>> for entry in fs.get_root_dir_entries(): 19 | >>> print(v, entry.get_start_cluster()) 20 | 21 | """ 22 | import typing as typ 23 | from construct import Struct, Array, Padding, Embedded, Bytes, this 24 | from .bootsector import FAT12_16_BOOTSECTOR 25 | from .dir_entry import DIR_ENTRY_DEFINITION as DIR_ENTRY 26 | from .fat import FAT 27 | from .fat_entry import FAT12Entry 28 | 29 | 30 | FAT12_PRE_DATA_REGION = Struct( 31 | "bootsector" / Embedded(FAT12_16_BOOTSECTOR), 32 | Padding((this.reserved_sector_count - 1) * this.sector_size), 33 | # FATs. 34 | "fats" / Array(this.fat_count, Bytes(this.sectors_per_fat * this.sector_size)), 35 | # RootDir Table 36 | "rootdir" / Bytes(this.rootdir_entry_count * DIR_ENTRY.sizeof()) 37 | ) 38 | 39 | 40 | class FAT12(FAT): 41 | """ 42 | FAT12 filesystem implementation. 43 | """ 44 | def __init__(self, stream): 45 | """ 46 | :param stream: filedescriptor of a FAT12 filesystem 47 | """ 48 | super().__init__(stream, FAT12_PRE_DATA_REGION) 49 | self.entries_per_fat = int(self.pre.sectors_per_fat 50 | * self.pre.sector_size 51 | * 8 / 12) 52 | self._fat_entry = FAT12Entry 53 | self.fat_type = 'FAT12' 54 | 55 | def get_cluster_value(self, cluster_id: int) -> typ.Union[int, str]: 56 | """ 57 | finds the value that is written into fat 58 | for given cluster_id 59 | :param cluster_id: int, cluster that will be looked up 60 | :return: int or string 61 | """ 62 | # as python read does not allow to read simply 12 bit, 63 | # we need to do some fancy stuff to extract those from 64 | # 16 bit long reads 65 | # this strange layout results from the little endianess 66 | # which causes that: 67 | # * clusternumbers beginning at the start of a byte use this 68 | # byte + the second nibble of the following byte. 69 | # * clusternumbers beginning in the middle of a byte use 70 | # the first nibble of this byte + the second byte 71 | # because of little endianess these nibbles need to be 72 | # reordered as by default int() interpretes hexstrings as 73 | # big endian 74 | # 75 | byte = cluster_id + int(cluster_id/2) 76 | byte_slice = self.pre.fats[0][byte:byte+2] 77 | if cluster_id % 2 == 0: 78 | # if cluster_number is even, we need to wipe the third nibble 79 | hexvalue = byte_slice.hex() 80 | value = int(hexvalue[3] + hexvalue[0:2], 16) 81 | else: 82 | # if cluster_number is odd, we need to wipe the second nibble 83 | hexvalue = byte_slice.hex() 84 | value = int(hexvalue[2:4] + hexvalue[0], 16) 85 | return self._fat_entry.parse(value.to_bytes(2, 'little')) 86 | 87 | def write_fat_entry(self, cluster_id: int, 88 | value: typ.Union[int, str]) -> None: 89 | # make sure cluster_id is valid 90 | if cluster_id < 0 or cluster_id >= self.entries_per_fat: 91 | raise AttributeError("cluster_id out of bounds") 92 | # make sure user does not input invalid values as next cluster 93 | if isinstance(value, int): 94 | assert value <= 4086, "next_cluster value must be <= 4086. For " \ 95 | + "last cluster use 'last_cluster'. For " \ 96 | + "bad_cluster use 'bad_cluster'" 97 | assert value >= 2, "next_cluster value must be >= 2. For " \ 98 | + "free_cluster use 'free_cluster'" 99 | # get start position of FAT0 100 | fat0_start = self.offset + 512 + (self.pre.sector_size - 512) + \ 101 | (self.pre.reserved_sector_count - 1) * self.pre.sector_size 102 | fat1_start = fat0_start + self.pre.sectors_per_fat \ 103 | * self.pre.sector_size 104 | # read current entry 105 | byte = cluster_id + int(cluster_id/2) 106 | self.stream.seek(fat0_start + byte) 107 | current_entry = self.stream.read(2).hex() 108 | new_entry_hex = self._fat_entry.build(value).hex() 109 | # calculate new entry as next entry overlaps with current bytes 110 | if cluster_id % 2 == 0: 111 | # if cluster_number is even, we need to keep the third nibble 112 | new_entry = new_entry_hex[0:2] + current_entry[2] \ 113 | + new_entry_hex[3] 114 | else: 115 | # if cluster_number is odd, we need to keep the second nibble 116 | new_entry = new_entry_hex[1] + current_entry[1] + \ 117 | new_entry_hex[3] + new_entry_hex[0] 118 | # convert hex to bytes 119 | new_entry = bytes.fromhex(new_entry) 120 | # write new value to first fat on disk 121 | self.stream.seek(fat0_start + byte) 122 | self.stream.write(new_entry) 123 | # write new value to second fat on disk if it exists 124 | if self.pre.fat_count > 1: 125 | self.stream.seek(fat1_start + byte) 126 | self.stream.write(new_entry) 127 | # flush changes to disk 128 | self.stream.flush() 129 | # re-read fats into memory 130 | fat_definition = Array(self.pre.fat_count, 131 | Bytes(self.pre.sectors_per_fat * 132 | self.pre.sector_size)) 133 | self.stream.seek(fat0_start) 134 | self.pre.fats = fat_definition.parse_stream(self.stream) 135 | 136 | def _root_to_stream(self, stream: typ.BinaryIO) -> None: 137 | """ 138 | write root directory into a given stream 139 | :param stream: stream, where the root directory will be written into 140 | """ 141 | stream.write(self.pre.rootdir) 142 | --------------------------------------------------------------------------------