├── .gitignore ├── .gitmodules ├── .travis.yml ├── ChangeLog ├── LICENSE ├── MANIFEST.in ├── README.md ├── _version.py ├── aff4.py ├── pyaff4 ├── __init__.py ├── _version.py ├── abort_test.py ├── aes_keywrap.py ├── aff4.py ├── aff4_cloud.py ├── aff4_directory.py ├── aff4_directory_test.py ├── aff4_file.py ├── aff4_image.py ├── aff4_image_test.py ├── aff4_image_test2.py ├── aff4_imager_utils.py ├── aff4_map.py ├── aff4_map_test.py ├── aff4_metadata.py ├── aff4_utils.py ├── block_hasher.py ├── container.py ├── container_test.py ├── crypt_image_test.py ├── data_store.py ├── data_store_test.py ├── dedup_test.py ├── encrypted_stream.py ├── encryptedstream_test.py ├── escaping.py ├── escaping_test.py ├── hashes.py ├── hashing_test.py ├── hexdump.py ├── keybag.py ├── lexicon.py ├── linear_hasher.py ├── logical.py ├── logical_append_test.py ├── logical_test.py ├── plugins.py ├── random_imagestream_test.py ├── rdfvalue.py ├── rdfvalue_test.py ├── reference_test.py ├── registry.py ├── standards_test.py ├── statx.py ├── stream_factory.py ├── stream_test.py ├── streams.py ├── struct_parser.py ├── symbolic_streams.py ├── test.sh ├── test_crypto.py ├── turtle.py ├── utils.py ├── version.py ├── zip.py ├── zip_test.py ├── zip_test_extended.py ├── zip_test_unicode.py └── zip_test_unicode2.py ├── requirements.txt ├── samples ├── extract_streams.py └── simple_block_read.py ├── setup.py ├── test_images ├── AFF4-L │ ├── broken-dedupe.aff4 │ ├── dream.aff4 │ ├── dream.aff4.information.turtle │ ├── dream.txt │ ├── information.turtle │ ├── paper-hash_based_disk_imaging_using_aff4.pdf │ ├── paper-hash_based_disk_imaging_using_aff4.pdf.frag.1 │ ├── paper-hash_based_disk_imaging_using_aff4.pdf.frag.2 │ ├── unicode.aff4 │ ├── unicode.zip │ ├── utf8segment-macos.zip │ └── ネコ.txt ├── AFF4PreStd │ ├── Base-Allocated.af4 │ ├── Base-Linear-ReadError.af4 │ ├── Base-Linear.af4 │ └── README.txt ├── AFF4Std │ ├── Base-Allocated.aff4 │ ├── Base-Linear-AllHashes.aff4 │ ├── Base-Linear-ReadError.aff4 │ ├── Base-Linear.aff4 │ ├── README.txt │ └── Striped │ │ ├── Base-Linear_1.aff4 │ │ └── Base-Linear_2.aff4 ├── README.md └── keys │ ├── certificate.pem │ └── key.pem ├── version.py └── version.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info/ 3 | dist/ 4 | /.idea/ 5 | /build/ 6 | *~ 7 | /venv/ 8 | /venv2/ 9 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "test_images"] 2 | path = test_images 3 | url = https://github.com/aff4/ReferenceImages.git 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - '2.7' 4 | cache: pip 5 | install: 6 | - pip install -r requirements.txt 7 | script: 8 | - python setup.py test 9 | deploy: 10 | provider: pypi 11 | user: blschatz 12 | password: 13 | secure: YRp+RZXF6Qkdjyt/5tEp/Bagj+SxlUUHokyY35AKLPqLYZktA1sJ8t1LRtJQ8vMv8pYWkpHbSF0tmmPOfhRCFs5CxAOACPXFnSRJRbe32E9u8g/Ffko+0V1WS4EFbQwNqgdpMKZ7LymGN01cYWPsZgN4wVUvXiJGK9yh4lTSiEhvaiIG8LuEofOuD9VCofBdMowkCAxw81v6lKGAhljZ2KuGbAmzczLDKl7CamdjpvZaFk2Yt9M0ZdNiDvZMumwnxaYuy4MGvTe+v0ccZ4WxwD6FcbVTdcPiZChW1M5fUqweKW0cXIDHDl15UhniGKZbuLAdapr7aaeXo71fkQ1nuJqGVtGT0yD7tK4EdGWAOG452kRiHmvuLkUf4c5VSwNU0P7h5VvARGWbSgj91YTR81+0qjD6W/J0bSd6SutRmzwzUo3wxtpUdl9YHToXaTYF3fq8/sxIzjvC1Y/appM4vnRg1VUR9qkzqatnBYdPCNmyu97uWV7LtmolegdZsVz8Teb3ksBMvd5wZVNs8+A7mBiDyMLNSm/8jYm1C6lbUyyO3Gvp5kEhmJ/fNum+bfrGuK2sjt7AlPNSW8LawkDdX0D/QqDYe/TS1KJXCJ/WZgzJam1SVnI3Ti7Z36cyZFNxgAC5tbyxvxYIp+O5VB9LppYfdHSTl48sR53yhI8BvyI= 14 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | --v0.33 2 | + Fix performance regressions. 3 | + Make logging more efficient. 4 | + Remove invalid unit test. 5 | --v0.32 6 | + Fix bug in reference counting 7 | + Tune lower encryption container for efficiency 8 | + Improve logging 9 | -- v0.31 10 | + Support for encrypted image streams. 11 | + Enable abort of single logical file acqusitions before close, reclaiming space. 12 | + Random IO image stream. 13 | -- v0.30 14 | + In-progress support for encrypted image streams. 15 | -- v0.29 16 | + Fix double load of RDF 17 | -- v0.28 18 | + Replace dependency pyblake2 with pynacl (Apache Licence) 19 | + Add another sample - simple_block_read.py 20 | + Fix invalid dateTime RDF 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.txt 2 | 3 | recursive-include pyaff4 * 4 | 5 | recursive-exclude * *.pyc 6 | exclude .gitignore 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AFF4 -The Advanced Forensics File Format 2 | 3 | The Advanced Forensics File Format 4 (AFF4) is an open source format used for 4 | the storage of digital evidence and data. 5 | 6 | It was originally designed and published in [1] and has since been standardised 7 | as the AFF4 Standard v1.0, which is available at 8 | https://github.com/aff4/Standard. This project is a work in progress 9 | implementation, providing two library implementations, C/C++ and Python. 10 | 11 | ## What is currently supported. 12 | 13 | The focus of this implementation is reading physical images conforming with the 14 | AFF4 Standard v1.0, and for the ongoing development of an AFF4 based logical 15 | image standard. 16 | 17 | Canonical images for the v1.0 physical image specification are provided in the 18 | AFF4 Reference Images github project at https://github.com/aff4/ReferenceImages 19 | 20 | 1. Reading, writing & appending to ZipFile style volumes. 21 | 2. Reading striped ZipFile volumes. 22 | 2. Reading & writing AFF4 ImageStreams using the deflate or snappy compressor. 23 | 3. Reading RDF metadata using Turtle (and to some degree YAML). 24 | 4. Verification of linear and block hashed images. 25 | 5. Reading & writing logical images (*new*) . 26 | 6. Reading & writing deduplicated logical images (*new*). 27 | 7. Encrypted AFF4 logical volumes (*new*). 28 | 29 | ## What is not yet supported: 30 | 31 | The write support in the libraries is currently broken and being worked on. 32 | Other aspects of the AFF4 that have not yet been implemented in this codebase 33 | include: 34 | 35 | 1. Persistent data store (resolver). 36 | 2. HTTP backed streams. 37 | 3. Support for signed statements or Bill of Materials. 38 | 4. Directory based volumes. 39 | 40 | ## Notice 41 | 42 | This is not an official Google product (experimental or otherwise), it is just 43 | code that happens to be owned by Google and Schatz Forensic. 44 | 45 | ## References 46 | 47 | [1] "Extending the advanced forensic format to accommodate multiple data 48 | sources, logical evidence, arbitrary information and forensic workflow" M.I. 49 | Cohen, Simson Garfinkel and Bradley Schatz, digital investigation 6 (2009) 50 | S57-S68. 51 | -------------------------------------------------------------------------------- /_version.py: -------------------------------------------------------------------------------- 1 | 2 | # Machine Generated - do not edit! 3 | 4 | # This file is produced when the main "version.py update" command is run. That 5 | # command copies this file to all sub-packages which contain 6 | # setup.py. Configuration is maintain in version.yaml at the project's top 7 | # level. 8 | 9 | def get_versions(): 10 | return tag_version_data(raw_versions(), """version.yaml""") 11 | 12 | def raw_versions(): 13 | return json.loads(""" 14 | { 15 | "post": "0", 16 | "rc": "0", 17 | "version": "0.32" 18 | } 19 | """) 20 | 21 | import json 22 | import os 23 | import subprocess 24 | 25 | try: 26 | # We are looking for the git repo which contains this file. 27 | MY_DIR = os.path.dirname(os.path.abspath(__file__)) 28 | except: 29 | MY_DIR = None 30 | 31 | def is_tree_dirty(): 32 | try: 33 | return bool(subprocess.check_output( 34 | ["git", "diff", "--name-only"], stderr=subprocess.PIPE, 35 | cwd=MY_DIR, 36 | ).splitlines()) 37 | except (OSError, subprocess.CalledProcessError): 38 | return False 39 | 40 | def get_version_file_path(version_file="version.yaml"): 41 | try: 42 | return os.path.join(subprocess.check_output( 43 | ["git", "rev-parse", "--show-toplevel"], stderr=subprocess.PIPE, 44 | cwd=MY_DIR, 45 | ).decode("utf-8").strip(), version_file) 46 | except (OSError, subprocess.CalledProcessError): 47 | return None 48 | 49 | def number_of_commit_since(version_file="version.yaml"): 50 | """Returns the number of commits since version.yaml was changed.""" 51 | try: 52 | last_commit_to_touch_version_file = subprocess.check_output( 53 | ["git", "log", "--no-merges", "-n", "1", "--pretty=format:%H", 54 | version_file], cwd=MY_DIR, stderr=subprocess.PIPE, 55 | ).strip() 56 | 57 | all_commits = subprocess.check_output( 58 | ["git", "log", "--no-merges", "-n", "1000", "--pretty=format:%H"], 59 | stderr=subprocess.PIPE, cwd=MY_DIR, 60 | ).splitlines() 61 | return all_commits.index(last_commit_to_touch_version_file) 62 | except (OSError, subprocess.CalledProcessError, ValueError): 63 | return None 64 | 65 | 66 | def get_current_git_hash(): 67 | try: 68 | return subprocess.check_output( 69 | ["git", "log", "--no-merges", "-n", "1", "--pretty=format:%H"], 70 | stderr=subprocess.PIPE, cwd=MY_DIR, 71 | ).strip() 72 | except (OSError, subprocess.CalledProcessError): 73 | return None 74 | 75 | def tag_version_data(version_data, version_path="version.yaml"): 76 | current_hash = get_current_git_hash() 77 | # Not in a git repository. 78 | if current_hash is None: 79 | version_data["error"] = "Not in a git repository." 80 | 81 | else: 82 | version_data["revisionid"] = current_hash 83 | version_data["dirty"] = is_tree_dirty() 84 | version_data["dev"] = number_of_commit_since( 85 | get_version_file_path(version_path)) 86 | 87 | # Format the version according to pep440: 88 | pep440 = version_data["version"] 89 | if int(version_data.get("post", 0)) > 0: 90 | pep440 += ".post" + version_data["post"] 91 | 92 | elif int(version_data.get("rc", 0)) > 0: 93 | pep440 += ".rc" + version_data["rc"] 94 | 95 | if version_data.get("dev", 0): 96 | # A Development release comes _before_ the main release. 97 | last = version_data["version"].rsplit(".", 1) 98 | version_data["version"] = "%s.%s" % (last[0], int(last[1]) + 1) 99 | pep440 = version_data["version"] + ".dev" + str(version_data["dev"]) 100 | 101 | version_data["pep440"] = pep440 102 | 103 | return version_data 104 | -------------------------------------------------------------------------------- /pyaff4/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from ._version import get_versions 4 | __version__ = get_versions()['pep440'] 5 | 6 | # Add dummy imports for pyinstaller. These should probably belong in 7 | # future since they are needed for pyinstaller to properly handle 8 | # future.standard_library.install_aliases(). See 9 | # https://github.com/google/rekall/issues/303 10 | if 0: 11 | import UserList 12 | import UserString 13 | import UserDict 14 | import itertools 15 | import collections 16 | import future.backports.misc 17 | import commands 18 | import base64 19 | import __buildin__ 20 | import math 21 | import reprlib 22 | import functools 23 | import re 24 | import subprocess 25 | -------------------------------------------------------------------------------- /pyaff4/_version.py: -------------------------------------------------------------------------------- 1 | 2 | # Machine Generated - do not edit! 3 | 4 | # This file is produced when the main "version.py update" command is run. That 5 | # command copies this file to all sub-packages which contain 6 | # setup.py. Configuration is maintain in version.yaml at the project's top 7 | # level. 8 | 9 | def get_versions(): 10 | return tag_version_data(raw_versions(), """version.yaml""") 11 | 12 | def raw_versions(): 13 | return json.loads(""" 14 | { 15 | "post": "0", 16 | "rc": "0", 17 | "version": "0.32" 18 | } 19 | """) 20 | 21 | import json 22 | import os 23 | import subprocess 24 | 25 | try: 26 | # We are looking for the git repo which contains this file. 27 | MY_DIR = os.path.dirname(os.path.abspath(__file__)) 28 | except: 29 | MY_DIR = None 30 | 31 | def is_tree_dirty(): 32 | try: 33 | return bool(subprocess.check_output( 34 | ["git", "diff", "--name-only"], stderr=subprocess.PIPE, 35 | cwd=MY_DIR, 36 | ).splitlines()) 37 | except (OSError, subprocess.CalledProcessError): 38 | return False 39 | 40 | def get_version_file_path(version_file="version.yaml"): 41 | try: 42 | return os.path.join(subprocess.check_output( 43 | ["git", "rev-parse", "--show-toplevel"], stderr=subprocess.PIPE, 44 | cwd=MY_DIR, 45 | ).decode("utf-8").strip(), version_file) 46 | except (OSError, subprocess.CalledProcessError): 47 | return None 48 | 49 | def number_of_commit_since(version_file="version.yaml"): 50 | """Returns the number of commits since version.yaml was changed.""" 51 | try: 52 | last_commit_to_touch_version_file = subprocess.check_output( 53 | ["git", "log", "--no-merges", "-n", "1", "--pretty=format:%H", 54 | version_file], cwd=MY_DIR, stderr=subprocess.PIPE, 55 | ).strip() 56 | 57 | all_commits = subprocess.check_output( 58 | ["git", "log", "--no-merges", "-n", "1000", "--pretty=format:%H"], 59 | stderr=subprocess.PIPE, cwd=MY_DIR, 60 | ).splitlines() 61 | return all_commits.index(last_commit_to_touch_version_file) 62 | except (OSError, subprocess.CalledProcessError, ValueError): 63 | return None 64 | 65 | 66 | def get_current_git_hash(): 67 | try: 68 | return subprocess.check_output( 69 | ["git", "log", "--no-merges", "-n", "1", "--pretty=format:%H"], 70 | stderr=subprocess.PIPE, cwd=MY_DIR, 71 | ).strip() 72 | except (OSError, subprocess.CalledProcessError): 73 | return None 74 | 75 | def tag_version_data(version_data, version_path="version.yaml"): 76 | current_hash = get_current_git_hash() 77 | # Not in a git repository. 78 | if current_hash is None: 79 | version_data["error"] = "Not in a git repository." 80 | 81 | else: 82 | version_data["revisionid"] = current_hash 83 | version_data["dirty"] = is_tree_dirty() 84 | version_data["dev"] = number_of_commit_since( 85 | get_version_file_path(version_path)) 86 | 87 | # Format the version according to pep440: 88 | pep440 = version_data["version"] 89 | if int(version_data.get("post", 0)) > 0: 90 | pep440 += ".post" + version_data["post"] 91 | 92 | elif int(version_data.get("rc", 0)) > 0: 93 | pep440 += ".rc" + version_data["rc"] 94 | 95 | if version_data.get("dev", 0): 96 | # A Development release comes _before_ the main release. 97 | last = version_data["version"].rsplit(".", 1) 98 | version_data["version"] = "%s.%s" % (last[0], int(last[1]) + 1) 99 | pep440 = version_data["version"] + ".dev" + str(version_data["dev"]) 100 | 101 | version_data["pep440"] = pep440 102 | 103 | return version_data 104 | -------------------------------------------------------------------------------- /pyaff4/aes_keywrap.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Key wrapping and unwrapping as defined in RFC 3394. 3 | Also a padding mechanism that was used in openssl at one time. 4 | The purpose of this algorithm is to encrypt a key multiple times to add an extra layer of security. 5 | ''' 6 | import struct 7 | # TODO: dependency flexibility; make pip install aes_keywrap[cryptography], etc work 8 | from Crypto.Cipher import AES 9 | 10 | QUAD = struct.Struct('>Q') 11 | 12 | def aes_unwrap_key_and_iv(kek, wrapped): 13 | n = len(wrapped)//8 - 1 14 | #NOTE: R[0] is never accessed, left in for consistency with RFC indices 15 | R = [None]+[wrapped[i*8:i*8+8] for i in range(1, n+1)] 16 | A = QUAD.unpack(wrapped[:8])[0] 17 | decrypt = AES.new(kek, AES.MODE_ECB).decrypt 18 | for j in range(5,-1,-1): #counting down 19 | for i in range(n, 0, -1): #(n, n-1, ..., 1) 20 | ciphertext = QUAD.pack(A^(n*j+i)) + R[i] 21 | B = decrypt(ciphertext) 22 | A = QUAD.unpack(B[:8])[0] 23 | R[i] = B[8:] 24 | return b"".join(R[1:]), A 25 | 26 | 27 | def aes_unwrap_key(kek, wrapped, iv=0xa6a6a6a6a6a6a6a6): 28 | ''' 29 | key wrapping as defined in RFC 3394 30 | http://www.ietf.org/rfc/rfc3394.txt 31 | ''' 32 | key, key_iv = aes_unwrap_key_and_iv(kek, wrapped) 33 | if key_iv != iv: 34 | raise ValueError("Integrity Check Failed: "+hex(key_iv)+" (expected "+hex(iv)+")") 35 | return key 36 | 37 | 38 | def aes_unwrap_key_withpad(kek, wrapped): 39 | ''' 40 | alternate initial value for aes key wrapping, as defined in RFC 5649 section 3 41 | http://www.ietf.org/rfc/rfc5649.txt 42 | ''' 43 | if len(wrapped) == 16: 44 | plaintext = AES.new(kek, AES.MODE_ECB).decrypt(wrapped) 45 | key, key_iv = plaintext[:8], plaintext[8:] 46 | else: 47 | key, key_iv = aes_unwrap_key_and_iv(kek, wrapped) 48 | key_iv = "{0:016X}".format(key_iv) 49 | if key_iv[:8] != "A65959A6": 50 | raise ValueError("Integrity Check Failed: "+key_iv[:8]+" (expected A65959A6)") 51 | key_len = int(key_iv[8:], 16) 52 | return key[:key_len] 53 | 54 | def aes_wrap_key(kek, plaintext, iv=0xa6a6a6a6a6a6a6a6): 55 | n = len(plaintext)//8 56 | R = [None]+[plaintext[i*8:i*8+8] for i in range(0, n)] 57 | A = iv 58 | encrypt = AES.new(kek, AES.MODE_ECB).encrypt 59 | for j in range(6): 60 | for i in range(1, n+1): 61 | B = encrypt(QUAD.pack(A) + R[i]) 62 | A = QUAD.unpack(B[:8])[0] ^ (n*j + i) 63 | R[i] = B[8:] 64 | return QUAD.pack(A) + b"".join(R[1:]) 65 | 66 | def aes_wrap_key_withpad(kek, plaintext): 67 | iv = 0xA65959A600000000 + len(plaintext) 68 | plaintext = plaintext + b"\0" * ((8 - len(plaintext)) % 8) 69 | if len(plaintext) == 8: 70 | return AES.new(kek, AES.MODE_ECB).encrypt(QUAD.pack[iv] + plaintext) 71 | return aes_wrap_key(kek, plaintext, iv) 72 | 73 | def test(): 74 | #test vector from RFC 3394 75 | import binascii 76 | KEK = binascii.unhexlify("000102030405060708090A0B0C0D0E0F") 77 | CIPHER = binascii.unhexlify("1FA68B0A8112B447AEF34BD8FB5A7B829D3E862371D2CFE5") 78 | PLAIN = binascii.unhexlify("00112233445566778899AABBCCDDEEFF") 79 | assert aes_unwrap_key(KEK, CIPHER) == PLAIN 80 | assert aes_wrap_key(KEK, PLAIN) == CIPHER 81 | -------------------------------------------------------------------------------- /pyaff4/aff4_directory.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | """This module implements the Directory AFF4 Volume.""" 16 | from __future__ import unicode_literals 17 | import logging 18 | import os 19 | 20 | from pyaff4 import aff4 21 | from pyaff4 import aff4_utils 22 | from pyaff4 import lexicon 23 | from pyaff4 import rdfvalue 24 | from pyaff4 import registry 25 | from pyaff4 import utils 26 | from pyaff4 import escaping 27 | 28 | LOGGER = logging.getLogger("pyaff4") 29 | 30 | 31 | class AFF4Directory(aff4.AFF4Volume): 32 | 33 | root_path = "" 34 | 35 | @classmethod 36 | def NewAFF4Directory(cls, resolver, version, root_urn): 37 | result = AFF4Directory(resolver) 38 | result.version = version 39 | result.root_path = root_urn.ToFilename() 40 | 41 | mode = resolver.GetUnique(lexicon.transient_graph, root_urn, lexicon.AFF4_STREAM_WRITE_MODE) 42 | if mode == "truncate": 43 | aff4_utils.RemoveDirectory(result.root_path) 44 | 45 | if not (os.path.isdir(result.root_path) or 46 | os.path.isfile(result.root_path)): 47 | if mode == "truncate" or mode == "append": 48 | aff4_utils.MkDir(result.root_path) 49 | else: 50 | raise RuntimeError("Unknown mode") 51 | 52 | resolver.Set(lexicon.transient_graph, result.urn, lexicon.AFF4_TYPE, 53 | rdfvalue.URN(lexicon.AFF4_DIRECTORY_TYPE)) 54 | 55 | resolver.Set(lexicon.transient_graph, result.urn, lexicon.AFF4_STORED, 56 | rdfvalue.URN(root_urn)) 57 | 58 | result.LoadFromURN() 59 | 60 | return resolver.CachePut(result) 61 | 62 | def __init__(self, *args, **kwargs): 63 | super(AFF4Directory, self).__init__(*args, **kwargs) 64 | self.children = set() 65 | 66 | def ContainsSegment(self, child_arn): 67 | localpath = escaping.member_name_for_file_iri(child_arn.SerializeToString()) 68 | if os.path.exists(localpath): 69 | return True 70 | return False 71 | 72 | def CreateMember(self, child_urn): 73 | # Check that child is a relative path in our URN. 74 | relative_path = self.urn.RelativePath(child_urn) 75 | if relative_path == child_urn.SerializeToString(): 76 | raise IOError("Child URN is not within container URN.") 77 | 78 | # Use this filename. Note that since filesystems can not typically 79 | # represent files and directories as the same path component we can not 80 | # allow slashes in the filename. Otherwise we will fail to create 81 | # e.g. stream/0000000 and stream/0000000/index. 82 | filename = escaping.member_name_for_urn( 83 | child_urn, self.version, self.urn) 84 | 85 | # We are allowed to create any files inside the directory volume. 86 | self.resolver.Set(lexicon.transient_graph, child_urn, lexicon.AFF4_TYPE, 87 | rdfvalue.URN(lexicon.AFF4_FILE_TYPE)) 88 | self.resolver.Set(lexicon.transient_graph, child_urn, lexicon.AFF4_STREAM_WRITE_MODE, 89 | rdfvalue.XSDString("truncate")) 90 | self.resolver.Set(lexicon.transient_graph, child_urn, lexicon.AFF4_DIRECTORY_CHILD_FILENAME, 91 | rdfvalue.XSDString(filename)) 92 | 93 | # Store the member inside our storage location. 94 | self.resolver.Set(lexicon.transient_graph, 95 | child_urn, lexicon.AFF4_FILE_NAME, 96 | rdfvalue.XSDString(self.root_path + os.sep + filename)) 97 | 98 | result = self.resolver.AFF4FactoryOpen(child_urn) 99 | self.MarkDirty() 100 | self.children.add(child_urn) 101 | 102 | return result 103 | 104 | def LoadFromURN(self): 105 | self.storage = self.resolver.GetUnique(lexicon.transient_graph, self.urn, lexicon.AFF4_STORED) 106 | if not self.storage: 107 | LOGGER.error("Unable to find storage for AFF4Directory %s", 108 | self.urn) 109 | raise IOError("NOT_FOUND") 110 | 111 | # The actual filename for the root directory. 112 | self.root_path = self.storage.ToFilename() 113 | 114 | try: 115 | # We need to get the URN of the container before we can process 116 | # anything. 117 | 118 | containerDescriptionSegment = self.storage.Append(lexicon.AFF4_CONTAINER_DESCRIPTION) 119 | if self.ContainsSegment(containerDescriptionSegment): 120 | with self.resolver.AFF4FactoryOpen(containerDescriptionSegment) as desc: 121 | if desc: 122 | urn_string = utils.SmartUnicode(desc.Read(1000)) 123 | 124 | if (urn_string and 125 | self.urn.SerializeToString() != urn_string): 126 | self.resolver.DeleteSubject(self.urn) 127 | self.urn.Set(urn_string) 128 | 129 | # Set these triples with the new URN so we know how to open 130 | # it. 131 | self.resolver.Set(lexicon.transient_graph, self.urn, lexicon.AFF4_TYPE, 132 | rdfvalue.URN(lexicon.AFF4_DIRECTORY_TYPE)) 133 | 134 | self.resolver.Set(lexicon.transient_graph, self.urn, lexicon.AFF4_STORED, 135 | rdfvalue.URN(self.storage)) 136 | 137 | LOGGER.info("AFF4Directory volume found: %s", self.urn) 138 | 139 | # Try to load the RDF metadata file from the storage. 140 | with self.resolver.AFF4FactoryOpen( 141 | self.storage.Append( 142 | lexicon.AFF4_CONTAINER_INFO_TURTLE)) as turtle_stream: 143 | if turtle_stream: 144 | self.resolver.LoadFromTurtle(turtle_stream) 145 | 146 | # Find all the contained objects and adjust their filenames. 147 | for subject in self.resolver.SelectSubjectsByPrefix( 148 | utils.SmartUnicode(self.urn)): 149 | 150 | child_filename = self.resolver.Get( 151 | subject, lexicon.AFF4_DIRECTORY_CHILD_FILENAME) 152 | if child_filename: 153 | self.resolver.Set(lexicon.transient_graph, 154 | subject, lexicon.AFF4_FILE_NAME, 155 | rdfvalue.XSDString("%s%s%s" % ( 156 | self.root_path, os.sep, child_filename))) 157 | 158 | except IOError: 159 | pass 160 | 161 | 162 | def Flush(self): 163 | if self.IsDirty(): 164 | # Flush all children before us. This ensures that metadata is fully 165 | # generated for each child. 166 | for child_urn in list(self.children): 167 | obj = self.resolver.CacheGet(child_urn) 168 | if obj: 169 | obj.Flush() 170 | 171 | # Mark the container with its URN 172 | with self.CreateMember( 173 | self.urn.Append( 174 | lexicon.AFF4_CONTAINER_DESCRIPTION)) as desc: 175 | desc.Truncate() 176 | desc.Write(self.urn.SerializeToString()) 177 | desc.Flush() # Flush explicitly since we already flushed above. 178 | 179 | # Dump the resolver into the zip file. 180 | with self.CreateMember( 181 | self.urn.Append( 182 | lexicon.AFF4_CONTAINER_INFO_TURTLE)) as turtle_stream: 183 | # Overwrite the old turtle file with the newer data. 184 | turtle_stream.Truncate() 185 | self.resolver.DumpToTurtle(turtle_stream, verbose=False) 186 | turtle_stream.Flush() 187 | 188 | return super(AFF4Directory, self).Flush() 189 | 190 | 191 | registry.AFF4_TYPE_MAP[lexicon.AFF4_DIRECTORY_TYPE] = AFF4Directory 192 | -------------------------------------------------------------------------------- /pyaff4/aff4_directory_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | # Copyright 2015 Google Inc. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | import tempfile 16 | import unittest 17 | 18 | from pyaff4 import aff4_directory 19 | from pyaff4 import aff4_utils 20 | from pyaff4 import data_store 21 | from pyaff4 import lexicon 22 | from pyaff4 import rdfvalue 23 | from pyaff4 import container 24 | from pyaff4 import plugins 25 | 26 | from nose.tools import nottest 27 | 28 | class AFF4DirectoryTest(unittest.TestCase): 29 | root_path = tempfile.gettempdir() + "/aff4_directory/" 30 | segment_name = "Foobar.txt" 31 | 32 | def tearDown(self): 33 | aff4_utils.RemoveDirectory(self.root_path) 34 | 35 | def setUp(self): 36 | version = container.Version(1, 1, "pyaff4") 37 | with data_store.MemoryDataStore() as resolver: 38 | root_urn = rdfvalue.URN.NewURNFromFilename(self.root_path) 39 | 40 | resolver.Set(lexicon.transient_graph, root_urn, lexicon.AFF4_STREAM_WRITE_MODE, 41 | rdfvalue.XSDString("truncate")) 42 | 43 | with aff4_directory.AFF4Directory.NewAFF4Directory( 44 | resolver, version, root_urn) as volume: 45 | 46 | segment_urn = volume.urn.Append(self.segment_name) 47 | with volume.CreateMember(segment_urn) as member: 48 | member.Write(b"Hello world") 49 | resolver.Set(lexicon.transient_graph, 50 | member.urn, lexicon.AFF4_STREAM_ORIGINAL_FILENAME, 51 | rdfvalue.XSDString(self.root_path + self.segment_name)) 52 | 53 | @nottest 54 | def testCreateMember(self): 55 | version = container.Version(1, 1, "pyaff4") 56 | with data_store.MemoryDataStore() as resolver: 57 | root_urn = rdfvalue.URN.NewURNFromFilename(self.root_path) 58 | with aff4_directory.AFF4Directory.NewAFF4Directory( 59 | resolver, version, root_urn) as directory: 60 | 61 | # Check for member. 62 | child_urn = directory.urn.Append(self.segment_name) 63 | with resolver.AFF4FactoryOpen(child_urn) as child: 64 | self.assertEquals(child.Read(10000), b"Hello world") 65 | 66 | # Check that the metadata is carried over. 67 | filename = resolver.Get( 68 | child_urn, lexicon.AFF4_STREAM_ORIGINAL_FILENAME) 69 | 70 | self.assertEquals(filename, self.root_path + self.segment_name) 71 | 72 | if __name__ == '__main__': 73 | #logging.getLogger().setLevel(logging.DEBUG) 74 | unittest.main() 75 | -------------------------------------------------------------------------------- /pyaff4/aff4_file.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | """An implementation of AFF4 file backed objects.""" 16 | from __future__ import unicode_literals 17 | from future import standard_library 18 | standard_library.install_aliases() 19 | from builtins import str 20 | 21 | import logging 22 | import os 23 | import io 24 | 25 | from pyaff4 import aff4 26 | from pyaff4 import aff4_utils 27 | from pyaff4 import lexicon 28 | from pyaff4 import rdfvalue 29 | from pyaff4 import registry 30 | from pyaff4 import utils 31 | 32 | BUFF_SIZE = 64 * 1024 33 | 34 | 35 | LOGGER = logging.getLogger("pyaff4") 36 | 37 | 38 | class FileBackedObject(aff4.AFF4Stream): 39 | def __init__(self, *args, **kwargs): 40 | super(FileBackedObject, self).__init__( *args, **kwargs) 41 | 42 | def _GetFilename(self): 43 | filename = self.resolver.GetUnique(lexicon.transient_graph, self.urn, lexicon.AFF4_FILE_NAME) 44 | if filename: 45 | return filename 46 | 47 | # Only file:// URNs are supported. 48 | if self.urn.Scheme() == "file": 49 | return self.urn.ToFilename() 50 | 51 | @staticmethod 52 | def _CreateIntermediateDirectories(components): 53 | """Recursively create intermediate directories.""" 54 | path = os.sep 55 | 56 | if aff4.WIN32: 57 | # On windows we do not want a leading \ (e.g. C:\windows not 58 | # \C:\Windows) 59 | path = "" 60 | 61 | for component in components: 62 | path = path + component + os.sep 63 | if LOGGER.isEnabledFor(logging.INFO): 64 | LOGGER.info("Creating intermediate directories %s", path) 65 | 66 | if os.isdir(path): 67 | continue 68 | 69 | # Directory does not exist - Try to make it. 70 | try: 71 | aff4_utils.MkDir(path) 72 | continue 73 | except IOError as e: 74 | LOGGER.error( 75 | "Unable to create intermediate directory: %s", e) 76 | raise 77 | 78 | def LoadFromURN(self): 79 | flags = "rb" 80 | 81 | filename = self._GetFilename() 82 | if not filename: 83 | raise IOError("Unable to find storage for %s" % self.urn) 84 | 85 | filename = str(filename) 86 | 87 | directory_components = os.sep.split(filename) 88 | directory_components.pop(-1) 89 | 90 | mode = self.resolver.GetUnique(lexicon.transient_graph, self.urn, lexicon.AFF4_STREAM_WRITE_MODE) 91 | if mode == "truncate": 92 | flags = "w+b" 93 | #self.resolver.Set(lexicon.transient_graph, self.urn, lexicon.AFF4_STREAM_WRITE_MODE, 94 | # rdfvalue.XSDString("append")) 95 | self.properties.writable = True 96 | self._CreateIntermediateDirectories(directory_components) 97 | 98 | elif mode == "append": 99 | flags = "a+b" 100 | self.properties.writable = True 101 | self._CreateIntermediateDirectories(directory_components) 102 | 103 | elif mode == "random": 104 | flags = "r+b" 105 | self.properties.writable = True 106 | self._CreateIntermediateDirectories(directory_components) 107 | 108 | if LOGGER.isEnabledFor(logging.INFO): 109 | LOGGER.info("Opening file %s", filename) 110 | self.fd = open(filename, flags) 111 | try: 112 | self.fd.seek(0, 2) 113 | self.size = self.fd.tell() 114 | except IOError: 115 | self.properties.sizeable = False 116 | self.properties.seekable = False 117 | 118 | def Read(self, length): 119 | if self.fd.tell() != self.readptr: 120 | self.fd.seek(self.readptr) 121 | 122 | result = self.fd.read(length) 123 | self.readptr += len(result) 124 | return result 125 | 126 | def ReadAll(self): 127 | res = b"" 128 | while True: 129 | toRead = 32 * 1024 130 | data = self.Read(toRead) 131 | if data == None or len(data) == 0: 132 | # EOF 133 | return res 134 | else: 135 | res += data 136 | 137 | 138 | def WriteStream(self, stream, progress=None): 139 | """Copy the stream into this stream.""" 140 | while True: 141 | data = stream.read(BUFF_SIZE) 142 | if not data: 143 | break 144 | 145 | self.Write(data) 146 | progress.Report(self.readptr) 147 | 148 | def Write(self, data): 149 | if LOGGER.isEnabledFor(logging.INFO): 150 | LOGGER.info("ZipFileSegment.Write %s @ %x[%x]", self.urn, self.writeptr, len(data)) 151 | if not self.properties.writable: 152 | raise IOError("Attempt to write to read only object") 153 | self.MarkDirty() 154 | 155 | # On OSX, the following test doesn't work 156 | # so we need to do the seek every time 157 | if aff4.MacOS: 158 | self.fd.seek(self.writeptr) 159 | else: 160 | if self.fd.tell() != self.writeptr: 161 | self.fd.seek(self.writeptr) 162 | 163 | self.fd.write(utils.SmartStr(data)) 164 | # self.fd.flush() 165 | 166 | #self.size = len(data) 167 | #self.size = len(data) 168 | self.writeptr += len(data) 169 | if self.writeptr > self.size: 170 | self.size = self.writeptr 171 | 172 | def Flush(self): 173 | if self.IsDirty(): 174 | self.fd.flush() 175 | super(FileBackedObject, self).Flush() 176 | 177 | def Prepare(self): 178 | self.readptr = 0 179 | 180 | def Truncate(self): 181 | self.fd.truncate(0) 182 | 183 | def Trim(self, offset): 184 | self.fd.truncate(offset) 185 | self.seek(0, offset) 186 | 187 | def Size(self): 188 | self.fd.seek(0, 2) 189 | return self.fd.tell() 190 | 191 | def Close(self): 192 | self.resolver.flush_callbacks["FileBacking"] = self.CloseFile 193 | #self.fd.close() 194 | 195 | def CloseFile(self): 196 | self.fd.close() 197 | 198 | def GenericFileHandler(resolver, urn, *args, **kwargs): 199 | if os.path.isdir(urn.ToFilename()): 200 | directory_handler = registry.AFF4_TYPE_MAP[lexicon.AFF4_DIRECTORY_TYPE] 201 | result = directory_handler(resolver) 202 | resolver.Set(result.urn, lexicon.AFF4_STORED, urn) 203 | 204 | return result 205 | 206 | return FileBackedObject(resolver, urn) 207 | 208 | registry.AFF4_TYPE_MAP["file"] = GenericFileHandler 209 | registry.AFF4_TYPE_MAP[lexicon.AFF4_FILE_TYPE] = FileBackedObject 210 | 211 | 212 | class AFF4MemoryStream(FileBackedObject): 213 | 214 | def __init__(self, *args, **kwargs): 215 | super(AFF4MemoryStream, self).__init__(*args, **kwargs) 216 | self.fd = io.BytesIO() 217 | self.properties.writable = True 218 | -------------------------------------------------------------------------------- /pyaff4/aff4_image_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | # Copyright 2014 Google Inc. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | import tempfile 16 | 17 | from future import standard_library 18 | standard_library.install_aliases() 19 | from builtins import range 20 | import os 21 | import io 22 | import unittest 23 | 24 | from pyaff4 import aff4_image 25 | from pyaff4 import data_store 26 | from pyaff4 import lexicon 27 | from pyaff4 import rdfvalue 28 | from pyaff4 import zip 29 | from pyaff4 import container 30 | from pyaff4 import plugins 31 | 32 | 33 | class AFF4ImageTest(unittest.TestCase): 34 | filename = tempfile.gettempdir() + "/aff4_image_test.zip" 35 | filename_urn = rdfvalue.URN.FromFileName(filename) 36 | image_name = "image.dd" 37 | 38 | def setUp(self): 39 | try: 40 | os.unlink(self.filename) 41 | except (IOError, OSError): 42 | pass 43 | 44 | def tearDown(self): 45 | try: 46 | os.unlink(self.filename) 47 | except (IOError, OSError): 48 | pass 49 | 50 | def setUp(self): 51 | version = container.Version(0, 1, "pyaff4") 52 | with data_store.MemoryDataStore() as resolver: 53 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 54 | rdfvalue.XSDString("truncate")) 55 | 56 | with zip.ZipFile.NewZipFile(resolver, version, self.filename_urn) as zip_file: 57 | self.volume_urn = zip_file.urn 58 | image_urn = self.volume_urn.Append(self.image_name) 59 | 60 | # Use default compression. 61 | with aff4_image.AFF4Image.NewAFF4Image( 62 | resolver, image_urn, self.volume_urn) as image: 63 | image.chunk_size = 10 64 | image.chunks_per_segment = 3 65 | 66 | for i in range(100): 67 | image.Write(b"Hello world %02d!" % i) 68 | 69 | self.image_urn = image.urn 70 | 71 | # Write a snappy compressed image. 72 | self.image_urn_2 = self.image_urn.Append("2") 73 | with aff4_image.AFF4Image.NewAFF4Image( 74 | resolver, self.image_urn_2, self.volume_urn) as image_2: 75 | image_2.setCompressionMethod(lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY) 76 | image_2.Write(b"This is a test") 77 | 78 | # Use streaming API to write image. 79 | self.image_urn_3 = self.image_urn.Append("3") 80 | with aff4_image.AFF4Image.NewAFF4Image( 81 | resolver, self.image_urn_3, self.volume_urn) as image: 82 | image.chunk_size = 10 83 | image.chunks_per_segment = 3 84 | stream = io.BytesIO() 85 | for i in range(100): 86 | stream.write(b"Hello world %02d!" % i) 87 | 88 | stream.seek(0) 89 | image.WriteStream(stream) 90 | 91 | def testOpenImageByURN(self): 92 | resolver = data_store.MemoryDataStore() 93 | version = container.Version(1, 1, "pyaff4") 94 | # This is required in order to load and parse metadata from this volume 95 | # into a fresh empty resolver. 96 | with zip.ZipFile.NewZipFile(resolver, version, self.filename_urn) as zip_file: 97 | image_urn = zip_file.urn.Append(self.image_name) 98 | 99 | with resolver.AFF4FactoryOpen(image_urn) as image: 100 | self.assertEquals(image.chunk_size, 10) 101 | self.assertEquals(image.chunks_per_segment, 3) 102 | self.assertEquals( 103 | b"Hello world 00!Hello world 01!Hello world 02!Hello world 03!" + 104 | b"Hello world 04!Hello world 05!Hello worl", 105 | image.Read(100)) 106 | 107 | self.assertEquals(1500, image.Size()) 108 | 109 | # Now test snappy decompression. 110 | with resolver.AFF4FactoryOpen(self.image_urn_2) as image_2: 111 | self.assertEquals( 112 | resolver.GetUnique(zip_file.urn, image_2.urn, lexicon.AFF4_IMAGE_COMPRESSION), 113 | lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY) 114 | 115 | data = image_2.Read(100) 116 | self.assertEquals(data, b"This is a test") 117 | 118 | # Now test streaming API image. 119 | with resolver.AFF4FactoryOpen(self.image_urn_3) as image_3: 120 | self.assertEquals(image_3.chunk_size, 10) 121 | self.assertEquals(image_3.chunks_per_segment, 3) 122 | self.assertEquals( 123 | b"Hello world 00!Hello world 01!Hello world 02!Hello world 03!"+ 124 | b"Hello world 04!Hello world 05!Hello worl", 125 | image_3.Read(100)) 126 | 127 | 128 | if __name__ == '__main__': 129 | #logging.getLogger().setLevel(logging.DEBUG) 130 | unittest.main() 131 | -------------------------------------------------------------------------------- /pyaff4/aff4_image_test2.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | # Copyright 2014 Google Inc. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | import tempfile 16 | 17 | from future import standard_library 18 | standard_library.install_aliases() 19 | from builtins import range 20 | import os 21 | import io 22 | import unittest 23 | 24 | from pyaff4 import aff4_image 25 | from pyaff4 import data_store 26 | from pyaff4 import lexicon 27 | from pyaff4 import rdfvalue 28 | from pyaff4 import zip 29 | from pyaff4 import container 30 | from pyaff4 import plugins 31 | 32 | 33 | class AFF4ImageTest(unittest.TestCase): 34 | filename = tempfile.gettempdir() + "/aff4_image_test2.zip" 35 | filename_urn = rdfvalue.URN.FromFileName(filename) 36 | image_name = "image.dd" 37 | 38 | def setUp(self): 39 | try: 40 | os.unlink(self.filename) 41 | except (IOError, OSError): 42 | pass 43 | 44 | def tearDown(self): 45 | try: 46 | os.unlink(self.filename) 47 | except (IOError, OSError): 48 | pass 49 | 50 | 51 | 52 | #@unittest.skip 53 | def testLargerThanBevyWrite(self): 54 | version = container.Version(0, 1, "pyaff4") 55 | 56 | with data_store.MemoryDataStore() as resolver: 57 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 58 | rdfvalue.XSDString("truncate")) 59 | 60 | with zip.ZipFile.NewZipFile(resolver, version, self.filename_urn) as zip_file: 61 | self.volume_urn = zip_file.urn 62 | self.image_urn = self.volume_urn.Append(self.image_name) 63 | 64 | self.image_urn_2 = self.image_urn.Append("2") 65 | with aff4_image.AFF4Image.NewAFF4Image(resolver, self.image_urn_2, self.volume_urn) as image: 66 | image.chunk_size = 5 67 | image.chunks_per_segment = 2 68 | image.Write(b"abcdeabcdea") 69 | self.assertEquals(b"abcde", image.Read(5)) 70 | 71 | with data_store.MemoryDataStore() as resolver: 72 | with zip.ZipFile.NewZipFile(resolver, version, self.filename_urn) as zip_file: 73 | image_urn = zip_file.urn.Append(self.image_name) 74 | 75 | self.image_urn_2 = self.image_urn.Append("2") 76 | with resolver.AFF4FactoryOpen(self.image_urn_2) as image: 77 | self.assertEquals(11, image.Size()) 78 | self.assertEqual(b"abcdeabcdea", image.ReadAll()) 79 | 80 | if __name__ == '__main__': 81 | #logging.getLogger().setLevel(logging.DEBUG) 82 | unittest.main() 83 | -------------------------------------------------------------------------------- /pyaff4/aff4_imager_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | """Utilities for AFF4 imaging. 16 | 17 | These are mostly high level utilities used by the command line imager. 18 | """ 19 | from __future__ import unicode_literals 20 | -------------------------------------------------------------------------------- /pyaff4/aff4_map_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | # Copyright 2014 Google Inc. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | 16 | import os 17 | import tempfile 18 | import unittest 19 | 20 | from pyaff4 import aff4_file 21 | from pyaff4 import aff4_map 22 | from pyaff4 import data_store 23 | from pyaff4 import lexicon 24 | from pyaff4 import rdfvalue 25 | from pyaff4 import zip 26 | from pyaff4 import container 27 | 28 | class AFF4MapTest(unittest.TestCase): 29 | filename = tempfile.gettempdir() + u"/aff4_map_test.zip" 30 | filename_urn = rdfvalue.URN.FromFileName(filename) 31 | image_name = u"image.dd" 32 | 33 | def setUp(self): 34 | try: 35 | os.unlink(self.filename) 36 | except (IOError, OSError): 37 | pass 38 | 39 | def tearDown(self): 40 | try: 41 | os.unlink(self.filename) 42 | except (IOError, OSError): 43 | pass 44 | 45 | def setUp(self): 46 | version = container.Version(1, 1, "pyaff4") 47 | with data_store.MemoryDataStore() as resolver: 48 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 49 | rdfvalue.XSDString("truncate")) 50 | 51 | with zip.ZipFile.NewZipFile(resolver, version, self.filename_urn) as zip_file: 52 | self.volume_urn = zip_file.urn 53 | self.image_urn = self.volume_urn.Append(self.image_name) 54 | 55 | # Write Map image sequentially (Seek/Write method). 56 | with aff4_map.AFF4Map.NewAFF4Map( 57 | resolver, self.image_urn, self.volume_urn) as image: 58 | # Maps are written in random order. 59 | image.SeekWrite(50) 60 | image.Write(b"XX - This is the position.") 61 | 62 | image.SeekWrite(0) 63 | image.Write(b"00 - This is the position.") 64 | 65 | # We can "overwrite" data by writing the same range again. 66 | image.SeekWrite(50) 67 | image.Write(b"50") 68 | 69 | # Test the Stream method. 70 | with resolver.CachePut( 71 | aff4_file.AFF4MemoryStream(resolver)) as source: 72 | # Fill it with data. 73 | source.Write(b"AAAABBBBCCCCDDDDEEEEFFFFGGGGHHHH") 74 | 75 | # Make a temporary map that defines our plan. 76 | helper_map = aff4_map.AFF4Map(resolver) 77 | 78 | helper_map.AddRange(4, 0, 4, source.urn) # 0000AAAA 79 | helper_map.AddRange(0, 12, 4, source.urn) # DDDDAAAA 80 | helper_map.AddRange(12, 16, 4, source.urn)# DDDDAAAA0000EEEE 81 | 82 | image_urn_2 = self.volume_urn.Append( 83 | self.image_name).Append("streamed") 84 | 85 | with aff4_map.AFF4Map.NewAFF4Map( 86 | resolver, image_urn_2, self.volume_urn) as image: 87 | 88 | # Now we create the real map by copying the temporary 89 | # map stream. 90 | image.WriteStream(helper_map) 91 | 92 | def testAddRange(self): 93 | resolver = data_store.MemoryDataStore() 94 | version = container.Version(1, 1, "pyaff4") 95 | # This is required in order to load and parse metadata from this volume 96 | # into a fresh empty resolver. 97 | with zip.ZipFile.NewZipFile(resolver, version, self.filename_urn) as zip_file: 98 | image_urn = zip_file.urn.Append(self.image_name) 99 | 100 | with resolver.AFF4FactoryOpen(image_urn) as map: 101 | a = rdfvalue.URN("aff4://a") 102 | b = rdfvalue.URN("aff4://b") 103 | 104 | # First test - overlapping regions: 105 | map.AddRange(0, 0, 100, a) 106 | map.AddRange(10, 10, 100, a) 107 | 108 | # Should be merged into a single range. 109 | ranges = map.GetRanges() 110 | self.assertEquals(len(ranges), 1) 111 | self.assertEquals(ranges[0].length, 110) 112 | 113 | map.Clear() 114 | 115 | # Repeating regions - should not be merged but first region should 116 | # be truncated. 117 | map.AddRange(0, 0, 100, a) 118 | map.AddRange(50, 0, 100, a) 119 | 120 | ranges = map.GetRanges() 121 | self.assertEquals(len(ranges), 2) 122 | self.assertEquals(ranges[0].length, 50) 123 | 124 | # Inserted region. Should split existing region into three. 125 | map.Clear() 126 | 127 | map.AddRange(0, 0, 100, a) 128 | map.AddRange(50, 0, 10, b) 129 | 130 | ranges = map.GetRanges() 131 | self.assertEquals(len(ranges), 3) 132 | self.assertEquals(ranges[0].length, 50) 133 | self.assertEquals(ranges[0].target_id, 0) 134 | 135 | self.assertEquals(ranges[1].length, 10) 136 | self.assertEquals(ranges[1].target_id, 1) 137 | 138 | self.assertEquals(ranges[2].length, 40) 139 | self.assertEquals(ranges[2].target_id, 0) 140 | 141 | # New range overwrites all the old ranges. 142 | map.AddRange(0, 0, 100, b) 143 | 144 | ranges = map.GetRanges() 145 | self.assertEquals(len(ranges), 1) 146 | self.assertEquals(ranges[0].length, 100) 147 | self.assertEquals(ranges[0].target_id, 1) 148 | 149 | 150 | # Simulate writing contiguous regions. These should be merged into a 151 | # single region automatically. 152 | map.Clear() 153 | 154 | map.AddRange(0, 100, 10, a) 155 | map.AddRange(10, 110, 10, a) 156 | map.AddRange(20, 120, 10, a) 157 | map.AddRange(30, 130, 10, a) 158 | 159 | ranges = map.GetRanges() 160 | self.assertEquals(len(ranges), 1) 161 | self.assertEquals(ranges[0].length, 40) 162 | self.assertEquals(ranges[0].target_id, 0) 163 | 164 | # Writing sparse image. 165 | map.Clear() 166 | 167 | map.AddRange(0, 100, 10, a) 168 | map.AddRange(30, 130, 10, a) 169 | 170 | ranges = map.GetRanges() 171 | self.assertEquals(len(ranges), 2) 172 | self.assertEquals(ranges[0].length, 10) 173 | self.assertEquals(ranges[0].target_id, 0) 174 | self.assertEquals(ranges[1].length, 10) 175 | self.assertEquals(ranges[1].map_offset, 30) 176 | self.assertEquals(ranges[1].target_id, 0) 177 | 178 | # Now merge. Adding the missing region makes the image not sparse. 179 | map.AddRange(10, 110, 20, a) 180 | ranges = map.GetRanges() 181 | self.assertEquals(len(ranges), 1) 182 | self.assertEquals(ranges[0].length, 40) 183 | 184 | def testCreateMapStream(self): 185 | resolver = data_store.MemoryDataStore() 186 | version = container.Version(1, 1, "pyaff4") 187 | # This is required in order to load and parse metadata from this volume 188 | # into a fresh empty resolver. 189 | with zip.ZipFile.NewZipFile(resolver, version, self.filename_urn) as zip_file: 190 | image_urn = zip_file.urn.Append(self.image_name) 191 | image_urn_2 = image_urn.Append("streamed") 192 | 193 | # Check the first stream. 194 | self.CheckImageURN(resolver, image_urn) 195 | 196 | # The second stream must be the same. 197 | self.CheckStremImageURN(resolver, image_urn_2) 198 | 199 | def CheckStremImageURN(self, resolver, image_urn_2): 200 | with resolver.AFF4FactoryOpen(image_urn_2) as map: 201 | self.assertEquals(map.Size(), 16) 202 | self.assertEquals(map.Read(100), b"DDDDAAAA\x00\x00\x00\x00EEEE") 203 | 204 | # The data stream should be packed without gaps. 205 | with resolver.AFF4FactoryOpen(image_urn_2.Append("data")) as image: 206 | self.assertEquals(image.Read(100), b"DDDDAAAAEEEE") 207 | 208 | def CheckImageURN(self, resolver, image_urn): 209 | with resolver.AFF4FactoryOpen(image_urn) as map: 210 | map.SeekRead(50) 211 | self.assertEquals(map.Read(2), b"50") 212 | 213 | map.SeekRead(0) 214 | self.assertEquals(map.Read(2), b"00") 215 | 216 | ranges = map.GetRanges() 217 | self.assertEquals(len(ranges), 3) 218 | self.assertEquals(ranges[0].length, 26) 219 | self.assertEquals(ranges[0].map_offset, 0) 220 | self.assertEquals(ranges[0].target_offset, 26) 221 | 222 | # This is the extra "overwritten" 2 bytes which were appended to the 223 | # end of the target stream and occupy the map range from 50-52. 224 | self.assertEquals(ranges[1].length, 2) 225 | self.assertEquals(ranges[1].map_offset, 50) 226 | self.assertEquals(ranges[1].target_offset, 52) 227 | 228 | self.assertEquals(ranges[2].length, 24) 229 | self.assertEquals(ranges[2].map_offset, 52) 230 | self.assertEquals(ranges[2].target_offset, 2) 231 | 232 | # Test that reads outside the ranges null pad correctly. 233 | map.SeekRead(48) 234 | read_string = map.Read(4) 235 | self.assertEquals(read_string, b"\x00\x0050") 236 | 237 | 238 | if __name__ == '__main__': 239 | #logging.getLogger().setLevel(logging.DEBUG) 240 | unittest.main() 241 | -------------------------------------------------------------------------------- /pyaff4/aff4_metadata.py: -------------------------------------------------------------------------------- 1 | class RDFObject(object): 2 | def __init__(self, URN, resolver, lexicon): 3 | self.resolver = resolver 4 | self.urn = URN 5 | self.lexicon = lexicon 6 | 7 | def __getattr__(self, item): 8 | #print self.resolver.DumpToTurtle() 9 | val = self.resolver.Get(self.urn, self.lexicon.of(item)) 10 | return val -------------------------------------------------------------------------------- /pyaff4/aff4_utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | from future import standard_library 3 | standard_library.install_aliases() 4 | from builtins import chr 5 | import os 6 | import re 7 | import shutil 8 | import string 9 | import urllib.parse 10 | 11 | from pyaff4 import rdfvalue 12 | from pyaff4 import utils 13 | 14 | 15 | def MkDir(path): 16 | try: 17 | os.mkdir(path) 18 | except OSError as e: 19 | if "File exists" in e.strerror: 20 | return 21 | 22 | raise 23 | 24 | def RemoveDirectory(path): 25 | try: 26 | shutil.rmtree(path) 27 | except OSError: 28 | pass 29 | 30 | def EnsureDirectoryExists(path): 31 | dirname = os.path.dirname(path) 32 | try: 33 | os.makedirs(dirname) 34 | except OSError: 35 | pass 36 | -------------------------------------------------------------------------------- /pyaff4/container_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import unicode_literals 3 | import os 4 | import unittest 5 | 6 | from pyaff4 import container 7 | from pyaff4 import hashing_test 8 | 9 | 10 | class ContainerTest(unittest.TestCase): 11 | 12 | @hashing_test.conditional_on_images 13 | def testOpen(self): 14 | fd = container.Container.open(hashing_test.stdLinear) 15 | self.assertEqual(fd.urn, 16 | u"aff4://fcbfdce7-4488-4677-abf6-08bc931e195b") 17 | 18 | 19 | if __name__ == '__main__': 20 | unittest.main() 21 | -------------------------------------------------------------------------------- /pyaff4/data_store_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | # Copyright 2015 Google Inc. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | 16 | 17 | from future import standard_library 18 | standard_library.install_aliases() 19 | from pyaff4 import aff4 20 | from pyaff4 import data_store 21 | from pyaff4 import lexicon 22 | from pyaff4 import rdfvalue 23 | from pyaff4 import streams 24 | import unittest 25 | 26 | import io 27 | 28 | 29 | class DataStoreTest(unittest.TestCase): 30 | def setUp(self): 31 | self.hello_urn = rdfvalue.URN("aff4://hello") 32 | self.store = data_store.MemoryDataStore() 33 | self.store.Set(None, 34 | self.hello_urn, rdfvalue.URN(lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY), 35 | rdfvalue.XSDString("foo")) 36 | 37 | self.store.Set(None, 38 | self.hello_urn, rdfvalue.URN(lexicon.AFF4_TYPE), 39 | rdfvalue.XSDString("bar")) 40 | 41 | def testDataStore(self): 42 | result = self.store.GetUnique(None,self.hello_urn, rdfvalue.URN( 43 | lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY)) 44 | self.assertEquals(type(result), rdfvalue.XSDString) 45 | 46 | self.assertEquals(result.SerializeToString(), b"foo") 47 | 48 | self.store.Set(None, 49 | self.hello_urn, rdfvalue.URN(lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY), 50 | rdfvalue.XSDString("bar")) 51 | 52 | # In the current implementation a second Set() overwrites the previous 53 | # value. 54 | self.assertEquals( 55 | self.store.GetUnique(None,self.hello_urn, rdfvalue.URN( 56 | lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY)), 57 | rdfvalue.XSDString("bar")) 58 | 59 | def testTurtleSerialization(self): 60 | data = self.store._DumpToTurtle(None, verbose=True) 61 | new_store = data_store.MemoryDataStore() 62 | stream = io.BytesIO(data.encode('utf-8')) 63 | new_store.LoadFromTurtle(stream, None) 64 | res = new_store.GetUnique(None,self.hello_urn, rdfvalue.URN( 65 | lexicon.AFF4_IMAGE_COMPRESSION_SNAPPY)) 66 | self.assertEquals(res, b"foo") 67 | 68 | 69 | class AFF4ObjectCacheMock(data_store.AFF4ObjectCache): 70 | def GetKeys(self): 71 | return [entry.key for entry in self.lru_list] 72 | 73 | def GetInUse(self): 74 | return [key for key in self.in_use] 75 | 76 | 77 | class AFF4ObjectCacheTest(unittest.TestCase): 78 | def testLRU(self): 79 | cache = AFF4ObjectCacheMock(3) 80 | resolver = data_store.MemoryDataStore() 81 | 82 | obj1 = aff4.AFF4Object(resolver, "a") 83 | obj2 = aff4.AFF4Object(resolver, "b") 84 | obj3 = aff4.AFF4Object(resolver, "c") 85 | obj4 = aff4.AFF4Object(resolver, "d") 86 | 87 | cache.Put(obj1) 88 | cache.Put(obj2) 89 | cache.Put(obj3) 90 | 91 | result = cache.GetKeys() 92 | 93 | # Keys are stored as serialized urns. 94 | self.assertEquals(result[0], "file:///c") 95 | self.assertEquals(result[1], "file:///b") 96 | self.assertEquals(result[2], "file:///a") 97 | 98 | # This removes the object from the cache and places it in the in_use 99 | # list. 100 | self.assertEquals(cache.Get("file:///a"), obj1) 101 | 102 | # Keys are stored as serialized urns. 103 | result = cache.GetKeys() 104 | self.assertEquals(len(result), 2) 105 | self.assertEquals(result[0], "file:///c") 106 | self.assertEquals(result[1], "file:///b") 107 | 108 | # Keys are stored as serialized urns. 109 | in_use = cache.GetInUse() 110 | self.assertEquals(len(in_use), 1) 111 | self.assertEquals(in_use[0], "file:///a") 112 | 113 | # Now we return the object. It should now appear in the lru lists. 114 | cache.Return(obj1) 115 | 116 | result = cache.GetKeys() 117 | self.assertEquals(len(result), 3) 118 | 119 | self.assertEquals(result[0], "file:///a") 120 | self.assertEquals(result[1], "file:///c") 121 | self.assertEquals(result[2], "file:///b") 122 | 123 | in_use = cache.GetInUse() 124 | self.assertEquals(len(in_use), 0) 125 | 126 | # Over flow the cache - this should expire the older object. 127 | cache.Put(obj4) 128 | result = cache.GetKeys() 129 | self.assertEquals(len(result), 3) 130 | 131 | self.assertEquals(result[0], "file:///d") 132 | self.assertEquals(result[1], "file:///a") 133 | self.assertEquals(result[2], "file:///c") 134 | 135 | # b is now expired so not in cache. 136 | self.assertEquals(cache.Get("file:///b"), None) 137 | 138 | # Check that remove works 139 | cache.Remove(obj4) 140 | 141 | self.assertEquals(cache.Get("file:///d"), None) 142 | result = cache.GetKeys() 143 | self.assertEquals(len(result), 2) 144 | 145 | 146 | if __name__ == '__main__': 147 | unittest.main() 148 | -------------------------------------------------------------------------------- /pyaff4/dedup_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2016-2018 Schatz Forensic Pty Ltd. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | # use this file except in compliance with the License. You may obtain a copy of 7 | # the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations under 15 | # the License. 16 | 17 | from pyaff4 import data_store 18 | from pyaff4 import container 19 | from pyaff4 import logical 20 | from pyaff4 import escaping 21 | from pyaff4 import lexicon 22 | from pyaff4 import rdfvalue 23 | from pyaff4 import linear_hasher 24 | from pyaff4.container import Container 25 | from pyaff4 import hashes 26 | import unittest, traceback 27 | from pyaff4 import utils 28 | import os, tempfile 29 | 30 | """ 31 | Tests logical file creation 32 | """ 33 | 34 | 35 | 36 | class DedupeTest(unittest.TestCase): 37 | testImagesPath = os.path.join(os.path.dirname(__file__), u"..", u"test_images", u"AFF4-L") 38 | containerName = tempfile.gettempdir() + "/test-append-dedup.aff4" 39 | 40 | def setUp(self): 41 | pass 42 | 43 | def onValidHash(self, typ, hash, imageStreamURI): 44 | self.assertTrue(True) 45 | 46 | def onInvalidHash(self, typ, hasha, hashb, streamURI): 47 | self.fail() 48 | 49 | def testCreateAndAppendSinglePathImage(self): 50 | try: 51 | try: 52 | os.unlink(self.containerName) 53 | except: 54 | pass 55 | 56 | container_urn = rdfvalue.URN.FromFileName(self.containerName) 57 | resolver = data_store.MemoryDataStore() 58 | urn = None 59 | 60 | frag1path = os.path.join(self.testImagesPath, "paper-hash_based_disk_imaging_using_aff4.pdf.frag.1") 61 | 62 | with container.Container.createURN(resolver, container_urn) as volume: 63 | with open(frag1path, "rb") as src: 64 | stream = linear_hasher.StreamHasher(src, [lexicon.HASH_SHA1]) 65 | urn = volume.writeLogicalStreamHashBased(frag1path, stream, 32768, False) 66 | for h in stream.hashes: 67 | hh = hashes.newImmutableHash(h.hexdigest(), stream.hashToType[h]) 68 | self.assertEqual("deb3fa3b60c6107aceb97f684899387c78587eae", hh.value) 69 | resolver.Add(volume.urn, urn, rdfvalue.URN(lexicon.standard.hash), hh) 70 | 71 | frag2path = os.path.join(self.testImagesPath, "paper-hash_based_disk_imaging_using_aff4.pdf.frag.2") 72 | 73 | with container.Container.openURNtoContainer(container_urn, mode="+") as volume: 74 | with open(frag2path, "rb") as src: 75 | stream = linear_hasher.StreamHasher(src, [lexicon.HASH_SHA1, lexicon.HASH_MD5 ]) 76 | urn = volume.writeLogicalStreamHashBased(frag2path, stream, 2*32768, False) 77 | for h in stream.hashes: 78 | hh = hashes.newImmutableHash(h.hexdigest(), stream.hashToType[h]) 79 | resolver.Add(volume.urn, urn, rdfvalue.URN(lexicon.standard.hash), hh) 80 | 81 | with container.Container.openURNtoContainer(container_urn) as volume: 82 | images = list(volume.images()) 83 | images = sorted(images, key=lambda x: utils.SmartUnicode(x.pathName), reverse=False) 84 | self.assertEqual(2, len(images), "Only two logical images") 85 | 86 | fragmentA = escaping.member_name_for_urn(images[0].urn.value, volume.version, base_urn=volume.urn, use_unicode=True) 87 | fragmentB = escaping.member_name_for_urn(images[1].urn.value, volume.version, base_urn=volume.urn, use_unicode=True) 88 | 89 | self.assertTrue(fragmentA.endswith("paper-hash_based_disk_imaging_using_aff4.pdf.frag.1")) 90 | self.assertTrue(fragmentB.endswith("paper-hash_based_disk_imaging_using_aff4.pdf.frag.2")) 91 | 92 | hasher = linear_hasher.LinearHasher2(volume.resolver, self) 93 | for image in volume.images(): 94 | print("\t%s <%s>" % (image.name(), image.urn)) 95 | hasher.hash(image) 96 | 97 | except: 98 | traceback.print_exc() 99 | self.fail() 100 | 101 | finally: 102 | #os.unlink(containerName) 103 | pass 104 | 105 | 106 | 107 | 108 | if __name__ == '__main__': 109 | unittest.main() -------------------------------------------------------------------------------- /pyaff4/escaping.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Copyright 2018 Schatz Forensic Pty Ltd. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 7 | # use this file except in compliance with the License. You may obtain a copy of 8 | # the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations under 16 | # the License. 17 | 18 | from __future__ import unicode_literals 19 | from future import standard_library 20 | standard_library.install_aliases() 21 | from builtins import chr 22 | import os 23 | import re 24 | import shutil 25 | import string 26 | import urllib.parse 27 | import pyaff4 28 | from pyaff4 import rdfvalue 29 | from pyaff4 import utils 30 | 31 | 32 | PRINTABLES = set(string.printable) 33 | for i in "!$\\:*%?\"<>|]": 34 | PRINTABLES.discard(i) 35 | 36 | PRINTABLES_NO_SLASH = PRINTABLES.copy() 37 | PRINTABLES_NO_SLASH.discard('/') 38 | 39 | FORBIDDEN = set() 40 | for c in "<>\^`{|}": 41 | FORBIDDEN.add(c) 42 | 43 | # convert a file path to an ARN fragment 44 | # a basic implementation that aims for compatibility with OS specific implementations 45 | # that produce file:// URI's 46 | def arnPathFragment_from_path(pathName): 47 | escaped_path = [] 48 | if pathName[0] == ".": 49 | if pathName[1] == ".": 50 | pathName = pathName[2:] 51 | else: 52 | pathName = pathName[1:] 53 | if pathName[0:3] == "\\\\.": 54 | escaped_path.append(".") 55 | pathName = pathName[3:] 56 | 57 | for c in pathName: 58 | if ord(c) >= 0 and ord(c)<= 0x1f: 59 | # control codes 60 | escaped_path.append("%%%02x" % ord(c)) 61 | elif c == '\\': 62 | escaped_path.append("/") 63 | elif c == ' ': 64 | escaped_path.append("%20") 65 | elif c == '"': 66 | escaped_path.append("%22") 67 | elif c == '%': 68 | escaped_path.append("%25") 69 | elif c in FORBIDDEN: 70 | escaped_path.append("%%%02x" % ord(c)) 71 | else: 72 | escaped_path.append(c) 73 | 74 | if escaped_path[0] == u"/": 75 | if len(escaped_path) > 1 and escaped_path[1] == u"/": 76 | # unc path 77 | escaped_path = escaped_path[2:] 78 | return "".join(escaped_path) 79 | else: 80 | # regular rooted path 81 | return "".join(escaped_path) 82 | #pass 83 | #escaped_path = escaped_path[1:] 84 | elif escaped_path[0] == u".": 85 | return "".join(escaped_path) 86 | else: 87 | # relative path or windows drive path 88 | return "/" + "".join(escaped_path) 89 | 90 | def member_name_for_urn(member_urn, version, base_urn=None, slash_ok=True, use_unicode=False): 91 | filename = base_urn.RelativePath(member_urn) 92 | # The member is not related to the base URN, just concatenate them together. 93 | 94 | if filename is None: 95 | filename = os.path.join( 96 | base_urn.Parse().path, member_urn.SerializeToString()) 97 | 98 | if filename.startswith("/"): 99 | # non-unc based path 100 | filename = filename[1:] 101 | 102 | if version.isLessThanOrEqual(1,0): 103 | if slash_ok: 104 | acceptable_set = PRINTABLES 105 | else: 106 | acceptable_set = PRINTABLES_NO_SLASH 107 | 108 | # original implementations of AFF4 (and Evimetry) escape the leading aff4:// 109 | if filename.startswith("aff4://"): 110 | return filename.replace("aff4://", "aff4%3A%2F%2F") 111 | 112 | if not use_unicode: 113 | if slash_ok: 114 | acceptable_set = PRINTABLES 115 | else: 116 | acceptable_set = PRINTABLES_NO_SLASH 117 | 118 | # Escape chars which are non printable. 119 | escaped_filename = [] 120 | for c in filename: 121 | if c in acceptable_set: 122 | escaped_filename.append(c) 123 | else: 124 | escaped_filename.append("%%%02x" % ord(c)) 125 | return "".join(escaped_filename) 126 | elif version.isGreaterThanOrEqual(1,1): 127 | #return toSegmentName(filename) 128 | filename = filename.replace("%20", " ") 129 | return filename 130 | else: 131 | raise Exception("Illegal version") 132 | 133 | #def member_name_for_urn(arn, version, base_urn): 134 | # a = utils.SmartUnicode(arn) 135 | # b = utils.SmartUnicode(base_urn) 136 | # return a[len(b):] 137 | 138 | def urn_from_member_name(member, base_urn, version): 139 | """Returns a URN object from a zip file's member name.""" 140 | member = utils.SmartUnicode(member) 141 | 142 | if version != pyaff4.version.basic_zip: 143 | if version.isLessThanOrEqual(1, 0): 144 | # Remove %xx escapes. 145 | member = re.sub( 146 | "%(..)", lambda x: chr(int("0x" + x.group(1), 0)), 147 | member) 148 | elif version.equals(1,1): 149 | member = member.replace(" ", "%20") 150 | 151 | # This is an absolute URN. 152 | if urllib.parse.urlparse(member).scheme == "aff4": 153 | result = member 154 | else: 155 | # Relative member becomes relative to the volume's URN. 156 | result = base_urn.Append(member, quote=False) 157 | 158 | return rdfvalue.URN(result) 159 | 160 | def member_name_for_file_iri(arn): 161 | return arn[len("file://"):] 162 | 163 | -------------------------------------------------------------------------------- /pyaff4/hashes.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | # Copyright 2016,2017 Schatz Forensic Pty Ltd. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | 16 | from pyaff4.rdfvalue import * 17 | from pyaff4 import lexicon 18 | import hashlib 19 | import nacl.hashlib 20 | 21 | def new(datatype): 22 | if datatype == lexicon.HASH_BLAKE2B: 23 | return nacl.hashlib.blake2b(digest_size=512//8) 24 | return hashNameToFunctionMap[datatype]() 25 | 26 | def newImmutableHash(value, datatype): 27 | if datatype == lexicon.HASH_SHA1: 28 | h = SHA1Hash() 29 | elif datatype == lexicon.HASH_MD5: 30 | h = MD5Hash() 31 | elif datatype == lexicon.HASH_SHA512: 32 | h = SHA512Hash() 33 | elif datatype == lexicon.HASH_SHA256: 34 | h = SHA256Hash() 35 | elif datatype == lexicon.HASH_BLAKE2B: 36 | h = Blake2bHash() 37 | elif datatype == lexicon.HASH_BLOCKMAPHASH_SHA512: 38 | h = SHA512BlockMapHash() 39 | else: 40 | raise Exception 41 | h.Set(value) 42 | return h 43 | 44 | 45 | def toShortAlgoName(datatype): 46 | return new(datatype).name 47 | 48 | 49 | def fromShortName(name): 50 | return nameMap[name] 51 | 52 | 53 | def length(datatype): 54 | return hashNameToLengthMap[datatype] 55 | 56 | 57 | hashNameToFunctionMap = { 58 | lexicon.HASH_MD5: hashlib.md5, 59 | lexicon.HASH_SHA1: hashlib.sha1, 60 | lexicon.HASH_SHA256: hashlib.sha256, 61 | lexicon.HASH_SHA512: hashlib.sha512 62 | } 63 | 64 | hashNameToLengthMap = { 65 | lexicon.HASH_MD5: new(lexicon.HASH_MD5).digest_size, 66 | lexicon.HASH_SHA1: new(lexicon.HASH_SHA1).digest_size, 67 | lexicon.HASH_SHA256: new(lexicon.HASH_SHA256).digest_size, 68 | lexicon.HASH_SHA512: new(lexicon.HASH_SHA512).digest_size, 69 | lexicon.HASH_BLAKE2B: new(lexicon.HASH_BLAKE2B).digest_size, 70 | } 71 | 72 | nameMap = dict(md5=lexicon.HASH_MD5, sha1=lexicon.HASH_SHA1, sha256=lexicon.HASH_SHA256, sha512=lexicon.HASH_SHA512, 73 | blake2b=lexicon.HASH_BLAKE2B, blockMapHashSHA512=lexicon.HASH_BLOCKMAPHASH_SHA512) 74 | -------------------------------------------------------------------------------- /pyaff4/hashing_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import unicode_literals 3 | # Copyright 2016,2017 Schatz Forensic Pty Ltd. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | # use this file except in compliance with the License. You may obtain a copy of 7 | # the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations under 15 | # the License. 16 | import os 17 | import unittest 18 | import logging 19 | 20 | from pyaff4 import data_store 21 | from pyaff4 import lexicon 22 | from pyaff4 import plugins 23 | from pyaff4 import rdfvalue 24 | from pyaff4 import zip 25 | from pyaff4 import hashes 26 | from pyaff4 import block_hasher 27 | from pyaff4 import linear_hasher 28 | 29 | LOGGER = logging.getLogger("pyaff4") 30 | 31 | referenceImagesPath = os.path.join(os.path.dirname(__file__), u"..", 32 | u"test_images") 33 | stdLinear = os.path.join(referenceImagesPath, u"AFF4Std", u"Base-Linear.aff4") 34 | preStdLinear = os.path.join(referenceImagesPath, u"AFF4PreStd/Base-Linear.af4") 35 | preStdAllocated = os.path.join(referenceImagesPath, u"AFF4PreStd", 36 | u"Base-Allocated.af4") 37 | stdLinear = os.path.join(referenceImagesPath, u"AFF4Std", u"Base-Linear.aff4") 38 | stdAllocated = os.path.join(referenceImagesPath, u"AFF4Std", 39 | u"Base-Allocated.aff4") 40 | stdLinearAllHashes = os.path.join(referenceImagesPath, 41 | u"AFF4Std", u"Base-Linear-AllHashes.aff4") 42 | stdLinearReadError = os.path.join(referenceImagesPath, 43 | u"AFF4Std", u"Base-Linear-ReadError.aff4") 44 | stripedLinearA = os.path.join(referenceImagesPath, 45 | u"AFF4Std", u"Striped", u"Base-Linear_1.aff4") 46 | stripedLinearB = os.path.join(referenceImagesPath, 47 | u"AFF4Std", u"Striped", u"Base-Linear_2.aff4") 48 | 49 | def conditional_on_images(f): 50 | if not os.access(preStdLinear, os.R_OK): 51 | LOGGER.info("Test images not cloned into repository. Tests disabled." 52 | "To enable type `git submodules init`") 53 | 54 | def _decorator(): 55 | print (f.__name__ + ' has been disabled') 56 | 57 | return _decorator 58 | return f 59 | 60 | 61 | 62 | class ValidatorTest(unittest.TestCase): 63 | preStdLinearURN = rdfvalue.URN.FromFileName(preStdLinear) 64 | preStdAllocatedURN = rdfvalue.URN.FromFileName(preStdAllocated) 65 | stdLinearURN = rdfvalue.URN.FromFileName(stdLinear) 66 | stdAllocatedURN = rdfvalue.URN.FromFileName(stdAllocated) 67 | stdLinearAllHashesURN = rdfvalue.URN.FromFileName(stdLinearAllHashes) 68 | stdLinearReadErrorURN = rdfvalue.URN.FromFileName(stdLinearReadError) 69 | stripedLinearAURN = rdfvalue.URN.FromFileName(stripedLinearA) 70 | stripedLinearBURN = rdfvalue.URN.FromFileName(stripedLinearB) 71 | 72 | @conditional_on_images 73 | def testBlockHashPreStdLinearImage(self): 74 | validator = block_hasher.Validator() 75 | validator.validateContainer(self.preStdLinearURN) 76 | 77 | @conditional_on_images 78 | def testLinearHashPreStdLinearImage(self): 79 | validator = linear_hasher.LinearHasher() 80 | hash = validator.hash( 81 | self.preStdLinearURN, 82 | u"aff4://085066db-6315-4369-a87e-bdc7bc777d45", 83 | lexicon.HASH_SHA1) 84 | print(dir(hash)) 85 | print(hash.value) 86 | self.assertEqual(hash.value, "5d5f183ae7355b8dc8938b67aab77c0215c29ab4") 87 | 88 | @conditional_on_images 89 | def testLinearHashPreStdPartialAllocatedImage(self): 90 | validator = linear_hasher.LinearHasher() 91 | hash = validator.hash( 92 | self.preStdAllocatedURN, 93 | u"aff4://48a85e17-1041-4bcc-8b2b-7fb2cd4f815b", lexicon.HASH_SHA1) 94 | print(dir(hash)) 95 | print(hash.value) 96 | self.assertEqual(hash.value, "a9f21b04a0a77613a5a34ecdd3af269464984035") 97 | 98 | @conditional_on_images 99 | def testBlockHashPreStdPartialAllocatedImage(self): 100 | validator = block_hasher.Validator() 101 | validator.validateContainer(self.preStdAllocatedURN) 102 | 103 | @conditional_on_images 104 | def testBlockHashStdLinearImage(self): 105 | validator = block_hasher.Validator() 106 | validator.validateContainer(self.stdLinearURN) 107 | 108 | @conditional_on_images 109 | def testBlockHashStdLinearReadError(self): 110 | validator = block_hasher.Validator() 111 | validator.validateContainer(self.stdLinearReadErrorURN) 112 | 113 | @conditional_on_images 114 | def testHashStdLinearImage(self): 115 | validator = linear_hasher.LinearHasher() 116 | hash = validator.hash( 117 | self.stdLinearURN, 118 | u"aff4://fcbfdce7-4488-4677-abf6-08bc931e195b", lexicon.HASH_SHA1) 119 | print(dir(hash)) 120 | print(hash.value) 121 | self.assertEqual(hash.value, "7d3d27f667f95f7ec5b9d32121622c0f4b60b48d") 122 | 123 | @conditional_on_images 124 | def testHashStdLinearReadError(self): 125 | validator = linear_hasher.LinearHasher() 126 | hash = validator.hash( 127 | self.stdLinearReadErrorURN, 128 | u"aff4://b282d5f4-333a-4f6a-b96f-0e5138bb18c8", lexicon.HASH_SHA1) 129 | print(dir(hash)) 130 | print(hash.value) 131 | self.assertEqual(hash.value, "67e245a640e2784ead30c1ff1a3f8d237b58310f") 132 | 133 | @conditional_on_images 134 | def testHashStdPartialAllocatedImage(self): 135 | validator = linear_hasher.LinearHasher() 136 | hash = validator.hash( 137 | self.stdAllocatedURN, 138 | u"aff4://e9cd53d3-b682-4f12-8045-86ba50a0239c", lexicon.HASH_SHA1) 139 | self.assertEqual(hash.value, "e8650e89b262cf0b4b73c025312488d5a6317a26") 140 | 141 | @conditional_on_images 142 | def testBlockHashStdLinearStriped(self): 143 | validator = block_hasher.Validator() 144 | validator.validateContainerMultiPart(self.stripedLinearBURN, 145 | self.stripedLinearAURN) 146 | 147 | @conditional_on_images 148 | def testHashStdLinearStriped(self): 149 | validator = linear_hasher.LinearHasher() 150 | hash = validator.hashMulti( 151 | self.stripedLinearBURN, self.stripedLinearAURN, 152 | u"aff4://2dd04819-73c8-40e3-a32b-fdddb0317eac", lexicon.HASH_SHA1) 153 | self.assertEqual(hash.value, "7d3d27f667f95f7ec5b9d32121622c0f4b60b48d") 154 | 155 | @conditional_on_images 156 | def testBlockHashStdContainerPartialAllocated(self): 157 | validator = block_hasher.Validator() 158 | validator.validateContainer(self.stdAllocatedURN) 159 | 160 | @conditional_on_images 161 | def testBlockHashPreStdLinearImage(self): 162 | validator = block_hasher.Validator() 163 | validator.validateContainer(self.preStdLinearURN) 164 | 165 | @conditional_on_images 166 | def testBlockHashStdLinearAllHashesImage(self): 167 | validator = block_hasher.Validator() 168 | validator.validateContainer(self.stdLinearAllHashesURN) 169 | 170 | @conditional_on_images 171 | def testHashStdLinearAllHashesImage(self): 172 | validator = linear_hasher.LinearHasher() 173 | hash = validator.hash( 174 | self.stdLinearAllHashesURN, 175 | u"aff4://2a497fe5-0221-4156-8b4d-176bebf7163f", 176 | lexicon.HASH_SHA1) 177 | 178 | print(dir(hash)) 179 | print(hash.value) 180 | self.assertEqual(hash.value, "7d3d27f667f95f7ec5b9d32121622c0f4b60b48d") 181 | 182 | if __name__ == '__main__': 183 | unittest.main() 184 | -------------------------------------------------------------------------------- /pyaff4/hexdump.py: -------------------------------------------------------------------------------- 1 | def group(a, *ns): 2 | for n in ns: 3 | a = [a[i:i+n] for i in range(0, len(a), n)] 4 | return a 5 | 6 | def join(a, *cs): 7 | return [cs[0].join(join(t, *cs[1:])) for t in a] if cs else a 8 | 9 | def hexdump(data): 10 | toHex = lambda c: '{:02X}'.format(c) 11 | toChr = lambda c: chr(c) if 32 <= c < 127 else '.' 12 | make = lambda f, *cs: join(group(list(map(f, data)), 8, 2), *cs) 13 | hs = make(toHex, ' ', ' ') 14 | cs = make(toChr, ' ', '') 15 | for i, (h, c) in enumerate(zip(hs, cs)): 16 | print ('{:010X}: {:48} {:16}'.format(i * 16, h, c)) -------------------------------------------------------------------------------- /pyaff4/keybag.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Schatz Forensic Pty Ltd All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | # 15 | # Author: Bradley L Schatz bradley@evimetry.com 16 | 17 | from __future__ import print_function 18 | from __future__ import absolute_import 19 | from __future__ import unicode_literals 20 | 21 | from builtins import next 22 | from builtins import str 23 | from builtins import object 24 | import binascii, rdflib, os 25 | from passlib.crypto import digest 26 | from pyaff4.aes_keywrap import aes_wrap_key, aes_unwrap_key 27 | from pyaff4.utils import SmartStr 28 | from pyaff4 import aff4, lexicon, rdfvalue 29 | from rdflib import URIRef 30 | from rdflib.namespace import RDF 31 | from Crypto import Random 32 | from Crypto.Cipher import PKCS1_OAEP 33 | from Crypto.PublicKey import RSA 34 | from Crypto.Hash import SHA256, SHA1 35 | from Crypto.Signature import pss 36 | from cryptography import x509 37 | from cryptography.hazmat.backends import default_backend 38 | 39 | keysize = 0x20 # in bytes 40 | iterations = 147256 41 | saltSize = 16 42 | 43 | class PasswordWrappedKeyBag: 44 | def __init__(self, salt, iterations, keySizeBytes, wrappedKey): 45 | self.salt = salt 46 | self.iterations = iterations 47 | self.keySizeBytes = keySizeBytes 48 | self.wrappedKey = wrappedKey 49 | self.ID = aff4.newARN() 50 | 51 | @staticmethod 52 | def create(password): 53 | salt = Random.get_random_bytes(saltSize) 54 | vek = Random.get_random_bytes(keysize) 55 | #print("VEK: " + str(binascii.hexlify(vek))) 56 | kek = digest.pbkdf2_hmac("sha256", password, salt, iterations, keysize); 57 | wrapped_key = aes_wrap_key(kek, vek) 58 | #print("WrappedKey: " + str(binascii.hexlify(wrapped_key))) 59 | return PasswordWrappedKeyBag(salt, iterations, keysize, wrapped_key) 60 | 61 | @staticmethod 62 | def load(graph): 63 | for kb, p, o in graph.triples((None, RDF.type, rdflib.URIRef("http://aff4.org/Schema#KeyBag"))): 64 | wk = graph.value(kb, rdflib.URIRef("http://aff4.org/Schema#wrappedKey"), None) 65 | salt = graph.value(kb, rdflib.URIRef("http://aff4.org/Schema#salt"), None) 66 | iterations = graph.value(kb, rdflib.URIRef("http://aff4.org/Schema#iterations"), None) 67 | keySizeInBytes = graph.value(kb, rdflib.URIRef("http://aff4.org/Schema#keySizeInBytes"), None) 68 | return PasswordWrappedKeyBag(salt._value, iterations._value, keySizeInBytes._value, wk._value) 69 | 70 | def unwrap_key(self, password): 71 | kek = digest.pbkdf2_hmac("sha256", password, self.salt, self.iterations, self.keySizeBytes); 72 | vek = aes_unwrap_key(kek, self.wrappedKey) 73 | #print("VEK: " + str(binascii.hexlify(vek))) 74 | return vek 75 | 76 | def write(self, resolver, volumeARN): 77 | resolver.Add(volumeARN, self.ID, lexicon.AFF4_TYPE, rdfvalue.URN(lexicon.AFF4_PASSWORD_WRAPPED_KEYBAG)) 78 | resolver.Set(volumeARN, self.ID, lexicon.AFF4_KEYSIZEBYTES, rdfvalue.XSDInteger(self.keySizeBytes)) 79 | resolver.Set(volumeARN, self.ID, lexicon.AFF4_ITERATIONS, rdfvalue.XSDInteger(self.iterations)) 80 | resolver.Set(volumeARN, self.ID, lexicon.AFF4_WRAPPEDKEY, rdfvalue.RDFBytes(self.wrappedKey)) 81 | resolver.Set(volumeARN, self.ID, lexicon.AFF4_SALT, rdfvalue.RDFBytes(self.salt)) 82 | 83 | @staticmethod 84 | def loadFromResolver(resolver, volumeARN, keyBagARN): 85 | salt = resolver.GetUnique(volumeARN, keyBagARN, lexicon.standard11.salt) 86 | iterations = resolver.GetUnique(volumeARN, keyBagARN, lexicon.standard11.iterations) 87 | keySizeInBytes = resolver.GetUnique(volumeARN, keyBagARN, lexicon.standard11.keySizeInBytes) 88 | wrappedKey = resolver.GetUnique(volumeARN, keyBagARN, lexicon.standard11.wrappedKey) 89 | #print("WrappedKey: " + str(binascii.hexlify(wrappedKey.value))) 90 | return PasswordWrappedKeyBag(salt.value, iterations.value, keySizeInBytes.value, wrappedKey.value) 91 | 92 | class CertEncryptedKeyBag: 93 | def __init__(self, subjectName, serialNumber, keySizeBytes, wrappedKey): 94 | self.subjectName = subjectName 95 | self.serialNumber = serialNumber 96 | self.keySizeBytes = keySizeBytes 97 | self.wrappedKey = wrappedKey 98 | self.ID = aff4.newARN() 99 | 100 | 101 | @staticmethod 102 | def create(vek, keySizeBytes, certificatePath): 103 | #print("VEK: " + str(binascii.hexlify(vek))) 104 | publicKeyPem = open(certificatePath).read() 105 | publicKey = RSA.importKey(publicKeyPem) 106 | # Convert from PEM to DER 107 | 108 | lines = publicKeyPem.replace(" ", '').split() 109 | publicKeyDer = binascii.a2b_base64(''.join(lines[1:-1])) 110 | 111 | cert = x509.load_pem_x509_certificate(SmartStr(publicKeyPem), default_backend()) 112 | subjectName = cert.subject.rfc4514_string() 113 | serial = cert.serial_number 114 | 115 | cipher = PKCS1_OAEP.new(key=publicKey, hashAlgo=SHA256, mgfunc=lambda x, y: pss.MGF1(x, y, SHA1)) 116 | wrapped_key = cipher.encrypt(vek) 117 | #print("WrappedKey: " + str(binascii.hexlify(wrapped_key))) 118 | 119 | return CertEncryptedKeyBag(subjectName, serial, keySizeBytes, wrapped_key) 120 | 121 | 122 | def unwrap_key(self, privateKey): 123 | key = RSA.importKey(open(privateKey).read()) 124 | cipher = PKCS1_OAEP.new(key=key, hashAlgo=SHA256, mgfunc=lambda x, y: pss.MGF1(x, y, SHA1)) 125 | vek = cipher.decrypt(self.wrappedKey) 126 | #print("VEK: " + str(binascii.hexlify(vek))) 127 | return vek 128 | 129 | def write(self, resolver, volumeARN): 130 | resolver.Add(volumeARN, self.ID, lexicon.AFF4_TYPE, rdfvalue.URN(lexicon.AFF4_CERT_ENCRYPTED_KEYBAG)) 131 | resolver.Set(volumeARN, self.ID, lexicon.AFF4_KEYSIZEBYTES, rdfvalue.XSDInteger(self.keySizeBytes)) 132 | resolver.Set(volumeARN, self.ID, lexicon.AFF4_SERIALNUMBER, rdfvalue.XSDInteger(self.serialNumber)) 133 | resolver.Set(volumeARN, self.ID, lexicon.AFF4_WRAPPEDKEY, rdfvalue.RDFBytes(self.wrappedKey)) 134 | resolver.Set(volumeARN, self.ID, lexicon.AFF4_SUBJECTNAME, rdfvalue.XSDString(self.subjectName)) 135 | 136 | 137 | @staticmethod 138 | def loadFromResolver(resolver, volumeARN, keyBagARN): 139 | subjectName = resolver.GetUnique(volumeARN, keyBagARN, lexicon.standard11.subjectName) 140 | serial = resolver.GetUnique(volumeARN, keyBagARN, lexicon.standard11.serialNumber) 141 | keySizeInBytes = resolver.GetUnique(volumeARN, keyBagARN, lexicon.standard11.keySizeInBytes) 142 | wrappedKey = resolver.GetUnique(volumeARN, keyBagARN, lexicon.standard11.wrappedKey) 143 | #print("WrappedKey: " + str(binascii.hexlify(wrappedKey.value))) 144 | return CertEncryptedKeyBag(subjectName.value, serial.value, keySizeInBytes.value, wrappedKey.value) 145 | -------------------------------------------------------------------------------- /pyaff4/linear_hasher.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import unicode_literals 3 | # Copyright 2016,2017 Schatz Forensic Pty Ltd. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | # use this file except in compliance with the License. You may obtain a copy of 7 | # the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations under 15 | # the License. 16 | 17 | from builtins import object 18 | import io 19 | from pyaff4 import block_hasher 20 | from pyaff4 import container 21 | from pyaff4 import data_store 22 | from pyaff4 import hashes 23 | from pyaff4 import lexicon 24 | from pyaff4 import zip 25 | 26 | 27 | class LinearHasher(object): 28 | def __init__(self, listener=None): 29 | if listener == None: 30 | self.listener = block_hasher.ValidationListener() 31 | else: 32 | self.listener = listener 33 | self.delegate = None 34 | 35 | def hash(self, urn, mapURI, hashDataType): 36 | (version, lex) = container.Container.identifyURN(urn) 37 | resolver = data_store.MemoryDataStore(lex) 38 | 39 | with zip.ZipFile.NewZipFile(resolver, version, urn) as zip_file: 40 | if lex == lexicon.standard: 41 | self.delegate = InterimStdLinearHasher(resolver, lex, self.listener) 42 | elif lex == lexicon.legacy: 43 | self.delegate = PreStdLinearHasher(resolver, lex, self.listener) 44 | elif lex == lexicon.scudette: 45 | self.delegate = ScudetteLinearHasher(resolver, lex, self.listener) 46 | else: 47 | raise ValueError 48 | 49 | self.delegate.volume_arn = zip_file.urn 50 | return self.delegate.doHash(mapURI, hashDataType) 51 | 52 | def hashMulti(self, urna, urnb, mapURI, hashDataType): 53 | (version, lex) = container.Container.identifyURN(urna) 54 | resolver = data_store.MemoryDataStore(lex) 55 | 56 | with zip.ZipFile.NewZipFile(resolver, version, urna) as zip_filea: 57 | with zip.ZipFile.NewZipFile(resolver, version, urnb) as zip_fileb: 58 | if lex == lexicon.standard: 59 | self.delegate = InterimStdLinearHasher(resolver, lex, self.listener) 60 | elif lex == lexicon.legacy: 61 | self.delegate = PreStdLinearHasher(resolver, lex, self.listener) 62 | else: 63 | raise ValueError 64 | 65 | self.delegate.volume_arn = zip_filea.urn 66 | return self.delegate.doHash(mapURI, hashDataType) 67 | 68 | def doHash(self, mapURI, hashDataType): 69 | hash = hashes.new(hashDataType) 70 | if not self.isMap(mapURI): 71 | import pdb; pdb.set_trace() 72 | 73 | if self.isMap(mapURI): 74 | with self.resolver.AFF4FactoryOpen(mapURI) as mapStream: 75 | remaining = mapStream.Size() 76 | count = 0 77 | while remaining > 0: 78 | toRead = min(32*1024, remaining) 79 | data = mapStream.Read(toRead) 80 | assert len(data) == toRead 81 | remaining -= len(data) 82 | hash.update(data) 83 | count = count + 1 84 | 85 | b = hash.hexdigest() 86 | return hashes.newImmutableHash(b, hashDataType) 87 | raise Exception("IllegalState") 88 | 89 | def doHash(self, mapURI, hashDataType): 90 | hash = hashes.new(hashDataType) 91 | if not self.isMap(mapURI): 92 | import pdb; pdb.set_trace() 93 | 94 | if self.isMap(mapURI): 95 | with self.resolver.AFF4FactoryOpen(mapURI) as mapStream: 96 | remaining = mapStream.Size() 97 | count = 0 98 | while remaining > 0: 99 | toRead = min(32*1024, remaining) 100 | data = mapStream.Read(toRead) 101 | assert len(data) == toRead 102 | remaining -= len(data) 103 | hash.update(data) 104 | count = count + 1 105 | 106 | b = hash.hexdigest() 107 | return hashes.newImmutableHash(b, hashDataType) 108 | raise Exception("IllegalState") 109 | 110 | def isMap(self, stream): 111 | for type in self.resolver.QuerySubjectPredicate(self.volume_arn, stream, lexicon.AFF4_TYPE): 112 | if self.lexicon.map == type: 113 | return True 114 | 115 | return False 116 | 117 | 118 | class PreStdLinearHasher(LinearHasher): 119 | def __init__(self, resolver, lex, listener=None): 120 | LinearHasher.__init__(self, listener) 121 | self.lexicon = lex 122 | self.resolver = resolver 123 | 124 | 125 | class InterimStdLinearHasher(LinearHasher): 126 | def __init__(self, resolver, lex, listener=None): 127 | LinearHasher.__init__(self, listener) 128 | self.lexicon = lex 129 | self.resolver = resolver 130 | 131 | 132 | class ScudetteLinearHasher(LinearHasher): 133 | def __init__(self, resolver, lex, listener=None): 134 | LinearHasher.__init__(self, listener) 135 | self.lexicon = lex 136 | self.resolver = resolver 137 | 138 | class LinearHasher2: 139 | def __init__(self, resolver, listener=None): 140 | if listener == None: 141 | self.listener = block_hasher.ValidationListener() 142 | else: 143 | self.listener = listener 144 | self.delegate = None 145 | self.resolver = resolver 146 | 147 | def hash(self, image): 148 | 149 | storedHashes = list(self.resolver.QuerySubjectPredicate(image.container.urn, image.urn, lexicon.standard.hash)) 150 | with self.resolver.AFF4FactoryOpen(image.urn, version=image.container.version) as stream: 151 | datatypes = [h.datatype for h in storedHashes] 152 | stream2 = StreamHasher(stream, datatypes) 153 | self.readall2(stream2) 154 | for storedHash in storedHashes: 155 | dt = storedHash.datatype 156 | shortHashAlgoName = storedHash.shortName() 157 | calculatedHashHexDigest = stream2.getHash(dt).hexdigest() 158 | storedHashHexDigest = storedHash.value 159 | if storedHashHexDigest == calculatedHashHexDigest: 160 | self.listener.onValidHash(shortHashAlgoName, calculatedHashHexDigest, image.urn) 161 | else: 162 | self.listener.onInvalidHash(shortHashAlgoName, storedHashHexDigest, calculatedHashHexDigest, image.urn) 163 | 164 | 165 | def readall2(self, stream): 166 | while True: 167 | toRead = 32 * 1024 168 | data = stream.read(toRead) 169 | if data == None or len(data) == 0: 170 | # EOF 171 | return 172 | 173 | 174 | class StreamHasher(object): 175 | def __init__(self, parent, hashDatatypes): 176 | self.parent = parent 177 | self.hashes = [] 178 | self.hashToType = {} 179 | for hashDataType in hashDatatypes: 180 | h = hashes.new(hashDataType) 181 | self.hashToType[h] = hashDataType 182 | self.hashes.append(h) 183 | 184 | def read(self, bytes): 185 | data = self.parent.read(bytes) 186 | datalen = len(data) 187 | if datalen > 0: 188 | for h in self.hashes: 189 | h.update(data) 190 | return data 191 | 192 | def getHash(self, dataType): 193 | return next(h for h in self.hashes if self.hashToType[h] == dataType) 194 | 195 | class PushHasher(object): 196 | def __init__(self, hashDatatypes): 197 | self.hashes = [] 198 | self.hashToType = {} 199 | for hashDataType in hashDatatypes: 200 | h = hashes.new(hashDataType) 201 | self.hashToType[h] = hashDataType 202 | self.hashes.append(h) 203 | 204 | def update(self, data): 205 | datalen = len(data) 206 | if datalen > 0: 207 | for h in self.hashes: 208 | h.update(data) 209 | 210 | def getHash(self, dataType): 211 | return next(h for h in self.hashes if self.hashToType[h] == dataType) -------------------------------------------------------------------------------- /pyaff4/logical.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016-2018 Schatz Forensic Pty Ltd. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | import os 16 | import platform 17 | from pyaff4 import lexicon, rdfvalue 18 | import tzlocal 19 | import pytz 20 | from datetime import datetime 21 | from dateutil.parser import parse 22 | import traceback 23 | 24 | if platform.system() == "Linux": 25 | from pyaff4 import statx 26 | 27 | class FSMetadata(object): 28 | def __init__(self, urn, name, length): 29 | self.name = name 30 | self.length = length 31 | self.urn = urn 32 | 33 | def store(self, resolver): 34 | resolver.Set(self.urn, rdfvalue.URN(lexicon.size), rdfvalue.XSDInteger(self.length)) 35 | resolver.Set(self.urn, rdfvalue.URN(lexicon.name), rdfvalue.XSDInteger(self.name)) 36 | 37 | @staticmethod 38 | def createFromTarInfo(filename, tarinfo): 39 | size = tarinfo.size 40 | local_tz = tzlocal.get_localzone() 41 | lastWritten = datetime.fromtimestamp(tarinfo.mtime, local_tz) 42 | accessed = datetime.fromtimestamp(int(tarinfo.pax_headers["atime"]), local_tz) 43 | recordChanged = datetime.fromtimestamp(int(tarinfo.pax_headers["ctime"]), local_tz) 44 | # addedDate ?? todo 45 | return UnixMetadata(filename, filename, size, lastWritten, accessed, recordChanged) 46 | 47 | @staticmethod 48 | def createFromSFTPAttr(filename, attr): 49 | size = attr.st_size 50 | local_tz = tzlocal.get_localzone() 51 | lastWritten = datetime.fromtimestamp(attr.st_mtime, local_tz) 52 | accessed = datetime.fromtimestamp(attr.st_atime, local_tz) 53 | #recordChanged = datetime.fromtimestamp(attr.st_ctime, local_tz) 54 | # addedDate ?? todo 55 | return UnixMetadata(filename, filename, size, lastWritten, accessed, 0) 56 | 57 | @staticmethod 58 | def create(filename): 59 | s = os.stat(filename) 60 | p = platform.system() 61 | local_tz = tzlocal.get_localzone() 62 | 63 | if p == "Windows": 64 | size = s.st_size 65 | birthTime = datetime.fromtimestamp(s.st_ctime, local_tz) 66 | lastWritten = datetime.fromtimestamp(s.st_mtime, local_tz) 67 | accessed = datetime.fromtimestamp(s.st_atime, local_tz) 68 | 69 | return WindowsFSMetadata(filename, filename, size, lastWritten, accessed, birthTime) 70 | elif p == "Darwin": 71 | # https://forensic4cast.com/2016/10/macos-file-movements/ 72 | size = s.st_size 73 | birthTime = datetime.fromtimestamp(s.st_birthtime, local_tz) 74 | lastWritten = datetime.fromtimestamp(s.st_mtime, local_tz) 75 | accessed = datetime.fromtimestamp(s.st_atime, local_tz) 76 | recordChanged = datetime.fromtimestamp(s.st_ctime, local_tz) 77 | # addedDate ?? todo 78 | return MacOSFSMetadata(filename, filename, size, lastWritten, accessed, recordChanged, birthTime) 79 | elif p == "Linux": 80 | size = s.st_size 81 | # TODO: birthTime 82 | lastWritten = datetime.fromtimestamp(s.st_mtime, local_tz) 83 | accessed = datetime.fromtimestamp(s.st_atime, local_tz) 84 | recordChanged = datetime.fromtimestamp(s.st_ctime, local_tz) 85 | 86 | sx = statx.statx(filename) 87 | birthTime = datetime.fromtimestamp(sx.get_btime(), local_tz) 88 | return LinuxFSMetadata(filename, filename, size, lastWritten, accessed, recordChanged, birthTime) 89 | 90 | class ClassicUnixMetadata(FSMetadata): 91 | def __init__(self, urn, name, size, lastWritten, lastAccessed, recordChanged): 92 | super(ClassicUnixMetadata, self).__init__(urn, name, size) 93 | self.lastWritten = lastWritten 94 | self.lastAccessed = lastAccessed 95 | self.recordChanged = recordChanged 96 | 97 | 98 | def store(self, resolver): 99 | resolver.Set(self.urn, self.urn, rdfvalue.URN(lexicon.AFF4_STREAM_SIZE), rdfvalue.XSDInteger(self.length)) 100 | resolver.Set(self.urn, self.urn, rdfvalue.URN(lexicon.standard11.lastWritten), rdfvalue.XSDDateTime(self.lastWritten)) 101 | resolver.Set(self.urn, self.urn, rdfvalue.URN(lexicon.standard11.lastAccessed), rdfvalue.XSDDateTime(self.lastAccessed)) 102 | resolver.Set(self.urn, self.urn, rdfvalue.URN(lexicon.standard11.recordChanged), rdfvalue.XSDDateTime(self.recordChanged)) 103 | 104 | class ModernUnixMetadata(ClassicUnixMetadata): 105 | def __init__(self, urn, name, size, lastWritten, lastAccessed, recordChanged, birthTime): 106 | super(ModernUnixMetadata, self).__init__(urn, name, size, lastWritten, lastAccessed, recordChanged) 107 | self.birthTime = birthTime 108 | 109 | def store(self, resolver): 110 | super(ModernUnixMetadata, self).store(resolver) 111 | resolver.Set(self.urn, self.urn, rdfvalue.URN(lexicon.standard11.birthTime), rdfvalue.XSDDateTime(self.birthTime)) 112 | 113 | class LinuxFSMetadata(ModernUnixMetadata): 114 | def __init__(self, urn, name, size, lastWritten, lastAccessed, recordChanged, birthTime): 115 | super(LinuxFSMetadata, self).__init__(urn, name, size, lastWritten, lastAccessed, recordChanged, birthTime) 116 | 117 | 118 | class MacOSFSMetadata(ModernUnixMetadata): 119 | def __init__(self, urn, name, size, lastWritten, lastAccessed, recordChanged, birthTime): 120 | super(MacOSFSMetadata, self).__init__(urn, name, size, lastWritten, lastAccessed, recordChanged, birthTime) 121 | 122 | class WindowsFSMetadata(FSMetadata): 123 | def __init__(self, urn, name, size, lastWritten, lastAccessed, birthTime): 124 | super(WindowsFSMetadata, self).__init__(urn, name, size) 125 | self.lastWritten = lastWritten 126 | self.lastAccessed = lastAccessed 127 | self.birthTime = birthTime 128 | 129 | def store(self, resolver): 130 | resolver.Set(self.urn, self.urn, rdfvalue.URN(lexicon.AFF4_STREAM_SIZE), rdfvalue.XSDInteger(self.length)) 131 | resolver.Set(self.urn, self.urn, rdfvalue.URN(lexicon.standard11.lastWritten), rdfvalue.XSDDateTime(self.lastWritten)) 132 | resolver.Set(self.urn, self.urn, rdfvalue.URN(lexicon.standard11.lastAccessed), rdfvalue.XSDDateTime(self.lastAccessed)) 133 | resolver.Set(self.urn, self.urn, rdfvalue.URN(lexicon.standard11.birthTime), rdfvalue.XSDDateTime(self.birthTime)) 134 | 135 | def resetTimestampsPosix(destFile, lastWritten, lastAccessed, recordChanged, birthTime): 136 | if lastWritten == None or lastAccessed == None: 137 | return 138 | try: 139 | lw = parse(lastWritten.value) 140 | la = parse(lastAccessed.value) 141 | os.utime(destFile, ((la - epoch).total_seconds(), (lw - epoch).total_seconds())) 142 | except Exception: 143 | traceback.print_exc() 144 | 145 | # default implementation does nothing at present on non posix environments 146 | def resetTimestampsNone(destFile, lastWritten, lastAccessed, recordChanged, birthTime): 147 | pass 148 | 149 | resetTimestamps = resetTimestampsNone 150 | epoch = datetime(1970, 1, 1, tzinfo=pytz.utc) 151 | 152 | p = platform.system() 153 | if p == "Darwin" or p == "Linux": 154 | resetTimestamps = resetTimestampsPosix -------------------------------------------------------------------------------- /pyaff4/logical_append_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2016-2018 Schatz Forensic Pty Ltd. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | # use this file except in compliance with the License. You may obtain a copy of 7 | # the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations under 15 | # the License. 16 | import tempfile 17 | 18 | from pyaff4 import data_store, container, logical 19 | from pyaff4 import escaping 20 | from pyaff4 import lexicon 21 | from pyaff4 import rdfvalue 22 | from pyaff4 import aff4_map 23 | from pyaff4.container import Container 24 | from pyaff4 import zip 25 | import unittest, traceback 26 | from pyaff4 import utils 27 | import os, io 28 | 29 | """ 30 | Tests logical file creation 31 | """ 32 | class LogicalAppendTest(unittest.TestCase): 33 | testImagesPath = os.path.join(os.path.dirname(__file__), u"..", u"test_images", u"AFF4-L") 34 | 35 | 36 | def setUp(self): 37 | pass 38 | 39 | @unittest.skip 40 | def testCreateAndAppendSinglePathImageLarge2(self): 41 | try: 42 | containerName = tempfile.gettempdir() + u"/test-append-large2.aff4" 43 | pathA = u"/a.txt" 44 | pathB = u"/b.txt" 45 | largedata = io.BytesIO(os.urandom(1100000)) 46 | container_urn = rdfvalue.URN.FromFileName(containerName) 47 | resolver = data_store.MemoryDataStore() 48 | urn = None 49 | with container.Container.createURN(resolver, container_urn) as volume: 50 | src = io.BytesIO("hello".encode('utf-8')) 51 | urn = volume.writeLogical(pathA, src, 10) 52 | 53 | urn = None 54 | with container.Container.openURNtoContainer(container_urn, mode="+") as volume: 55 | 56 | src = largedata 57 | urn = volume.writeLogical(pathB, src, 110000) 58 | 59 | with container.Container.openURNtoContainer(container_urn) as volume: 60 | images = list(volume.images()) 61 | images = sorted(images, key=lambda x: utils.SmartUnicode(x.pathName), reverse=False) 62 | self.assertEqual(2, len(images), "Only two logical images") 63 | 64 | fragmentA = escaping.member_name_for_urn(images[0].urn.value, volume.version, base_urn=volume.urn, use_unicode=True) 65 | fragmentB = escaping.member_name_for_urn(images[1].urn.value, volume.version, base_urn=volume.urn, use_unicode=True) 66 | 67 | self.assertEqual(pathA, fragmentA) 68 | self.assertEqual(pathB, fragmentB) 69 | 70 | try: 71 | with volume.resolver.AFF4FactoryOpen(images[0].urn) as fd: 72 | txt = fd.ReadAll() 73 | self.assertEqual(b"hello", txt, "content should be same") 74 | with volume.resolver.AFF4FactoryOpen(images[1].urn) as fd: 75 | index = 0 76 | while index < 110000: 77 | fd.SeekRead(index) 78 | bufa = fd.Read(1000) 79 | largedata.seek(index) 80 | bufb = largedata.read(1000) 81 | index = index + 1000 82 | self.assertEqual(bufa, bufb, "content should be same") 83 | except Exception: 84 | traceback.print_exc() 85 | self.fail() 86 | 87 | except: 88 | traceback.print_exc() 89 | self.fail() 90 | 91 | finally: 92 | pass 93 | # os.unlink(containerName) 94 | 95 | @unittest.skip 96 | def testCreateAndAppendSinglePathImageLarge(self): 97 | try: 98 | length = 10000 99 | containerName = tempfile.gettempdir() + u"/test-append-large.aff4" 100 | pathA = u"/a.txt" 101 | pathB = u"/b.txt" 102 | largedata = io.BytesIO(os.urandom(1100000)) 103 | container_urn = rdfvalue.URN.FromFileName(containerName) 104 | resolver = data_store.MemoryDataStore() 105 | urn = None 106 | with container.Container.createURN(resolver, container_urn) as volume: 107 | src = io.BytesIO("hello".encode('utf-8')) 108 | urn = volume.writeLogical(pathA, src, 10) 109 | 110 | urn = None 111 | with container.Container.openURNtoContainer(container_urn, mode="+") as volume: 112 | src = largedata 113 | 114 | with volume.newLogicalStream(pathB, length) as image: 115 | image.chunk_size = 10 116 | image.chunks_per_segment = 3 117 | 118 | index = 0 119 | while index < length: 120 | src.seek(index) 121 | image.Write(src.read(1000)) 122 | index = index + 1000 123 | 124 | foo = 11 125 | image.Close() 126 | 127 | with container.Container.openURNtoContainer(container_urn) as volume: 128 | images = list(volume.images()) 129 | images = sorted(images, key=lambda x: utils.SmartUnicode(x.pathName), reverse=False) 130 | self.assertEqual(2, len(images), "Only two logical images") 131 | 132 | fragmentA = escaping.member_name_for_urn(images[0].urn.value, volume.version, base_urn=volume.urn, use_unicode=True) 133 | fragmentB = escaping.member_name_for_urn(images[1].urn.value, volume.version, base_urn=volume.urn, use_unicode=True) 134 | 135 | if fragmentA != u"/a.txt": 136 | ffffff = 1 137 | 138 | self.assertEqual(pathA, fragmentA) 139 | self.assertEqual(pathB, fragmentB) 140 | 141 | try: 142 | with volume.resolver.AFF4FactoryOpen(images[0].urn) as fd: 143 | txt = fd.ReadAll() 144 | self.assertEqual(b"hello", txt, "content should be same") 145 | with volume.resolver.AFF4FactoryOpen(images[1].urn) as fd: 146 | blen = fd.Length() 147 | self.assertEqual(length, blen) 148 | except Exception: 149 | traceback.print_exc() 150 | self.fail() 151 | 152 | except: 153 | traceback.print_exc() 154 | self.fail() 155 | 156 | finally: 157 | pass 158 | # os.unlink(containerName) 159 | 160 | def testCreateAndAppendSinglePathImage(self): 161 | try: 162 | containerName = tempfile.gettempdir() + u"/test-append.aff4" 163 | pathA = u"/a.txt" 164 | pathB = u"/b.txt" 165 | 166 | try: 167 | os.unlink(containerName) 168 | except: 169 | pass 170 | 171 | container_urn = rdfvalue.URN.FromFileName(containerName) 172 | resolver = data_store.MemoryDataStore() 173 | urn = None 174 | with container.Container.createURN(resolver, container_urn) as volume: 175 | src = io.BytesIO("hello".encode('utf-8')) 176 | urn = volume.writeLogical(pathA, src, 10) 177 | 178 | urn = None 179 | with container.Container.openURNtoContainer(container_urn, mode="+") as volume: 180 | src = io.BytesIO("hello2".encode('utf-8')) 181 | urn = volume.writeLogical(pathB, src, 12) 182 | 183 | with container.Container.openURNtoContainer(container_urn) as volume: 184 | images = list(volume.images()) 185 | images = sorted(images, key=lambda x: utils.SmartUnicode(x.pathName), reverse=False) 186 | self.assertEqual(2, len(images), "Only two logical images") 187 | 188 | fragmentA = escaping.member_name_for_urn(images[0].urn.value, volume.version, base_urn=volume.urn, use_unicode=True) 189 | fragmentB = escaping.member_name_for_urn(images[1].urn.value, volume.version, base_urn=volume.urn, use_unicode=True) 190 | 191 | self.assertEqual(pathA, fragmentA) 192 | self.assertEqual(pathB, fragmentB) 193 | 194 | try: 195 | with volume.resolver.AFF4FactoryOpen(images[0].urn) as fd: 196 | txt = fd.ReadAll() 197 | self.assertEqual(b"hello", txt, "content should be same") 198 | with volume.resolver.AFF4FactoryOpen(images[1].urn) as fd: 199 | txt = fd.ReadAll() 200 | self.assertEqual(b"hello2", txt, "content should be same") 201 | except Exception: 202 | traceback.print_exc() 203 | self.fail() 204 | 205 | except: 206 | traceback.print_exc() 207 | self.fail() 208 | 209 | finally: 210 | os.unlink(containerName) 211 | 212 | 213 | 214 | 215 | if __name__ == '__main__': 216 | unittest.main() -------------------------------------------------------------------------------- /pyaff4/plugins.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | # Copyright 2014 Google Inc. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | from pyaff4 import aff4 16 | from pyaff4 import aff4_directory 17 | try: 18 | from pyaff4 import aff4_cloud 19 | except ImportError: 20 | pass 21 | 22 | from pyaff4 import aff4_file 23 | from pyaff4 import aff4_image 24 | from pyaff4 import aff4_map 25 | from pyaff4 import zip 26 | -------------------------------------------------------------------------------- /pyaff4/rdfvalue_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | from pyaff4 import rdfvalue 3 | import unittest 4 | 5 | class URNTest(unittest.TestCase): 6 | 7 | def testXSDInt(self): 8 | i1 = rdfvalue.XSDInteger("100") 9 | self.assertLess(99, i1) 10 | self.assertEqual(100, i1) 11 | self.assertGreater(101, 100) 12 | self.assertGreater(101, i1) 13 | self.assertTrue(99 < i1) 14 | self.assertTrue(101 > i1) 15 | self.assertTrue(100 == i1) 16 | 17 | def testURN(self): 18 | url = "http://www.google.com/path/to/element#hash_data" 19 | self.assertEquals(rdfvalue.URN(url), url) 20 | # dont overload the constructor of URN to support a non-URN as an input 21 | # self.assertEquals(rdfvalue.URN("//etc/passwd"), 22 | # "file://etc/passwd") 23 | 24 | def testTrailingSlashURN(self): 25 | url = "http://code.google.com/p/snappy/" 26 | test = rdfvalue.URN(url) 27 | self.assertEquals(test.SerializeToString(), 28 | "http://code.google.com/p/snappy/") 29 | 30 | def testAppend(self): 31 | test = rdfvalue.URN("http://www.google.com") 32 | aff4volume = rdfvalue.URN("aff4://volumeguid/image/0000") 33 | 34 | self.assertEquals(aff4volume.Append("index").SerializeToString(), 35 | "aff4://volumeguid/image/0000/index") 36 | 37 | self.assertEquals(test.Append("foobar").SerializeToString(), 38 | "http://www.google.com/foobar") 39 | 40 | self.assertEquals(test.Append("/foobar").SerializeToString(), 41 | "http://www.google.com/foobar") 42 | 43 | self.assertEquals(test.Append("..").SerializeToString(), 44 | "http://www.google.com/") 45 | 46 | self.assertEquals(test.Append("../../../..").SerializeToString(), 47 | "http://www.google.com/") 48 | 49 | self.assertEquals(test.Append("aa/bb/../..").SerializeToString(), 50 | "http://www.google.com/") 51 | 52 | self.assertEquals(test.Append("aa//../c").SerializeToString(), 53 | "http://www.google.com/c") 54 | 55 | self.assertEquals( 56 | test.Append("aa///////////.///./c").SerializeToString(), 57 | "http://www.google.com/aa/c") 58 | 59 | 60 | if __name__ == '__main__': 61 | unittest.main() 62 | -------------------------------------------------------------------------------- /pyaff4/reference_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Schatz Forensic Pty Ltd. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | import tempfile 15 | 16 | from pyaff4 import data_store 17 | from pyaff4 import hashes 18 | from pyaff4 import lexicon 19 | from pyaff4 import rdfvalue 20 | from pyaff4 import aff4_map 21 | from pyaff4.container import Container 22 | from pyaff4 import zip 23 | from pyaff4 import utils 24 | from pyaff4 import version 25 | 26 | import unittest 27 | import os 28 | 29 | 30 | """ 31 | Creates an container with virtual file called "pdf1" which is backed by a byte range in another image. 32 | The byte range is the file called "2015-Schatz-Extending AFF4 for Scalable Acquisition Live Analysis.pdf", found 33 | in the AFF4 Canonical Reference Image "AFF4Std/Base-Linear.aff4" 34 | """ 35 | class ReferenceTest(unittest.TestCase): 36 | referenceImagesPath = os.path.join(os.path.dirname(__file__), u"..", u"test_images") 37 | stdLinear = os.path.join(referenceImagesPath, u"AFF4Std", u"Base-Linear.aff4") 38 | fileName = tempfile.gettempdir() + u"/reference.aff4" 39 | 40 | def setUp(self): 41 | with data_store.MemoryDataStore() as resolver: 42 | # Use the AFF4 Standard Lexicon 43 | self.lexicon = lexicon.standard 44 | 45 | 46 | with zip.ZipFile.NewZipFile(resolver, version.aff4v10, rdfvalue.URN.FromFileName(self.stdLinear)) as image_container: 47 | # there is generally only one Image in a container. Get the underlying Map 48 | imageURN = next(resolver.QueryPredicateObject(image_container.urn, lexicon.AFF4_TYPE, self.lexicon.Image)) 49 | datastreams = list(resolver.QuerySubjectPredicate(image_container.urn, imageURN, self.lexicon.dataStream)) 50 | imageMapURN = datastreams[0] 51 | 52 | # get a reference to the actual bytestream that is the forensic image 53 | with resolver.AFF4FactoryOpen(imageMapURN) as mapStream: 54 | 55 | # now that we have a reference to the forensic image, we start building up a new container 56 | # to store our new artefacts in 57 | 58 | # create a second resolver so we dont pollute our metadata with that of the first container 59 | with data_store.MemoryDataStore() as resolver2: 60 | 61 | # create our new container 62 | destFileURN = rdfvalue.URN.FromFileName(self.fileName) 63 | resolver2.Set(lexicon.transient_graph, destFileURN, lexicon.AFF4_STREAM_WRITE_MODE, 64 | rdfvalue.XSDString(u"truncate")) 65 | with zip.ZipFile.NewZipFile(resolver2, version.aff4v10, destFileURN) as image_container: 66 | self.volume_urn = image_container.urn 67 | 68 | # create a "version.txt" file so readers can tell it is an AFF4 Standard v1.0 container 69 | version_urn = self.volume_urn.Append("version.txt") 70 | with resolver2.AFF4FactoryOpen(self.volume_urn ) as volume: 71 | with volume.CreateMember(version_urn) as versionFile: 72 | versionFile.Write(utils.SmartStr(u"major=1\nminor=0\ntool=pyaff4\n")) 73 | 74 | # create a map to represent the byte range we are interested in 75 | self.image_urn = self.volume_urn.Append("pdf1") 76 | with aff4_map.AFF4Map.NewAFF4Map( 77 | resolver2, self.image_urn, self.volume_urn) as imageURN: 78 | # add the segment that refers to the file in the destination address space 79 | # the locations were determined by opening in a forensic tool 80 | partitionOffset = 0x10000 81 | fileOffset = 0xfc3000 82 | diskOffset = partitionOffset + fileOffset 83 | fileSize = 629087 84 | imageURN.AddRange(0, diskOffset, fileSize, mapStream.urn) 85 | 86 | 87 | 88 | def testReadMapOfImage(self): 89 | fileSize = 629087 90 | 91 | # take the lexicon from our new container 92 | (version, lex) = Container.identify(self.fileName) 93 | 94 | # setup a resolver 95 | resolver = data_store.MemoryDataStore(lex) 96 | 97 | # open the two containers within the same resolver (needed so the transitive links work) 98 | with zip.ZipFile.NewZipFile(resolver, version, rdfvalue.URN.FromFileName(self.stdLinear)) as targetContainer: 99 | with zip.ZipFile.NewZipFile(resolver, version, rdfvalue.URN.FromFileName(self.fileName)) as sourceContainer: 100 | 101 | # open the virtual file and read 102 | image_urn = sourceContainer.urn.Append("pdf1") 103 | with resolver.AFF4FactoryOpen(image_urn) as image: 104 | # check the size is right 105 | self.assertEquals(629087, image.Size()) 106 | 107 | # read the header of the virtual file 108 | image.SeekRead(0, 0) 109 | self.assertEquals(b"%PDF", image.Read(4)) 110 | 111 | # read the whole virtual file and compare with a known hash of it 112 | image.SeekRead(0, 0) 113 | buf = image.Read(629087) 114 | hash = hashes.new(lexicon.HASH_SHA1) 115 | hash.update(buf) 116 | self.assertEquals("5A2FEE16139C7B017B7F1961D842D355A860C7AC".lower(), hash.hexdigest()) 117 | 118 | 119 | 120 | if __name__ == '__main__': 121 | unittest.main() -------------------------------------------------------------------------------- /pyaff4/registry.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | """Various registries.""" 16 | from __future__ import unicode_literals 17 | 18 | # Global registry for AFF4 object implementations. 19 | AFF4_TYPE_MAP = {} 20 | 21 | # Registry for RDF type implementations. 22 | RDF_TYPE_MAP = {} 23 | -------------------------------------------------------------------------------- /pyaff4/standards_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import unicode_literals 3 | # Copyright 2016,2017 Schatz Forensic Pty Ltd. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | # use this file except in compliance with the License. You may obtain a copy of 7 | # the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations under 15 | # the License. 16 | 17 | from future import standard_library 18 | standard_library.install_aliases() 19 | 20 | import logging 21 | import os 22 | import io 23 | import unittest 24 | 25 | from pyaff4 import data_store 26 | from pyaff4 import lexicon 27 | from pyaff4 import plugins 28 | from pyaff4 import rdfvalue 29 | from pyaff4 import zip 30 | from pyaff4 import hashes 31 | from pyaff4 import version 32 | 33 | LOGGER = logging.getLogger("pyaff4") 34 | 35 | referenceImagesPath = os.path.join(os.path.dirname(__file__), "..", "test_images") 36 | stdLinear = os.path.join(referenceImagesPath, "AFF4Std", "Base-Linear.aff4") 37 | 38 | def conditional_on_images(f): 39 | if not os.access(stdLinear, os.R_OK): 40 | LOGGER.info("Test images not cloned into repository. Tests disabled." 41 | "To enable type `git submodules init`") 42 | 43 | def _decorator(): 44 | print (f.__name__ + ' has been disabled') 45 | 46 | return _decorator 47 | return f 48 | 49 | 50 | class StandardsTest(unittest.TestCase): 51 | stdLinearURN = rdfvalue.URN.FromFileName(stdLinear) 52 | 53 | @conditional_on_images 54 | def testLocateImage(self): 55 | resolver = data_store.MemoryDataStore() 56 | 57 | with zip.ZipFile.NewZipFile(resolver, version.aff4v10, self.stdLinearURN) as zip_file: 58 | for subject in resolver.QueryPredicateObject(zip_file.urn, 59 | "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", 60 | "http://aff4.org/Schema#DiskImage"): 61 | self.assertEquals( 62 | subject, 63 | "aff4://cf853d0b-5589-4c7c-8358-2ca1572b87eb") 64 | 65 | for subject in resolver.QueryPredicateObject(zip_file.urn, 66 | "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", 67 | "http://aff4.org/Schema#Image"): 68 | self.assertEquals( 69 | subject, 70 | "aff4://cf853d0b-5589-4c7c-8358-2ca1572b87eb") 71 | 72 | for subject in resolver.QueryPredicateObject(zip_file.urn, 73 | "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", 74 | "http://aff4.org/Schema#ContiguousImage"): 75 | self.assertEquals( 76 | subject, 77 | "aff4://cf853d0b-5589-4c7c-8358-2ca1572b87eb") 78 | 79 | @conditional_on_images 80 | def testReadMap(self): 81 | resolver = data_store.MemoryDataStore() 82 | 83 | with zip.ZipFile.NewZipFile(resolver, version.aff4v10, self.stdLinearURN) as zip_file: 84 | imageStream = resolver.AFF4FactoryOpen( 85 | "aff4://c215ba20-5648-4209-a793-1f918c723610") 86 | 87 | imageStream.SeekRead(0x163) 88 | res = imageStream.Read(17) 89 | self.assertEquals(res, b"Invalid partition") 90 | 91 | @conditional_on_images 92 | def testReadImageStream(self): 93 | resolver = data_store.MemoryDataStore() 94 | 95 | with zip.ZipFile.NewZipFile(resolver, version.aff4v10, self.stdLinearURN) as zip_file: 96 | mapStream = resolver.AFF4FactoryOpen( 97 | "aff4://c215ba20-5648-4209-a793-1f918c723610") 98 | 99 | mapStream.SeekRead(0x163) 100 | res = mapStream.Read(17) 101 | self.assertEquals(res, b"Invalid partition") 102 | 103 | 104 | if __name__ == '__main__': 105 | unittest.main() 106 | -------------------------------------------------------------------------------- /pyaff4/statx.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Arun Prasannan . All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | from __future__ import division 16 | import os 17 | import ctypes 18 | import platform 19 | 20 | from pyaff4 import utils 21 | 22 | 23 | # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/stat.h?h=v4.19#n46 24 | class StatxTimestamp(ctypes.Structure): 25 | _fields_ = [("tv_sec", ctypes.c_longlong), 26 | ("tv_nsec", ctypes.c_uint), 27 | ("__reserved", ctypes.c_int)] 28 | 29 | 30 | # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/stat.h?h=v4.19#n62 31 | class Statx(ctypes.Structure): 32 | _fields_ = [("stx_mask", ctypes.c_uint), 33 | ("stx_blksize", ctypes.c_uint), 34 | ("stx_attributes", ctypes.c_ulonglong), 35 | 36 | ("stx_nlink", ctypes.c_uint), 37 | ("stx_uid", ctypes.c_uint), 38 | ("stx_gid", ctypes.c_uint), 39 | ("stx_mode", ctypes.c_ushort), 40 | ("__spare0", ctypes.c_ushort * 1), 41 | 42 | ("stx_ino", ctypes.c_ulonglong), 43 | ("stx_size", ctypes.c_ulonglong), 44 | ("stx_blocks", ctypes.c_ulonglong), 45 | ("stx_attributes_mask", ctypes.c_ulonglong), 46 | 47 | ("stx_atime", StatxTimestamp), 48 | ("stx_btime", StatxTimestamp), 49 | ("stx_ctime", StatxTimestamp), 50 | ("stx_mtime", StatxTimestamp), 51 | 52 | ("stx_rdev_major", ctypes.c_uint), 53 | ("stx_rdev_minor", ctypes.c_uint), 54 | ("stx_dev_major", ctypes.c_uint), 55 | ("stx_dev_minor", ctypes.c_uint), 56 | 57 | ("__spare2", ctypes.c_ulonglong * 14)] 58 | 59 | def get_btime(self): 60 | return self.stx_btime.tv_sec + (self.stx_btime.tv_nsec / 1000000000) 61 | 62 | 63 | # https://github.com/hrw/syscalls-table 64 | SYSCALLS = { 65 | "alpha": 522, 66 | "arc": 291, 67 | "arm": 397, 68 | "arm64": 291, 69 | "armoabi": 9437581, 70 | "c6x": 291, 71 | "csky": 291, 72 | "h8300": 291, 73 | "hexagon": 291, 74 | "i386": 383, 75 | "m68k": 379, 76 | "metag": 291, 77 | "microblaze": 398, 78 | "mips64": 5326, 79 | "mips64n32": 6330, 80 | "mipso32": 4366, 81 | "nds32": 291, 82 | "nios2": 291, 83 | "openrisc": 291, 84 | "parisc": 349, 85 | "powerpc": 383, 86 | "powerpc64": 383, 87 | "riscv": 291, 88 | "s390": 379, 89 | "s390x": 379, 90 | "score": 291, 91 | "sparc": 360, 92 | "sparc64": 360, 93 | "tile": 291, 94 | "tile64": 291, 95 | "unicore32": 291, 96 | "x32": 1073742156, 97 | "x86_64": 332, 98 | "xtensa": 351, 99 | } 100 | 101 | 102 | # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/fcntl.h?h=v4.19 103 | AT_FDCWD = -100 # fcntl.h 104 | AT_SYMLINK_NOFOLLOW = 0x100 # fcntl.h 105 | STATX_ALL = 0xfff # stat.h 106 | SYS_STATX = SYSCALLS[platform.machine()] 107 | 108 | 109 | def statx(path): 110 | pathname = ctypes.c_char_p(utils.SmartStr(path)) 111 | statxbuf = ctypes.create_string_buffer(ctypes.sizeof(Statx)) 112 | 113 | lib = ctypes.CDLL(None, use_errno=True) 114 | syscall = lib.syscall 115 | 116 | # int statx(int dirfd, const char *pathname, int flags, unsigned int mask, struct statx *statxbuf); 117 | syscall.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_uint, ctypes.c_char_p] 118 | syscall.restype = ctypes.c_int 119 | 120 | if syscall(SYS_STATX, AT_FDCWD, pathname, AT_SYMLINK_NOFOLLOW, STATX_ALL, statxbuf): 121 | e = ctypes.get_errno() 122 | raise OSError(e, os.strerror(e), path) 123 | return Statx.from_buffer(statxbuf) 124 | 125 | -------------------------------------------------------------------------------- /pyaff4/stream_factory.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | # Copyright 2016,2017 Schatz Forensic Pty Ltd. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | 16 | from builtins import str 17 | from builtins import object 18 | from pyaff4.symbolic_streams import * 19 | from pyaff4 import rdfvalue 20 | import re 21 | 22 | class StreamFactory(object): 23 | def __init__(self, resolver, lex): 24 | self.lexicon = lex 25 | self.resolver = resolver 26 | self.symbolmatcher = re.compile("[0-9A-F]{2}") 27 | self.fixedSymbolics = [ self.lexicon.base + "Zero", 28 | self.lexicon.base + "UnknownData", 29 | self.lexicon.base + "UnreadableData", 30 | self.lexicon.base + "NoData"] 31 | 32 | # TODO: Refactor the below classes to split the subname from the NS 33 | # then do matching only on the subnname 34 | 35 | class PreStdStreamFactory(StreamFactory): 36 | def __init__(self, resolver, lex): 37 | StreamFactory.__init__(self, resolver, lex) 38 | self.fixedSymbolics.append(self.lexicon.base + "FF") 39 | 40 | def isSymbolicStream(self, urn): 41 | if type(urn) == rdfvalue.URN: 42 | urn = str(urn) 43 | if not urn.startswith("http://"): 44 | return False 45 | else: 46 | if urn in self.fixedSymbolics: 47 | return True 48 | 49 | # Pre-Std Evimetry Symbolic Streams are of the form 50 | # http://afflib.org/2009#FF 51 | if urn.startswith(self.lexicon.base) and len(urn) == len(self.lexicon.base) + 2: 52 | # now verify symbolic part 53 | shortName = urn[len(self.lexicon.base):].upper() 54 | 55 | if self.symbolmatcher.match(shortName) != None: 56 | return True 57 | 58 | if urn.startswith(self.lexicon.base + "SymbolicStream"): 59 | return True 60 | 61 | if urn.startswith("http://afflib.org/2012/SymbolicStream#"): 62 | return True 63 | 64 | return False 65 | 66 | def createSymbolic(self, urn): 67 | if type(urn) == rdfvalue.URN: 68 | urn = str(urn) 69 | 70 | if urn == self.lexicon.base + "Zero": 71 | return RepeatedStream(resolver=self.resolver, urn=urn, 72 | symbol=b"\x00") 73 | 74 | if urn == self.lexicon.base + "FF": 75 | return RepeatedStream(resolver=self.resolver, urn=urn, 76 | symbol=b"\xff") 77 | 78 | if urn == self.lexicon.base + "UnknownData": 79 | return RepeatedStringStream(resolver=self.resolver, urn=urn, 80 | repeated_string=GetUnknownString()) 81 | 82 | if (urn.startswith(self.lexicon.base + "SymbolicStream") and 83 | len(urn) == len(self.lexicon.base + "SymbolicStream") + 2): 84 | shortName = urn[len(self.lexicon.base + "SymbolicStream"):].upper() 85 | value = binascii.unhexlify(shortName) 86 | return RepeatedStream(resolver=self.resolver, urn=urn, symbol=value) 87 | 88 | if (urn.startswith("http://afflib.org/2012/SymbolicStream#") and 89 | len(urn) == len("http://afflib.org/2012/SymbolicStream#") + 2): 90 | shortName = urn[len("http://afflib.org/2012/SymbolicStream#"):].upper() 91 | value = binascii.unhexlify(shortName) 92 | return RepeatedStream(resolver=self.resolver, urn=urn, 93 | symbol=value) 94 | 95 | if urn.startswith(self.lexicon.base) and len(urn) == len(self.lexicon.base) + 2: 96 | shortName = urn[len(self.lexicon.base):].upper() 97 | value = binascii.unhexlify(shortName) 98 | return RepeatedStream(resolver=self.resolver, urn=urn, 99 | symbol=value) 100 | 101 | raise ValueError 102 | 103 | 104 | class StdStreamFactory(StreamFactory): 105 | 106 | def isSymbolicStream(self, urn): 107 | if type(urn) == rdfvalue.URN: 108 | urn = str(urn) 109 | if not urn.startswith("http://"): 110 | return False 111 | else: 112 | if urn in self.fixedSymbolics: 113 | return True 114 | 115 | if urn.startswith(self.lexicon.base + "SymbolicStream"): 116 | return True 117 | 118 | return False 119 | 120 | def createSymbolic(self, urn): 121 | if type(urn) == rdfvalue.URN: 122 | urn = str(urn) 123 | 124 | if urn == self.lexicon.base + "Zero": 125 | return RepeatedStream(resolver=self.resolver, urn=urn, 126 | symbol=b"\x00") 127 | 128 | if urn == self.lexicon.base + "UnknownData": 129 | return RepeatedStringStream( 130 | resolver=self.resolver, urn=urn, 131 | repeated_string=GetUnknownString()) 132 | 133 | if urn == self.lexicon.base + "UnreadableData": 134 | return RepeatedStringStream( 135 | resolver=self.resolver, urn=urn, 136 | repeated_string=GetUnreadableString()) 137 | 138 | if (urn.startswith(self.lexicon.base + "SymbolicStream") and 139 | len(urn) == len(self.lexicon.base + "SymbolicStream") + 2): 140 | shortName = urn[len(self.lexicon.base + "SymbolicStream"):].upper() 141 | value = binascii.unhexlify(shortName) 142 | return RepeatedStream(resolver=self.resolver, urn=urn, 143 | symbol=value) 144 | 145 | raise ValueError 146 | 147 | 148 | def _MakeTile(repeated_string): 149 | """Make exactly 1Mb tile of the repeated string.""" 150 | total_size = 1024*1024 151 | tile = repeated_string * (total_size // len(repeated_string)) 152 | tile += repeated_string[:total_size % len(repeated_string)] 153 | return tile 154 | 155 | # Exactly 1Mb. 156 | _UNKNOWN_STRING = None 157 | def GetUnknownString(): 158 | global _UNKNOWN_STRING 159 | if _UNKNOWN_STRING is not None: 160 | return _UNKNOWN_STRING 161 | 162 | _UNKNOWN_STRING = _MakeTile(b"UNKNOWN") 163 | return _UNKNOWN_STRING 164 | 165 | 166 | # Exactly 1Mb. 167 | _UNREADABLE_STRING = None 168 | def GetUnreadableString(): 169 | global _UNREADABLE_STRING 170 | if _UNREADABLE_STRING is not None: 171 | return _UNREADABLE_STRING 172 | 173 | _UNREADABLE_STRING = _MakeTile(b"UNREADABLEDATA") 174 | return _UNREADABLE_STRING 175 | -------------------------------------------------------------------------------- /pyaff4/stream_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | # Copyright 2014 Google Inc. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | 16 | import os 17 | import unittest 18 | 19 | from pyaff4 import data_store 20 | from pyaff4 import lexicon 21 | from pyaff4 import rdfvalue 22 | import tempfile 23 | import traceback 24 | 25 | class StreamTest(unittest.TestCase): 26 | def streamTest(self, stream): 27 | self.assertEquals(0, stream.TellRead()) 28 | self.assertEquals(0, stream.Size()) 29 | 30 | stream.Write(b"hello world") 31 | self.assertEquals(11, stream.TellWrite()) 32 | 33 | stream.SeekRead(0, 0) 34 | self.assertEquals(0, stream.TellRead()) 35 | 36 | self.assertEquals(b"hello world", 37 | stream.Read(1000)) 38 | 39 | self.assertEquals(11, stream.TellRead()) 40 | 41 | stream.SeekRead(-5, 2) 42 | self.assertEquals(6, stream.TellRead()) 43 | 44 | self.assertEquals(b"world", 45 | stream.Read(1000)) 46 | 47 | stream.SeekWrite(6, 0) 48 | self.assertEquals(6, stream.TellWrite()) 49 | 50 | stream.Write(b"Cruel world") 51 | stream.SeekRead(0, 0) 52 | self.assertEquals(0, stream.TellRead()) 53 | self.assertEquals(b"hello Cruel world", 54 | stream.Read(1000)) 55 | 56 | self.assertEquals(17, stream.TellRead()) 57 | 58 | stream.SeekRead(0, 0) 59 | 60 | self.assertEquals(b"he", 61 | stream.Read(2)) 62 | 63 | stream.SeekWrite(2,0) 64 | stream.Write(b"I have %d arms and %#x legs." % (2, 1025)) 65 | self.assertEquals(31, stream.TellWrite()) 66 | 67 | stream.SeekRead(0, 0) 68 | self.assertEquals(b"heI have 2 arms and 0x401 legs.", 69 | stream.Read(1000)) 70 | 71 | def testFileBackedStream(self): 72 | filename = tempfile.gettempdir() + "/test_filename.zip" 73 | fileURI = rdfvalue.URN.FromFileName(filename) 74 | 75 | try: 76 | with data_store.MemoryDataStore() as resolver: 77 | resolver.Set(lexicon.transient_graph, fileURI, lexicon.AFF4_STREAM_WRITE_MODE, 78 | rdfvalue.XSDString("truncate")) 79 | 80 | with resolver.AFF4FactoryOpen(fileURI) as file_stream: 81 | self.streamTest(file_stream) 82 | except: 83 | traceback.print_exc() 84 | self.fail() 85 | 86 | finally: 87 | os.unlink(filename) 88 | 89 | 90 | if __name__ == '__main__': 91 | unittest.main() 92 | -------------------------------------------------------------------------------- /pyaff4/streams.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016-2018 Schatz Forensic Pty Ltd. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | def ReadAll(stream): 16 | res = b"" 17 | while True: 18 | toRead = 32 * 1024 19 | data = stream.read(toRead) 20 | if data == None or len(data) == 0: 21 | # EOF 22 | return res 23 | else: 24 | res += data 25 | 26 | def WriteAll(fromstream, tostream): 27 | while True: 28 | toRead = 32 * 1024 29 | data = fromstream.read(toRead) 30 | if data == None or len(data) == 0: 31 | # EOF 32 | return 33 | else: 34 | tostream.write(data) -------------------------------------------------------------------------------- /pyaff4/struct_parser.py: -------------------------------------------------------------------------------- 1 | """An implementation of a struct parser which is fast and convenient.""" 2 | from __future__ import unicode_literals 3 | 4 | from builtins import zip 5 | from builtins import object 6 | import six 7 | import struct 8 | 9 | from pyaff4 import utils 10 | 11 | format_string_map = dict( 12 | uint64_t="Q", 13 | int64_t="q", 14 | uint32_t="I", 15 | uint16_t="H", 16 | int32_t="i", 17 | int16_t="h", 18 | ) 19 | 20 | class BaseParser(object): 21 | __slots__ = ("_data", "_fields", "_name", "_format_string", "_defaults") 22 | 23 | def __init__(self, data=None, **kwargs): 24 | if data is None: 25 | self._data = self._defaults[:] 26 | else: 27 | self._data = list( 28 | struct.unpack_from(self._format_string, data)) 29 | 30 | if kwargs: 31 | for k, v in list(kwargs.items()): 32 | setattr(self, k, v) 33 | 34 | def __str__(self): 35 | result = ["Struct %s" % self._name] 36 | for field, data in zip(self._fields, self._data): 37 | result.append(" %s: %s" % (field, data)) 38 | 39 | return "\n".join(result) 40 | 41 | def Pack(self): 42 | return struct.pack(self._format_string, *self._data) 43 | 44 | @classmethod 45 | def sizeof(cls): 46 | return struct.calcsize(cls._format_string) 47 | 48 | 49 | def CreateStruct(struct_name, definition): 50 | fields = [] 51 | format_string = ["<"] 52 | defaults = [] 53 | 54 | for line in definition.splitlines(): 55 | line = line.strip(" ;") 56 | components = line.split() 57 | if len(components) >= 2: 58 | type_format_char = format_string_map.get(components[0]) 59 | name = components[1] 60 | 61 | if type_format_char is None: 62 | raise RuntimeError("Invalid definition %r" % line) 63 | 64 | try: 65 | if components[2] != "=": 66 | raise RuntimeError("Invalid definition %r" % line) 67 | defaults.append(int(components[3], 0)) 68 | except IndexError: 69 | defaults.append(0) 70 | 71 | format_string.append(type_format_char) 72 | fields.append(name) 73 | 74 | properties = dict( 75 | _format_string="".join(format_string), 76 | _fields=fields, 77 | _defaults=defaults, 78 | _name=struct_name) 79 | 80 | # Make accessors for all fields. 81 | for i, field in enumerate(fields): 82 | def setx(self, value, i=i): 83 | self._data[i] = value 84 | 85 | def getx(self, i=i): 86 | return self._data[i] 87 | 88 | properties[field] = property(getx, setx) 89 | 90 | if six.PY2: 91 | return type(utils.SmartStr(struct_name), (BaseParser,), properties) 92 | else: 93 | return type(utils.SmartUnicode(struct_name), (BaseParser,), properties) 94 | -------------------------------------------------------------------------------- /pyaff4/symbolic_streams.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | from __future__ import absolute_import 3 | from __future__ import unicode_literals 4 | # Copyright 2016,2017 Schatz Forensic Pty Ltd. All rights reserved. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 7 | # use this file except in compliance with the License. You may obtain a copy of 8 | # the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 15 | # License for the specific language governing permissions and limitations under 16 | # the License. 17 | 18 | from builtins import str 19 | from past.utils import old_div 20 | from pyaff4 import aff4 21 | from pyaff4 import utils 22 | import sys 23 | import binascii 24 | import math 25 | 26 | class RepeatedStream(aff4.AFF4Stream): 27 | 28 | def __init__(self, resolver=None, urn=None, symbol=b"\x00"): 29 | super(RepeatedStream, self).__init__( 30 | resolver=resolver, urn=urn) 31 | self.symbol = symbol 32 | 33 | def Read(self, length): 34 | return self.symbol * length 35 | 36 | def Write(self, data): 37 | raise NotImplementedError() 38 | 39 | def WriteStream(self, source): 40 | raise NotImplementedError() 41 | 42 | def TellRead(self): 43 | return self.readptr 44 | 45 | def Size(self): 46 | return sys.maxsize 47 | 48 | def read(self, length=1024*1024): 49 | return self.Read(length) 50 | 51 | def seek(self, offset, whence=0): 52 | self.Seek(offset, whence=whence) 53 | 54 | def write(self, data): 55 | self.Write(data) 56 | 57 | def tell(self): 58 | return self.Tell() 59 | 60 | def flush(self): 61 | self.Flush() 62 | 63 | def Prepare(self): 64 | self.SeekRead(0) 65 | 66 | 67 | class RepeatedStringStream(aff4.AFF4Stream): 68 | 69 | def __init__(self, resolver=None, urn=None, repeated_string=None): 70 | super(RepeatedStringStream, self).__init__( 71 | resolver=resolver, urn=urn) 72 | 73 | self.tile = repeated_string 74 | self.tilesize = len(self.tile) 75 | 76 | def Read(self, length): 77 | toRead = length 78 | res = b"" 79 | while toRead > 0: 80 | offsetInTile = self.readptr % self.tilesize 81 | chunk = self.tile[offsetInTile : offsetInTile + toRead] 82 | res += chunk 83 | toRead -= len(chunk) 84 | self.readptr += len(chunk) 85 | 86 | return res 87 | 88 | def Write(self, data): 89 | raise NotImplementedError() 90 | 91 | def WriteStream(self, source): 92 | raise NotImplementedError() 93 | 94 | def Tell(self): 95 | return self.readptr 96 | 97 | def Size(self): 98 | return sys.maxsize 99 | 100 | def read(self, length=1024*1024): 101 | return self.Read(length) 102 | 103 | def seek(self, offset, whence=0): 104 | self.Seek(offset, whence=whence) 105 | 106 | def write(self, data): 107 | self.Write(data) 108 | 109 | def tell(self): 110 | return self.Tell() 111 | 112 | def flush(self): 113 | self.Flush() 114 | 115 | def Prepare(self): 116 | self.SeekRead(0) 117 | -------------------------------------------------------------------------------- /pyaff4/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python -m unittest discover -p '*test.py' -v 4 | -------------------------------------------------------------------------------- /pyaff4/test_crypto.py: -------------------------------------------------------------------------------- 1 | from future import standard_library 2 | standard_library.install_aliases() 3 | import os 4 | import io 5 | import unittest 6 | 7 | import binascii 8 | from pyaff4.aes_keywrap import aes_wrap_key, aes_unwrap_key 9 | from passlib.crypto import digest 10 | from CryptoPlus.Cipher import python_AES 11 | from Crypto.Cipher import PKCS1_OAEP 12 | from Crypto.PublicKey import RSA 13 | from Crypto.Hash import SHA256, SHA1 14 | from Crypto.Signature import pss 15 | import codecs 16 | import rdflib 17 | from pyaff4 import keybag 18 | 19 | referenceImagesPath = os.path.join(os.path.dirname(__file__), u"..", 20 | u"test_images") 21 | cert = os.path.join(referenceImagesPath, u"keys", u"certificate.pem") 22 | privateKey = os.path.join(referenceImagesPath, u"keys", u"key.pem") 23 | 24 | keybagturtle = """ 25 | @prefix : . 26 | @prefix rdf: . 27 | @prefix xsd: . 28 | @prefix aff4: . 29 | 30 | 31 | a aff4:KeyBag ; 32 | aff4:wrappedKey "5934f7d07e75f5ab55b9051ebd39331dbfba3c597589b203728043577bf93badeb9f07f528c8bd95"^^xsd:hexBinary ; 33 | aff4:salt "000102030405060708090a0b0c0d0e0f"^^xsd:hexBinary ; 34 | aff4:iterations 147256 ; 35 | aff4:keySizeInBytes 32 . 36 | """ 37 | 38 | src = "I am happy to join with you today in what will go down in history as the greatest demonstration for freedom in the history of our nation.".encode() 39 | 40 | target_ciphertext = binascii.unhexlify( 41 | "0c2e8b50c053afa8b09331096241490a6ab339f39f6530491bbbb6a75d46d6bc188fcda8e4b72c763e7dc7f55335f" + 42 | "909be8a049267f7c2e26189a352b7466520e6af498e9d674e99efd8753f1a46733072dfbb43a1665f1aec207bfc023998edc3ff0d9ca1" + 43 | "42013e7cfef85236649e0b4ae51b6a4758742bb4b17ec6e2cd47235e41739e464c5128c466c4d6f16d97724ebc764b1f91bb28313d3e2" + 44 | "8e3a54d73543f173d93c9b4cbba16d8bca5300095d0412057118551d9adb5142a5c3b4e0ab12f4858c608165eb24891e8e815a3815c06" + 45 | "9cce94ce75f018a01856a01e0a952e1d8015fb46ca80fd0fb17f2a9c348be6a86be3a202a7dec76ef04e7e04483eb9ccd2dbcf7943e59" + 46 | "0c7c03e2e0ed297b08a09984ff9f9c89c32c0fdcd8f814e8e9d4b39c1bf082b2f1a0f852dc3f48fc014b2300e75c85d6ce7f4ef3c5afa" + 47 | "cbf49ba2e00288a23e57196dc3558821578a9e452a687eb7b53b3477d3eda4c6febbbec59fc7bef46cbad3abbc6b4aefaf9aeb6b935ba" + 48 | "55afc2747ad4fe53ddcfed77caaa2628ce0aa4d836703d134ace22d9c6ac0f9d65113c21e05d49913ba6650ca3c1a34640a876218de50" + 49 | "f07cf743ff8902c456ae7ef8cee28ec0e0c5dbdcdce173dde8bb80f69d84a38a4cd580149100a144cbe844f9fe355186654ab525b28db" + 50 | "411c49d4c96b84670471f60765d048e03178663b4fec9d9bb05835c52f7") 51 | 52 | class CryptoTest(unittest.TestCase): 53 | 54 | def testCreateWrappedKey(self): 55 | wrapped = binascii.unhexlify("5934f7d07e75f5ab55b9051ebd39331dbfba3c597589b203728043577bf93badeb9f07f528c8bd95") 56 | g = rdflib.Graph() 57 | g.parse(data=keybagturtle, format="turtle") 58 | 59 | kb = keybag.PasswordWrappedKeyBag.load(g) 60 | self.assertEquals(wrapped, kb.wrappedKey) 61 | 62 | def testExtractWrappedKey(self): 63 | wrapped = binascii.unhexlify("5934f7d07e75f5ab55b9051ebd39331dbfba3c597589b203728043577bf93badeb9f07f528c8bd95") 64 | target_kek = binascii.unhexlify("9bc68a8c80a008d758de97cebc7ec39d6274512e3ddbdd5baf4eb8557ab7e58f") 65 | target_vek = binascii.unhexlify("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f") 66 | 67 | g = rdflib.Graph() 68 | g.parse(data=keybagturtle, format="turtle") 69 | kb = keybag.PasswordWrappedKeyBag.load(g) 70 | 71 | key = "password" 72 | kek = digest.pbkdf2_hmac("sha256", key, kb.salt, kb.iterations, kb.keySizeBytes); 73 | self.assertEquals(target_kek, kek) 74 | vek = aes_unwrap_key(kek, kb.wrappedKey) 75 | self.assertEquals(target_vek, vek) 76 | 77 | def testEncrypt(self): 78 | vek = binascii.unhexlify("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f") 79 | plaintext = src + b'\x00' * (512-len(src)) 80 | 81 | key1 = vek[0:16] 82 | key2 = vek[16:] 83 | 84 | tweak = codecs.decode('00', 'hex') 85 | 86 | cipher = python_AES.new((key1,key2), python_AES.MODE_XTS) 87 | ciphertext = cipher.encrypt(plaintext, tweak) 88 | 89 | self.assertEqual(target_ciphertext, ciphertext) 90 | 91 | def testDecrypt(self): 92 | 93 | g = rdflib.Graph() 94 | g.parse(data=keybagturtle, format="turtle") 95 | kb = keybag.PasswordWrappedKeyBag.load(g) 96 | 97 | key = "password" 98 | kek = digest.pbkdf2_hmac("sha256", key, kb.salt, kb.iterations, kb.keySizeBytes); 99 | vek = aes_unwrap_key(kek, kb.wrappedKey) 100 | 101 | key1 = vek[0:16] 102 | key2 = vek[16:] 103 | tweak = codecs.decode('00', 'hex') 104 | 105 | cipher = python_AES.new((key1, key2), python_AES.MODE_XTS) 106 | text = cipher.decrypt(target_ciphertext, tweak) 107 | 108 | self.assertEqual(src[0:len(src)], text[0:len(src)]) 109 | 110 | 111 | def testWrap(self): 112 | keysize = 0x20 # in bytes 113 | key = "password" 114 | iterations = 147256 115 | saltSize = 16 116 | salt = binascii.unhexlify("000102030405060708090a0b0c0d0e0f") 117 | 118 | 119 | #hhh = hashlib.pbkdf2_hmac("sha256", key.encode(), salt, iterations, keysize); 120 | #print(len(hhh)) 121 | #print(binascii.hexlify(hhh)) 122 | 123 | kek = digest.pbkdf2_hmac("sha256", key, salt, iterations, keysize); 124 | print(binascii.hexlify(kek)) 125 | 126 | #h = pbkdf2_sha256.encrypt(key, rounds=iterations, salt_size=saltSize) 127 | # print(h) 128 | 129 | 130 | 131 | vek = binascii.unhexlify("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f") 132 | print(len(vek)) 133 | wrapped_key= aes_wrap_key(kek, vek) 134 | print(binascii.hexlify(wrapped_key)) 135 | 136 | plaintext = src + b'\x00' * (512-len(src)) 137 | 138 | #msg = dict_xts_aes['msg%i' % i].decode('hex') 139 | #key = (dict_xts_aes['key1_%i' % i].decode('hex'), dict_xts_aes['key2_%i' % i].decode('hex')) 140 | key1 = vek[0:16] 141 | key2 = vek[16:] 142 | #cip = dict_xts_aes['cip%i' % i].decode('hex') 143 | #n = dict_xts_aes['n%i' % i].decode('hex') 144 | tweak = codecs.decode('00', 'hex') 145 | print(len(tweak)) 146 | cipher = python_AES.new((key1,key2), python_AES.MODE_XTS) 147 | ciphertext = cipher.encrypt(plaintext, tweak) 148 | 149 | print(len(ciphertext)) 150 | print(binascii.hexlify(ciphertext)) 151 | 152 | def testPKIWrap(self): 153 | vek = binascii.unhexlify("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f") 154 | 155 | publicKey = RSA.importKey(open(cert).read()) 156 | cipher = PKCS1_OAEP.new(key=publicKey, hashAlgo=SHA256, mgfunc=lambda x, y: pss.MGF1(x, y, SHA1)) 157 | ciphertext = cipher.encrypt(vek) 158 | 159 | key = RSA.importKey(open(privateKey).read()) 160 | cipher = PKCS1_OAEP.new(key=key, hashAlgo=SHA256, mgfunc=lambda x, y: pss.MGF1(x, y, SHA1)) 161 | vek2 = cipher.decrypt(ciphertext) 162 | self.assertEquals(vek, vek2) 163 | 164 | -------------------------------------------------------------------------------- /pyaff4/turtle.py: -------------------------------------------------------------------------------- 1 | 2 | def toDirectivesAndTripes(text): 3 | directives = [] 4 | triples = [] 5 | 6 | in_directives = True 7 | for line in text.splitlines(): 8 | if in_directives: 9 | if line.startswith("@"): 10 | directives.append(line) 11 | continue 12 | elif line == "": 13 | in_directives = False 14 | else: 15 | triples.append(line) 16 | 17 | return (u"\r\n".join(directives), u"\r\n".join(triples)) 18 | 19 | 20 | 21 | def difference(a, b): 22 | aset = set(a.split(u"\r\n")) 23 | bset = set(b.split(u"\r\n")) 24 | return aset.difference(bset) -------------------------------------------------------------------------------- /pyaff4/utils.py: -------------------------------------------------------------------------------- 1 | """Some utility functions.""" 2 | from __future__ import unicode_literals 3 | __author__ = "Michael Cohen " 4 | 5 | import six 6 | from future import types 7 | 8 | def SmartStr(string, encoding="utf8"): 9 | """Forces the string to be an encoded byte string.""" 10 | if six.PY3: 11 | if isinstance(string, str): 12 | return string.encode(encoding, "ignore") 13 | 14 | elif isinstance(string, bytes): 15 | return string 16 | 17 | elif hasattr(string, "__bytes__"): 18 | return string.__bytes__() 19 | 20 | return str(string).encode(encoding) 21 | 22 | if six.PY2: 23 | if type(string) is str: 24 | return string 25 | 26 | elif type(string) is unicode: 27 | return string.encode(encoding) 28 | 29 | elif hasattr(string, "__bytes__"): 30 | return string.__bytes__() 31 | 32 | return unicode(string).encode(encoding) 33 | 34 | 35 | def SmartUnicode(string, encoding="utf8"): 36 | """Forces the string into a unicode object.""" 37 | if six.PY3: 38 | if isinstance(string, bytes): 39 | return string.decode(encoding) 40 | 41 | # Call the object's __str__ method which should return an unicode 42 | # object. 43 | return str(string) 44 | 45 | elif six.PY2: 46 | if isinstance(string, str): 47 | return string.decode(encoding) 48 | 49 | return unicode(string) 50 | 51 | 52 | def AssertStr(string): 53 | if six.PY3: 54 | if type(string) is not bytes: 55 | raise RuntimeError("String must be bytes.") 56 | 57 | elif six.PY2: 58 | if type(string) not in (str, types.newstr): 59 | raise RuntimeError("String must be bytes.") 60 | 61 | 62 | def AssertUnicode(string): 63 | if six.PY3: 64 | if type(string) is not str: 65 | raise RuntimeError("String must be unicode.") 66 | 67 | elif six.PY2: 68 | if type(string) not in (unicode, types.newstr): 69 | raise RuntimeError("String must be unicode.") 70 | 71 | # TODO. This is so ugly. Need to go through and fix all calls to Get to make 72 | # sure they are expecting an array or generator in response 73 | def asList(a, b): 74 | a_islist = isinstance(a, list) 75 | b_islist = isinstance(b, list) 76 | 77 | if a == None: 78 | if b_islist: 79 | return b 80 | else: 81 | return [b] 82 | elif b == None: 83 | if a_islist: 84 | return a 85 | else: 86 | return [a] 87 | else: 88 | if a_islist and b_islist: 89 | a.extend(b) 90 | return a 91 | elif b_islist: 92 | b.append(a) 93 | return b 94 | elif a_islist: 95 | a.append(b) 96 | return a 97 | else: 98 | return [a,b] 99 | -------------------------------------------------------------------------------- /pyaff4/version.py: -------------------------------------------------------------------------------- 1 | class Version(object): 2 | def __init__(self, major, minor, tool): 3 | self.major = major 4 | self.minor = minor 5 | self.tool = tool 6 | 7 | @staticmethod 8 | def create(dic): 9 | return Version(int(dic["major"]),int(dic["minor"]),dic["tool"]) 10 | 11 | def is10(self): 12 | if self.major == 1 and self.minor == 0: 13 | return True 14 | return False 15 | 16 | def is11(self): 17 | if self.major == 1 and self.minor == 1: 18 | return True 19 | return False 20 | 21 | def isLessThanOrEqual(self, major, minor): 22 | if self.major < major: 23 | return True 24 | if self.major == major: 25 | if self.minor <= minor: 26 | return True 27 | return False 28 | 29 | def isGreaterThanOrEqual(self, major, minor): 30 | if self.major > major: 31 | return True 32 | if self.major == major: 33 | if self.minor >= minor: 34 | return True 35 | return False 36 | 37 | def equals(self, major, minor): 38 | return self.major == major and self.minor == minor 39 | 40 | def __str__(self): 41 | return u"major=%d\nminor=%d\ntool=%s\n" % (self.major, self.minor, self.tool) 42 | 43 | basic_zip = Version(0, 0, "pyaff4") 44 | aff4v10 = Version(1, 0, "pyaff4") 45 | aff4v11 = Version(1, 1, "pyaff4") -------------------------------------------------------------------------------- /pyaff4/zip_test.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | # Copyright 2015 Google Inc. All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | 16 | 17 | from future import standard_library 18 | standard_library.install_aliases() 19 | import os 20 | import io 21 | import unittest 22 | import tempfile 23 | 24 | from pyaff4 import data_store 25 | from pyaff4 import lexicon 26 | from pyaff4 import plugins 27 | from pyaff4 import rdfvalue 28 | from pyaff4 import zip 29 | from pyaff4 import version, hexdump 30 | 31 | 32 | class ZipTest(unittest.TestCase): 33 | filename = tempfile.gettempdir() + "/aff4_ziptest.zip" 34 | filename_urn = rdfvalue.URN.FromFileName(filename) 35 | segment_name = "Foobar.txt" 36 | streamed_segment = "streamed.txt" 37 | data1 = b"I am a segment!" 38 | data2 = b"I am another segment!" 39 | 40 | def setUp(self): 41 | try: 42 | os.unlink(self.filename) 43 | except (IOError, OSError): 44 | pass 45 | 46 | with data_store.MemoryDataStore() as resolver: 47 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 48 | rdfvalue.XSDString("truncate")) 49 | 50 | with zip.ZipFile.NewZipFile(resolver, version.aff4v10, self.filename_urn) as zip_file: 51 | self.volume_urn = zip_file.urn 52 | segment_urn = self.volume_urn.Append(self.segment_name) 53 | 54 | with zip_file.CreateMember(segment_urn) as segment: 55 | segment.Write(self.data1) 56 | 57 | with zip_file.CreateMember(segment_urn) as segment2: 58 | segment2.SeekWrite(0, 2) 59 | segment2.Write(self.data2) 60 | 61 | streamed_urn = self.volume_urn.Append(self.streamed_segment) 62 | with zip_file.CreateMember(streamed_urn) as streamed: 63 | streamed.compression_method = zip.ZIP_DEFLATE 64 | src = io.BytesIO(self.data1) 65 | streamed.WriteStream(src) 66 | 67 | def tearDown(self): 68 | try: 69 | os.unlink(self.filename) 70 | except (IOError, OSError): 71 | pass 72 | 73 | def testStreamedSegment(self): 74 | resolver = data_store.MemoryDataStore() 75 | 76 | # This is required in order to load and parse metadata from this volume 77 | # into a fresh empty resolver. 78 | with zip.ZipFile.NewZipFile(resolver, version.aff4v10, self.filename_urn) as zip_file: 79 | segment_urn = zip_file.urn.Append(self.streamed_segment) 80 | 81 | with resolver.AFF4FactoryOpen(segment_urn) as segment: 82 | self.assertEquals(segment.Read(1000), self.data1) 83 | 84 | def testOpenSegmentByURN(self): 85 | resolver = data_store.MemoryDataStore() 86 | 87 | # This is required in order to load and parse metadata from this volume 88 | # into a fresh empty resolver. 89 | with zip.ZipFile.NewZipFile(resolver, version.aff4v10, self.filename_urn) as zip_file: 90 | segment_urn = zip_file.urn.Append(self.segment_name) 91 | with resolver.AFF4FactoryOpen(segment_urn) as segment: 92 | self.assertEquals(segment.Read(1000), self.data1 + self.data2) 93 | 94 | def testSeekThrowsWhenWriting(self): 95 | resolver = data_store.MemoryDataStore() 96 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 97 | rdfvalue.XSDString("truncate")) 98 | 99 | with zip.ZipFile.NewZipFile(resolver, version.aff4v10, self.filename_urn) as zip_file: 100 | segment_urn = zip_file.urn.Append(self.streamed_segment) 101 | 102 | with zip_file.CreateMember(segment_urn) as segment: 103 | try: 104 | segment.SeekWrite(100,0) 105 | segment.Write("foo") 106 | hexdump.hexdump(segment.fd.getvalue()) 107 | segment.SeekWrite(0, 0) 108 | segment.Write("bar") 109 | hexdump.hexdump(segment.fd.getvalue()) 110 | self.fail("Seeking when writing not supported") 111 | except: 112 | pass 113 | 114 | 115 | if __name__ == '__main__': 116 | unittest.main() 117 | -------------------------------------------------------------------------------- /pyaff4/zip_test_extended.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2018-2019 Schatz Forensic Pty Ltd All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | # 16 | # Author: Bradley L Schatz bradley@evimetry.com 17 | 18 | from __future__ import unicode_literals 19 | import tempfile 20 | 21 | from future import standard_library 22 | standard_library.install_aliases() 23 | import os 24 | import unittest 25 | 26 | from pyaff4 import data_store, escaping 27 | from pyaff4 import lexicon 28 | from pyaff4 import rdfvalue 29 | from pyaff4 import zip 30 | from pyaff4.version import Version 31 | import traceback 32 | 33 | class ZipTest(unittest.TestCase): 34 | filename = tempfile.gettempdir() + u"/aff4_zip_extended_test.zip" 35 | filename_urn = rdfvalue.URN.FromFileName(filename) 36 | data1 = b"I am a plain old segment!!!" 37 | data2 = b"I am an overwritten segment" 38 | segment_name = "foo" 39 | 40 | #@unittest.skip 41 | def testRemoveDoesntRewindForNonLastSegment(self): 42 | try: 43 | os.unlink(self.filename) 44 | except (IOError, OSError): 45 | pass 46 | 47 | with data_store.MemoryDataStore() as resolver: 48 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 49 | rdfvalue.XSDString("truncate")) 50 | 51 | with zip.ZipFile.NewZipFile(resolver, Version(1, 1, "pyaff4"), self.filename_urn) as zip_container: 52 | self.volume_urn = zip_container.urn 53 | 54 | with zip_container.CreateZipSegment("foo") as segment: 55 | segment.Write(self.data1) 56 | segment.Flush() 57 | 58 | with zip_container.CreateZipSegment("bar") as segment: 59 | segment.Write(self.data1) 60 | segment.Flush() 61 | 62 | backing_store_urn = resolver.GetUnique(lexicon.transient_graph, self.volume_urn, lexicon.AFF4_STORED) 63 | with resolver.AFF4FactoryOpen(backing_store_urn) as backing_store: 64 | print() 65 | self.assertEquals(93, backing_store.writeptr) 66 | 67 | try: 68 | zip_container.RemoveSegment("foo") 69 | self.fail() 70 | except: 71 | pass 72 | 73 | self.assertEquals(687, os.stat(self.filename).st_size) 74 | 75 | #@unittest.skip 76 | def testEditInplaceZip(self): 77 | try: 78 | os.unlink(self.filename) 79 | except (IOError, OSError): 80 | pass 81 | 82 | with data_store.MemoryDataStore() as resolver: 83 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 84 | rdfvalue.XSDString("truncate")) 85 | 86 | with zip.ZipFile.NewZipFile(resolver, Version(1, 1, "pyaff4"), self.filename_urn) as zip_container: 87 | self.volume_urn = zip_container.urn 88 | 89 | with zip_container.CreateZipSegment("foo") as segment: 90 | segment.compression_method = zip.ZIP_STORED 91 | segment.Write(b'abcdefghijk') 92 | segment.Flush() 93 | 94 | with zip_container.CreateZipSegment("bar") as segment: 95 | segment.compression_method = zip.ZIP_STORED 96 | segment.Write(b'alkjflajdflaksjdadflkjd') 97 | segment.Flush() 98 | 99 | backing_store_urn = resolver.GetUnique(lexicon.transient_graph, self.volume_urn, lexicon.AFF4_STORED) 100 | with resolver.AFF4FactoryOpen(backing_store_urn) as backing_store: 101 | print() 102 | 103 | self.assertEquals(716, os.stat(self.filename).st_size) 104 | 105 | 106 | with data_store.MemoryDataStore() as resolver: 107 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 108 | rdfvalue.XSDString("random")) 109 | 110 | with zip.ZipFile.NewZipFile(resolver, Version(1, 1, "pyaff4"), self.filename_urn) as zip_file: 111 | self.volume_urn = zip_file.urn 112 | 113 | with zip_file.OpenZipSegment("foo") as segment: 114 | segment.SeekWrite(0,0) 115 | segment.Write(b'0000') 116 | 117 | self.assertEquals(716, os.stat(self.filename).st_size) 118 | 119 | #@unittest.skip 120 | def testRemoveDoesRewind(self): 121 | try: 122 | os.unlink(self.filename) 123 | except (IOError, OSError): 124 | pass 125 | 126 | with data_store.MemoryDataStore() as resolver: 127 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 128 | rdfvalue.XSDString("truncate")) 129 | 130 | with zip.ZipFile.NewZipFile(resolver, Version(1, 1, "pyaff4"), self.filename_urn) as zip_container: 131 | self.volume_urn = zip_container.urn 132 | 133 | with zip_container.CreateZipSegment("foo") as segment: 134 | segment.Write(self.data1) 135 | segment.Flush() 136 | 137 | with zip_container.CreateZipSegment("bar") as segment: 138 | segment.Write(self.data1) 139 | segment.Flush() 140 | 141 | backing_store_urn = resolver.GetUnique(lexicon.transient_graph, self.volume_urn, lexicon.AFF4_STORED) 142 | with resolver.AFF4FactoryOpen(backing_store_urn) as backing_store: 143 | print() 144 | self.assertEquals(93, backing_store.writeptr) 145 | 146 | zip_container.RemoveSegment("bar") 147 | 148 | with zip_container.CreateZipSegment("nar") as segment: 149 | segment.Write(self.data2) 150 | segment.Flush() 151 | 152 | with data_store.MemoryDataStore() as resolver: 153 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 154 | rdfvalue.XSDString("append")) 155 | 156 | with zip.ZipFile.NewZipFile(resolver, Version(1, 1, "pyaff4"), self.filename_urn) as zip_file: 157 | self.volume_urn = zip_file.urn 158 | segment_urn = self.volume_urn.Append(escaping.arnPathFragment_from_path(self.segment_name), 159 | quote=False) 160 | self.assertFalse(zip_file.ContainsSegment("bar")) 161 | self.assertTrue(zip_file.ContainsSegment("foo")) 162 | self.assertTrue(zip_file.ContainsSegment("nar")) 163 | 164 | with zip_file.OpenZipSegment("foo") as segment: 165 | self.assertEquals(self.data1, segment.Read(len(self.data1))) 166 | 167 | with zip_file.OpenZipSegment("nar") as segment: 168 | self.assertEquals(self.data2, segment.Read(len(self.data2))) 169 | 170 | self.assertEquals(736, os.stat(self.filename).st_size) 171 | 172 | #@unittest.skip 173 | def testRemoveIsEmpty(self): 174 | try: 175 | os.unlink(self.filename) 176 | except (IOError, OSError): 177 | pass 178 | 179 | with data_store.MemoryDataStore() as resolver: 180 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 181 | rdfvalue.XSDString("truncate")) 182 | 183 | with zip.ZipFile.NewZipFile(resolver, Version(1, 1, "pyaff4"), self.filename_urn) as zip_file: 184 | self.volume_urn = zip_file.urn 185 | segment_urn = self.volume_urn.Append(escaping.arnPathFragment_from_path(self.segment_name), quote=False) 186 | 187 | with zip_file.CreateMember(segment_urn) as segment: 188 | segment.Write(self.data1) 189 | segment.Flush() 190 | 191 | zip_file.RemoveMember(segment_urn) 192 | 193 | with data_store.MemoryDataStore() as resolver: 194 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 195 | rdfvalue.XSDString("append")) 196 | 197 | with zip.ZipFile.NewZipFile(resolver, Version(1, 1, "pyaff4"), self.filename_urn) as zip_file: 198 | self.volume_urn = zip_file.urn 199 | segment_urn = self.volume_urn.Append(escaping.arnPathFragment_from_path(self.segment_name), 200 | quote=False) 201 | self.assertFalse(zip_file.ContainsMember(segment_urn)) 202 | 203 | self.assertEquals(518, os.stat(self.filename).st_size) 204 | 205 | #@unittest.skip 206 | def testRemoveThenReAdd(self): 207 | try: 208 | os.unlink(self.filename) 209 | except (IOError, OSError): 210 | pass 211 | with data_store.MemoryDataStore() as resolver: 212 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 213 | rdfvalue.XSDString("truncate")) 214 | 215 | with zip.ZipFile.NewZipFile(resolver, Version(1, 1, "pyaff4"), self.filename_urn) as zip_file: 216 | self.volume_urn = zip_file.urn 217 | segment_urn = self.volume_urn.Append(escaping.arnPathFragment_from_path(self.segment_name), quote=False) 218 | 219 | with zip_file.CreateMember(segment_urn) as segment: 220 | segment.Write(self.data1) 221 | segment.Flush() 222 | 223 | zip_file.RemoveMember(segment_urn) 224 | 225 | with zip_file.CreateMember(segment_urn) as segment: 226 | segment.Write(self.data2) 227 | 228 | 229 | with data_store.MemoryDataStore() as resolver: 230 | with zip.ZipFile.NewZipFile(resolver, Version(1, 1, "pyaff4"), self.filename_urn) as zip_file: 231 | self.volume_urn = zip_file.urn 232 | segment_urn = self.volume_urn.Append(escaping.arnPathFragment_from_path(self.segment_name), 233 | quote=False) 234 | self.assertTrue(zip_file.ContainsMember(segment_urn)) 235 | 236 | with zip_file.OpenMember(segment_urn) as segment: 237 | self.assertEquals(self.data2, segment.Read(len(self.data2))) 238 | 239 | self.assertEquals(629, os.stat(self.filename).st_size) 240 | 241 | if __name__ == '__main__': 242 | unittest.main() 243 | -------------------------------------------------------------------------------- /pyaff4/zip_test_unicode.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright 2018 Schatz Forensic Pty Ltd. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | # use this file except in compliance with the License. You may obtain a copy of 7 | # the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations under 15 | # the License. 16 | 17 | from __future__ import unicode_literals 18 | from future import standard_library 19 | standard_library.install_aliases() 20 | import os 21 | import unittest 22 | 23 | from pyaff4 import data_store, escaping 24 | from pyaff4 import lexicon 25 | from pyaff4 import rdfvalue 26 | from pyaff4 import zip 27 | from pyaff4.version import Version 28 | import traceback, tempfile 29 | 30 | class ZipTest(unittest.TestCase): 31 | filename = tempfile.gettempdir() + "/aff4_test.zip" 32 | filename_urn = rdfvalue.URN.FromFileName(filename) 33 | segment_name = "/犬/ネコ.txt" 34 | unc_segment_name = "\\\\foo\\bar\\ネコ.txt" 35 | period_start_segment_name = "./foo/bar/foo.txt" 36 | data1 = b"I am a segment!" 37 | 38 | def setUp(self): 39 | with data_store.MemoryDataStore() as resolver: 40 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 41 | rdfvalue.XSDString("truncate")) 42 | 43 | with zip.ZipFile.NewZipFile(resolver, Version(1, 1, "pyaff4"), self.filename_urn) as zip_file: 44 | self.volume_urn = zip_file.urn 45 | segment_urn = self.volume_urn.Append(escaping.arnPathFragment_from_path(self.segment_name), quote=False) 46 | 47 | with zip_file.CreateMember(segment_urn) as segment: 48 | segment.Write(self.data1) 49 | 50 | unc_segment_urn = self.volume_urn.Append(escaping.arnPathFragment_from_path(self.unc_segment_name), quote=False) 51 | 52 | with zip_file.CreateMember(unc_segment_urn) as segment: 53 | segment.Write(self.data1) 54 | 55 | period_start_segment_urn = self.volume_urn.Append(self.period_start_segment_name, quote=False) 56 | 57 | with zip_file.CreateMember(period_start_segment_urn) as segment: 58 | segment.Write(self.data1) 59 | 60 | def tearDown(self): 61 | try: 62 | os.unlink(self.filename) 63 | except (IOError, OSError): 64 | pass 65 | 66 | 67 | def testOpenSegmentByURN(self): 68 | try: 69 | resolver = data_store.MemoryDataStore() 70 | 71 | # This is required in order to load and parse metadata from this volume 72 | # into a fresh empty resolver. 73 | with zip.ZipFile.NewZipFile(resolver, Version(1, 1, "pyaff4"), self.filename_urn) as zip_file: 74 | segment_urn = zip_file.urn.Append(escaping.arnPathFragment_from_path(self.segment_name), quote=False) 75 | unc_segment_urn = zip_file.urn.Append(escaping.arnPathFragment_from_path(self.unc_segment_name), quote=False) 76 | period_start_segment_urn = self.volume_urn.Append( 77 | escaping.arnPathFragment_from_path(self.period_start_segment_name), quote=False) 78 | 79 | with resolver.AFF4FactoryOpen(segment_urn) as segment: 80 | self.assertEquals(segment.Read(1000), self.data1 ) 81 | with resolver.AFF4FactoryOpen(unc_segment_urn) as segment: 82 | self.assertEquals(segment.Read(1000), self.data1 ) 83 | with resolver.AFF4FactoryOpen(unc_segment_urn) as segment: 84 | self.assertEquals(segment.Read(1000), self.data1) 85 | 86 | except Exception: 87 | traceback.print_exc() 88 | self.fail() 89 | 90 | if __name__ == '__main__': 91 | unittest.main() 92 | -------------------------------------------------------------------------------- /pyaff4/zip_test_unicode2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2019 Schatz Forensic Pty Ltd All rights reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 5 | # use this file except in compliance with the License. You may obtain a copy of 6 | # the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13 | # License for the specific language governing permissions and limitations under 14 | # the License. 15 | # 16 | # Author: Bradley L Schatz bradley@evimetry.com 17 | 18 | from __future__ import unicode_literals 19 | import tempfile 20 | 21 | from future import standard_library 22 | standard_library.install_aliases() 23 | import os 24 | import unittest 25 | 26 | from pyaff4 import data_store 27 | from pyaff4 import lexicon 28 | from pyaff4 import rdfvalue 29 | from pyaff4 import zip 30 | from pyaff4 import version 31 | 32 | 33 | class ZipTest(unittest.TestCase): 34 | filename = tempfile.gettempdir() + "/aff4_unicode2_test.zip" 35 | filename_urn = rdfvalue.URN.FromFileName(filename) 36 | segment_name = "\\犬\\ネコ.txt" 37 | data1 = b"I am a segment!" 38 | 39 | def setUp(self): 40 | with data_store.MemoryDataStore() as resolver: 41 | resolver.Set(lexicon.transient_graph, self.filename_urn, lexicon.AFF4_STREAM_WRITE_MODE, 42 | rdfvalue.XSDString("truncate")) 43 | 44 | with zip.ZipFile.NewZipFile(resolver, version.aff4v11, self.filename_urn) as zip_file: 45 | self.volume_urn = zip_file.urn 46 | 47 | with zip_file.CreateZipSegment(self.segment_name, arn=None) as segment: 48 | segment.Write(self.data1) 49 | 50 | 51 | def tearDown(self): 52 | try: 53 | os.unlink(self.filename) 54 | except (IOError, OSError): 55 | pass 56 | 57 | 58 | def testOpenSegmentByURN(self): 59 | resolver = data_store.MemoryDataStore() 60 | 61 | # This is required in order to load and parse metadata from this volume 62 | # into a fresh empty resolver. 63 | with zip.ZipFile.NewZipFile(resolver, version.aff4v11, self.filename_urn) as zip_file: 64 | segment_urn = zip_file.urn.Append(self.segment_name, quote=False) 65 | with resolver.AFF4FactoryOpen(segment_urn) as segment: 66 | self.assertEquals(segment.Read(1000), self.data1 ) 67 | 68 | if __name__ == '__main__': 69 | unittest.main() 70 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | future == 0.17.1 2 | aff4-snappy == 0.5.1 3 | rdflib[sparql] == 4.2.2 4 | intervaltree == 2.1.0 5 | pyyaml == 5.1 6 | tzlocal == 2.1 7 | html5lib == 1.0.1 8 | python-dateutil == 2.8.0 9 | fastchunking == 0.0.3 10 | hexdump 11 | pynacl 12 | pycryptodome 13 | pycryptoplus 14 | aes-keywrap 15 | passlib 16 | cryptography 17 | expiringdict 18 | lz4 19 | -------------------------------------------------------------------------------- /samples/extract_streams.py: -------------------------------------------------------------------------------- 1 | # This script demonstrates how to extract AFF4 streams from a volume. 2 | from pyaff4 import data_store 3 | from pyaff4 import aff4_image 4 | from pyaff4 import lexicon 5 | from pyaff4 import rdfvalue 6 | from pyaff4 import zip 7 | 8 | import re 9 | import sys 10 | 11 | # Convert a filename to URN. AFF4 uses URNs to refer to everything. 12 | volume_path_urn = rdfvalue.URN.NewURNFromFilename(sys.argv[1]) 13 | 14 | # We need to make a resolver to hold all the RDF metadata 15 | resolver = data_store.MemoryDataStore() 16 | 17 | # Open the AFF4 volume from a ZipFile. 18 | with zip.ZipFile.NewZipFile(resolver, volume_path_urn) as volume: 19 | volume_urn = volume.urn 20 | 21 | # This will dump out the resolver. 22 | resolver.Dump() 23 | 24 | # Find all subjects with a type of Image. Alternatively if you 25 | # know the subject URN in advance just open it. Replace the 26 | # AFF4_IMAGE_TYPE with AFF4_MAP_TYPE for maps. 27 | for subject in resolver.QueryPredicateObject( 28 | lexicon.AFF4_TYPE, lexicon.AFF4_IMAGE_TYPE): 29 | 30 | # This should be able to open the URN. 31 | with resolver.AFF4FactoryOpen(subject) as in_fd: 32 | 33 | # Escape the subject to make something like a valid filename. 34 | filename = re.sub("[^a-z0-9A-Z-]", 35 | lambda m: "%%%02x" % ord(m[0]), 36 | str(subject)) 37 | print ("Dumping %s to file %s" % (subject, filename)) 38 | 39 | with open(filename, "wb") as out_fd: 40 | 41 | # Just copy the output to the file. 42 | while 1: 43 | data = in_fd.read(1024 * 1024) 44 | if not data: 45 | break 46 | out_fd.write(data) 47 | -------------------------------------------------------------------------------- /samples/simple_block_read.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Schatz Forensic Pty Ltd. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | from pyaff4 import container 16 | import binascii 17 | import os 18 | import hexdump 19 | 20 | referenceImagesPath = os.path.join(os.path.dirname(__file__), u"..", u"test_images") 21 | stdLinear = os.path.join(referenceImagesPath, u"AFF4Std", u"Base-Linear.aff4") 22 | 23 | with container.Container.open(stdLinear) as mapStream: 24 | buf = mapStream.Read(4096) 25 | print(hexdump.hexdump(buf)) 26 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Google Inc. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not 4 | # use this file except in compliance with the License. You may obtain a copy of 5 | # the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 | # License for the specific language governing permissions and limitations under 13 | # the License. 14 | 15 | """This module installs the pyaff4 library.""" 16 | 17 | from setuptools import setup 18 | from setuptools.command.test import test as TestCommand 19 | 20 | try: 21 | with open('README.md') as file: 22 | long_description = file.read() 23 | except IOError: 24 | long_description = ( 25 | 'Advanced Forensic Format Version 4 (AFF4) Python module.') 26 | 27 | ENV = {"__file__": __file__} 28 | exec(open("pyaff4/_version.py").read(), ENV) 29 | VERSION = ENV["get_versions"]() 30 | 31 | with open('requirements.txt') as f: 32 | requirements = f.read().splitlines() 33 | 34 | class NoseTestCommand(TestCommand): 35 | def finalize_options(self): 36 | TestCommand.finalize_options(self) 37 | self.test_args = [] 38 | self.test_suite = True 39 | 40 | def run_tests(self): 41 | # Run nose ensuring that argv simulates running nosetests directly 42 | import nose 43 | nose.run_exit(argv=['nosetests']) 44 | 45 | 46 | commands = {} 47 | commands["test"] = NoseTestCommand 48 | 49 | setup( 50 | name='pyaff4', 51 | long_description=long_description, 52 | long_description_content_type="text/markdown", 53 | version=VERSION["pep440"], 54 | cmdclass=commands, 55 | description='Advanced Forensic Format Version 4 (AFF4) Python module.', 56 | author='Michael Cohen, Bradley Schatz', 57 | author_email='scudette@gmail.com, bradley@evimetry.com', 58 | url='https://www.aff4.org/', 59 | packages=['pyaff4'], 60 | package_dir={"pyaff4": "pyaff4"}, 61 | install_requires=requirements, 62 | extras_require=dict( 63 | cloud="google-api-python-client" 64 | ) 65 | ) 66 | -------------------------------------------------------------------------------- /test_images/AFF4-L/broken-dedupe.aff4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4-L/broken-dedupe.aff4 -------------------------------------------------------------------------------- /test_images/AFF4-L/dream.aff4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4-L/dream.aff4 -------------------------------------------------------------------------------- /test_images/AFF4-L/dream.aff4.information.turtle: -------------------------------------------------------------------------------- 1 | @prefix aff4: . 2 | @prefix rdf: . 3 | @prefix rdfs: . 4 | @prefix xml: . 5 | @prefix xsd: . 6 | 7 | a aff4:FileImage, 8 | aff4:Image, 9 | aff4:zip_segment ; 10 | aff4:birthTime "2018-09-17T13:42:20+10:00"^^xsd:datetime ; 11 | aff4:hash "75d83773f8d431a3ca91bfb8859e486d"^^aff4:MD5, 12 | "9ae1b46bead70c322eef7ac8bc36a8ea2055595c"^^aff4:SHA1 ; 13 | aff4:lastAccessed "2018-10-23T11:08:19+10:00"^^xsd:datetime ; 14 | aff4:lastWritten "2018-09-17T13:42:20+10:00"^^xsd:datetime ; 15 | aff4:originalFileName "./test_images/AFF4-L/dream.txt"^^xsd:string ; 16 | aff4:recordChanged "2018-09-17T13:42:20+10:00"^^xsd:datetime ; 17 | aff4:size 8688 ; 18 | aff4:stored . 19 | 20 | 21 | -------------------------------------------------------------------------------- /test_images/AFF4-L/dream.txt: -------------------------------------------------------------------------------- 1 | I have a Dream by Martin Luther King, Jr; August 28, 1963 2 | Delivered on the steps at the Lincoln Memorial in Washington D.C. on August 28, 1963 3 | 4 | Five score years ago, a great American, in whose symbolic shadow we stand signed the Emancipation Proclamation. This momentous decree came as a great beacon light of hope to millions of Negro slaves who had been seared in the flames of withering injustice. It came as a joyous daybreak to end the long night of captivity. 5 | 6 | But one hundred years later, we must face the tragic fact that the Negro is still not free. One hundred years later, the life of the Negro is still sadly crippled by the manacles of segregation and the chains of discrimination. One hundred years later, the Negro lives on a lonely island of poverty in the midst of a vast ocean of material prosperity. One hundred years later, the Negro is still languishing in the corners of American society and finds himself an exile in his own land. So we have come here today to dramatize an appalling condition. 7 | 8 | In a sense we have come to our nation's capital to cash a check. When the architects of our republic wrote the magnificent words of the Constitution and the declaration of Independence, they were signing a promissory note to which every American was to fall heir. This note was a promise that all men would be guaranteed the inalienable rights of life, liberty, and the pursuit of happiness. 9 | 10 | It is obvious today that America has defaulted on this promissory note insofar as her citizens of color are concerned. Instead of honoring this sacred obligation, America has given the Negro people a bad check which has come back marked "insufficient funds." But we refuse to believe that the bank of justice is bankrupt. We refuse to believe that there are insufficient funds in the great vaults of opportunity of this nation. So we have come to cash this check -- a check that will give us upon demand the riches of freedom and the security of justice. We have also come to this hallowed spot to remind America of the fierce urgency of now. This is no time to engage in the luxury of cooling off or to take the tranquilizing drug of gradualism. Now is the time to rise from the dark and desolate valley of segregation to the sunlit path of racial justice. Now is the time to open the doors of opportunity to all of God's children. Now is the time to lift our nation from the quicksands of racial injustice to the solid rock of brotherhood. 11 | 12 | It would be fatal for the nation to overlook the urgency of the moment and to underestimate the determination of the Negro. This sweltering summer of the Negro's legitimate discontent will not pass until there is an invigorating autumn of freedom and equality. Nineteen sixty-three is not an end, but a beginning. Those who hope that the Negro needed to blow off steam and will now be content will have a rude awakening if the nation returns to business as usual. There will be neither rest nor tranquility in America until the Negro is granted his citizenship rights. The whirlwinds of revolt will continue to shake the foundations of our nation until the bright day of justice emerges. 13 | 14 | But there is something that I must say to my people who stand on the warm threshold which leads into the palace of justice. In the process of gaining our rightful place we must not be guilty of wrongful deeds. Let us not seek to satisfy our thirst for freedom by drinking from the cup of bitterness and hatred. 15 | 16 | We must forever conduct our struggle on the high plane of dignity and discipline. We must not allow our creative protest to degenerate into physical violence. Again and again we must rise to the majestic heights of meeting physical force with soul force. The marvelous new militancy which has engulfed the Negro community must not lead us to distrust of all white people, for many of our white brothers, as evidenced by their presence here today, have come to realize that their destiny is tied up with our destiny and their freedom is inextricably bound to our freedom. We cannot walk alone. 17 | 18 | And as we walk, we must make the pledge that we shall march ahead. We cannot turn back. There are those who are asking the devotees of civil rights, "When will you be satisfied?" We can never be satisfied as long as our bodies, heavy with the fatigue of travel, cannot gain lodging in the motels of the highways and the hotels of the cities. We cannot be satisfied as long as the Negro's basic mobility is from a smaller ghetto to a larger one. We can never be satisfied as long as a Negro in Mississippi cannot vote and a Negro in New York believes he has nothing for which to vote. No, no, we are not satisfied, and we will not be satisfied until justice rolls down like waters and righteousness like a mighty stream. 19 | 20 | I am not unmindful that some of you have come here out of great trials and tribulations. Some of you have come fresh from narrow cells. Some of you have come from areas where your quest for freedom left you battered by the storms of persecution and staggered by the winds of police brutality. You have been the veterans of creative suffering. Continue to work with the faith that unearned suffering is redemptive. 21 | 22 | Go back to Mississippi, go back to Alabama, go back to Georgia, go back to Louisiana, go back to the slums and ghettos of our northern cities, knowing that somehow this situation can and will be changed. Let us not wallow in the valley of despair. 23 | 24 | I say to you today, my friends, that in spite of the difficulties and frustrations of the moment, I still have a dream. It is a dream deeply rooted in the American dream. 25 | 26 | I have a dream that one day this nation will rise up and live out the true meaning of its creed: "We hold these truths to be self-evident: that all men are created equal." 27 | 28 | I have a dream that one day on the red hills of Georgia the sons of former slaves and the sons of former slaveowners will be able to sit down together at a table of brotherhood. 29 | 30 | I have a dream that one day even the state of Mississippi, a desert state, sweltering with the heat of injustice and oppression, will be transformed into an oasis of freedom and justice. 31 | 32 | I have a dream that my four children will one day live in a nation where they will not be judged by the color of their skin but by the content of their character. 33 | 34 | I have a dream today. 35 | 36 | I have a dream that one day the state of Alabama, whose governor's lips are presently dripping with the words of interposition and nullification, will be transformed into a situation where little black boys and black girls will be able to join hands with little white boys and white girls and walk together as sisters and brothers. 37 | 38 | I have a dream today. 39 | 40 | I have a dream that one day every valley shall be exalted, every hill and mountain shall be made low, the rough places will be made plain, and the crooked places will be made straight, and the glory of the Lord shall be revealed, and all flesh shall see it together. 41 | 42 | This is our hope. This is the faith with which I return to the South. With this faith we will be able to hew out of the mountain of despair a stone of hope. With this faith we will be able to transform the jangling discords of our nation into a beautiful symphony of brotherhood. With this faith we will be able to work together, to pray together, to struggle together, to go to jail together, to stand up for freedom together, knowing that we will be free one day. 43 | 44 | This will be the day when all of God's children will be able to sing with a new meaning, "My country, 'tis of thee, sweet land of liberty, of thee I sing. Land where my fathers died, land of the pilgrim's pride, from every mountainside, let freedom ring." 45 | 46 | And if America is to be a great nation this must become true. So let freedom ring from the prodigious hilltops of New Hampshire. Let freedom ring from the mighty mountains of New York. Let freedom ring from the heightening Alleghenies of Pennsylvania! 47 | 48 | Let freedom ring from the snowcapped Rockies of Colorado! 49 | 50 | Let freedom ring from the curvaceous peaks of California! 51 | 52 | But not only that; let freedom ring from Stone Mountain of Georgia! 53 | 54 | Let freedom ring from Lookout Mountain of Tennessee! 55 | 56 | Let freedom ring from every hill and every molehill of Mississippi. From every mountainside, let freedom ring. 57 | 58 | When we let freedom ring, when we let it ring from every village and every hamlet, from every state and every city, we will be able to speed up that day when all of God's children, black men and white men, Jews and Gentiles, Protestants and Catholics, will be able to join hands and sing in the words of the old Negro spiritual, "Free at last! free at last! Thank God Almighty, we are free at last!" 59 | -------------------------------------------------------------------------------- /test_images/AFF4-L/information.turtle: -------------------------------------------------------------------------------- 1 | test -------------------------------------------------------------------------------- /test_images/AFF4-L/paper-hash_based_disk_imaging_using_aff4.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4-L/paper-hash_based_disk_imaging_using_aff4.pdf -------------------------------------------------------------------------------- /test_images/AFF4-L/paper-hash_based_disk_imaging_using_aff4.pdf.frag.1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4-L/paper-hash_based_disk_imaging_using_aff4.pdf.frag.1 -------------------------------------------------------------------------------- /test_images/AFF4-L/paper-hash_based_disk_imaging_using_aff4.pdf.frag.2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4-L/paper-hash_based_disk_imaging_using_aff4.pdf.frag.2 -------------------------------------------------------------------------------- /test_images/AFF4-L/unicode.aff4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4-L/unicode.aff4 -------------------------------------------------------------------------------- /test_images/AFF4-L/unicode.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4-L/unicode.zip -------------------------------------------------------------------------------- /test_images/AFF4-L/utf8segment-macos.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4-L/utf8segment-macos.zip -------------------------------------------------------------------------------- /test_images/AFF4-L/ネコ.txt: -------------------------------------------------------------------------------- 1 | foo 2 | -------------------------------------------------------------------------------- /test_images/AFF4PreStd/Base-Allocated.af4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4PreStd/Base-Allocated.af4 -------------------------------------------------------------------------------- /test_images/AFF4PreStd/Base-Linear-ReadError.af4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4PreStd/Base-Linear-ReadError.af4 -------------------------------------------------------------------------------- /test_images/AFF4PreStd/Base-Linear.af4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4PreStd/Base-Linear.af4 -------------------------------------------------------------------------------- /test_images/AFF4PreStd/README.txt: -------------------------------------------------------------------------------- 1 | Evimetry Pre Standard Images 2 | 3 | These images are produced using the Evimetry 2.0/2.1 AFF4 implementation. 4 | 5 | Base-Linear.af4 : Linear image of the entire source device. 6 | Base-Linear-ReadError.af4 : Linear image of an entire source device, where a read error occurred. 7 | Base-Allocated.af4 : Linear image of volume & FS metadata, and all allocated blocks in the filesytem. 8 | -------------------------------------------------------------------------------- /test_images/AFF4Std/Base-Allocated.aff4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4Std/Base-Allocated.aff4 -------------------------------------------------------------------------------- /test_images/AFF4Std/Base-Linear-AllHashes.aff4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4Std/Base-Linear-AllHashes.aff4 -------------------------------------------------------------------------------- /test_images/AFF4Std/Base-Linear-ReadError.aff4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4Std/Base-Linear-ReadError.aff4 -------------------------------------------------------------------------------- /test_images/AFF4Std/Base-Linear.aff4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4Std/Base-Linear.aff4 -------------------------------------------------------------------------------- /test_images/AFF4Std/README.txt: -------------------------------------------------------------------------------- 1 | Evimetry Standard v1.0 Images 2 | 3 | These images are produced using the Evimetry 2.2 AFF4 implementation. 4 | 5 | Base-Linear.aff4 : Linear image of the entire source device. 6 | Base-Linear-AllHashes.aff4 : Linear image of the entire source device, using (MD5, SHA1, SHA256, SHA512, Blake2b) blockhashes 7 | Base-Linear-ReadError.aff4 : Linear image of an entire source device, where a read error occurred. 8 | Base-Allocated.aff4 : Linear image of volume & FS metadata, and all allocated blocks in the filesytem. 9 | SpannedImages\ 10 | Base-Linear_1.aff4 : Linear image of the entire source device (striped). 11 | Base-Linear_2.aff4 : Linear image of the entire source device (striped). 12 | -------------------------------------------------------------------------------- /test_images/AFF4Std/Striped/Base-Linear_1.aff4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4Std/Striped/Base-Linear_1.aff4 -------------------------------------------------------------------------------- /test_images/AFF4Std/Striped/Base-Linear_2.aff4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aff4/pyaff4/94a3583475c07ad92147f70ff8a19e9e36f12aa9/test_images/AFF4Std/Striped/Base-Linear_2.aff4 -------------------------------------------------------------------------------- /test_images/README.md: -------------------------------------------------------------------------------- 1 | #AFF4 Canonical Reference Images v1.0-RC1 2 | 3 | The reference images provided in this project represent ground truth for the AFF4. 4 | 5 | AFF4Std: This folder contains the AFF4 Canonical Reference Images v1.0 (produced by Evimetry 3.0) 6 | 7 | AFF4PreStd: This folder contains reference images produced prior to the definition of the AFF4 Standard v1.0 (Evimetry 2.x) 8 | 9 | AFF4-L: This folder contains a number of files for AFF4 Logical unit testing. 10 | 11 | For an explanation of the images see the README under the folders. 12 | More detail on the standard can be found at https://github.com/aff4/Standard . 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /test_images/keys/certificate.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICmjCCAYICCQCDBqFau6EdljANBgkqhkiG9w0BAQsFADAPMQ0wCwYDVQQKDARB 3 | Q01FMB4XDTE5MTAyMzIzMjkzMVoXDTIwMTAyMjIzMjkzMVowDzENMAsGA1UECgwE 4 | QUNNRTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALmySMKJrbyDhyxy 5 | h7QCGpWg5xa5NssSZdeM9jbQquTZpfE6bFk3/sfM4eCzq+YtmyA7F3gHtS+bmws9 6 | udX/zXU/OWqMu99VGWWYO1kyhuR8Gw0SlpENtVev5o96I0QXFznBlWKeBZdfsaCt 7 | AcxvDPJo6d0475q7FSC9evfthE0hXseSE0hhLMdgnRyKS6VP3tB/W8OuGqblxnOX 8 | ZzqcmiXOca2uQm5OqwVAQLaEaX3hK+9O/AHAlZGy94SpcxwRXFpk9EZHRmc2Zc/3 9 | enx3YhgSKSpvYMyIPMg9sLpVFj++c89K9hMcwn+U1AU51D/wBw0ljYP7uY6X1TdL 10 | cPFW26UCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAo0T3EiKXPCSONSvuK5wv2RZ4 11 | B/qs/RxopXQa1bk6BfTjSSKXDZVSVbeiinQdWL9J65W+P93U+kH+YOhp3d7jyQ8j 12 | lDM7/bzJ+pV1xJuDirFhoXEdR4zWn4gZhTy8jyMjNFD27JTDV95hf2SBwgFGcFY9 13 | 1MMI9bxdyzJ37XhMU3sZDlQJ6mJ50Sk/FSEfXPH2naNdS2wOkrtDd0YithyW3hXe 14 | tp4+ASIKT69YW29ppm5VHnT0/MU0ITBepI0vKXFFLGHSbiwRmzBUJADxPciG+qks 15 | oDj8ATZY4dt0+PJ7ejn8UF5E3jqIguO1oL6fRFsMYtGOFxEFDb394rFE4Vtncg== 16 | -----END CERTIFICATE----- 17 | -------------------------------------------------------------------------------- /test_images/keys/key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC5skjCia28g4cs 3 | coe0AhqVoOcWuTbLEmXXjPY20Krk2aXxOmxZN/7HzOHgs6vmLZsgOxd4B7Uvm5sL 4 | PbnV/811PzlqjLvfVRllmDtZMobkfBsNEpaRDbVXr+aPeiNEFxc5wZVingWXX7Gg 5 | rQHMbwzyaOndOO+auxUgvXr37YRNIV7HkhNIYSzHYJ0cikulT97Qf1vDrhqm5cZz 6 | l2c6nJolznGtrkJuTqsFQEC2hGl94SvvTvwBwJWRsveEqXMcEVxaZPRGR0ZnNmXP 7 | 93p8d2IYEikqb2DMiDzIPbC6VRY/vnPPSvYTHMJ/lNQFOdQ/8AcNJY2D+7mOl9U3 8 | S3DxVtulAgMBAAECggEAfcBzL3KG+ftkJcBlj4xBLoTpGWVR6tFJsF/gOJy0rgeK 9 | LcLvrreRzQI9EKr7xQOrYndB3GHz3AqXQ1SIrZfuLfEj07j8XTBG45rkzfG+kapN 10 | s1ucJRzQalJPY2hFy42Lh+xFWqRCixEdu/6NEkE91kcf4FT3RaRdmW9Xf+AT0RqY 11 | X+q4yI0taVHUMMVQ4FJbUEh7KCLo3TROoFdQ2u8zKgEQK7fGYnmzLygchp0gqTG9 12 | HNGxvq8sk+ptL6vUTq6aR5+S+s+4TUpHFG1nol0GwJEF9aMOcuzjd4F7xhZ7GOd2 13 | TYwHQq5YoYL+2rxRUMqgHlvDq7Sw0T1qNfjchoPJIQKBgQDwPsR23VSEi9SyAaCS 14 | XaAfRBF4VKxUa8Nl+t45kUV/oorWcdVTPmsudAb03veVE4qVrxEbaw1+SV+w5FTT 15 | p4+ic4BSuV1PJlgVoXwfa6royKX9XnqV6hMgDP0Gjygm6Z6R7anuaNaZfJC1qSld 16 | RieGr4VPCQ/MKXDqIHN84JgdDQKBgQDF38DC2DUt4TV3h4I0Cqs18wwdQyr/+eOI 17 | bKUe/0fl3uQkHvpKmzqs5+mzxRUM//m3caRYwwpbZ6QZjAa6kBdJATbh8d2TNAxN 18 | G9p6tKoocSOgcaMaL7catdYlmwagguN56K0HazTajByjImdwGW8QwxQNf9b9y2vl 19 | 5UVcxKqC+QKBgH+mejFGLNA4lAz4/F6IzLmQK1AqfV5l2+7luwoPfEPzE54Z4eVX 20 | Nw/5qnCVwCs/tNUPriFJYmQFjIiq6b+EOrdwc3CA+WlC0G693Pu885S4eCoF91CM 21 | jRjsjczcZ9M1MoILK813ev8WxrUfatEao4nW3Rg/kltCcbKVB4gEtg5JAoGAFdtE 22 | SIFnRl0U8dIHAMaY6Mmi8eyEaGfqyRvvFUBvaaW4H4FIdks1LPok51WCoL/5jljA 23 | JYnNsBD/oE7GQ389AzReIpD7v5HFNhM4i8s+7F6q58MFmBPjLtEdCWRZVAuBIusf 24 | ia0+1lEZrK9VK52zle9mXKNdVQVOMsOjaL7UcskCgYBPfkZgig+UiGzGYzkfLH/M 25 | tpD0rsnDYKuBGt0DfRPI+ZxPRHPnV12ok1g9KLnnY7HNhDYg0GmgMhW1kqZZKfIC 26 | o8u4Y2Lk3phxVQz0VW18BCI1TehRzf/Cw6FEvy6hlOWkixdpsJTTkASFqEvxx68S 27 | pstsJENgN6H0GsdozLjz6w== 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | """Global version file. 4 | 5 | This program is used to manage versions. Prior to each release, please run it 6 | with update. 7 | """ 8 | from __future__ import print_function 9 | import arrow 10 | import argparse 11 | import json 12 | import os 13 | import yaml 14 | 15 | _VERSION_CODE = ''' 16 | import json 17 | import os 18 | import subprocess 19 | 20 | try: 21 | # We are looking for the git repo which contains this file. 22 | MY_DIR = os.path.dirname(os.path.abspath(__file__)) 23 | except: 24 | MY_DIR = None 25 | 26 | def is_tree_dirty(): 27 | try: 28 | return bool(subprocess.check_output( 29 | ["git", "diff", "--name-only"], stderr=subprocess.PIPE, 30 | cwd=MY_DIR, 31 | ).splitlines()) 32 | except (OSError, subprocess.CalledProcessError): 33 | return False 34 | 35 | def get_version_file_path(version_file="version.yaml"): 36 | try: 37 | return os.path.join(subprocess.check_output( 38 | ["git", "rev-parse", "--show-toplevel"], stderr=subprocess.PIPE, 39 | cwd=MY_DIR, 40 | ).decode("utf-8").strip(), version_file) 41 | except (OSError, subprocess.CalledProcessError): 42 | return None 43 | 44 | def number_of_commit_since(version_file="version.yaml"): 45 | """Returns the number of commits since version.yaml was changed.""" 46 | try: 47 | last_commit_to_touch_version_file = subprocess.check_output( 48 | ["git", "log", "--no-merges", "-n", "1", "--pretty=format:%H", 49 | version_file], cwd=MY_DIR, stderr=subprocess.PIPE, 50 | ).strip() 51 | 52 | all_commits = subprocess.check_output( 53 | ["git", "log", "--no-merges", "-n", "1000", "--pretty=format:%H"], 54 | stderr=subprocess.PIPE, cwd=MY_DIR, 55 | ).splitlines() 56 | return all_commits.index(last_commit_to_touch_version_file) 57 | except (OSError, subprocess.CalledProcessError, ValueError): 58 | return None 59 | 60 | 61 | def get_current_git_hash(): 62 | try: 63 | return subprocess.check_output( 64 | ["git", "log", "--no-merges", "-n", "1", "--pretty=format:%H"], 65 | stderr=subprocess.PIPE, cwd=MY_DIR, 66 | ).strip() 67 | except (OSError, subprocess.CalledProcessError): 68 | return None 69 | 70 | def tag_version_data(version_data, version_path="version.yaml"): 71 | current_hash = get_current_git_hash() 72 | # Not in a git repository. 73 | if current_hash is None: 74 | version_data["error"] = "Not in a git repository." 75 | 76 | else: 77 | version_data["revisionid"] = current_hash 78 | version_data["dirty"] = is_tree_dirty() 79 | version_data["dev"] = number_of_commit_since( 80 | get_version_file_path(version_path)) 81 | 82 | # Format the version according to pep440: 83 | pep440 = version_data["version"] 84 | if int(version_data.get("post", 0)) > 0: 85 | pep440 += ".post" + version_data["post"] 86 | 87 | elif int(version_data.get("rc", 0)) > 0: 88 | pep440 += ".rc" + version_data["rc"] 89 | 90 | if version_data.get("dev", 0): 91 | # A Development release comes _before_ the main release. 92 | last = version_data["version"].rsplit(".", 1) 93 | version_data["version"] = "%s.%s" % (last[0], int(last[1]) + 1) 94 | pep440 = version_data["version"] + ".dev" + str(version_data["dev"]) 95 | 96 | version_data["pep440"] = pep440 97 | 98 | return version_data 99 | ''' 100 | 101 | ENV = {"__file__": __file__} 102 | exec(_VERSION_CODE, ENV) 103 | is_tree_dirty = ENV["is_tree_dirty"] 104 | number_of_commit_since = ENV["number_of_commit_since"] 105 | get_current_git_hash = ENV["get_current_git_hash"] 106 | tag_version_data = ENV["tag_version_data"] 107 | 108 | 109 | _VERSION_TEMPLATE = """ 110 | # Machine Generated - do not edit! 111 | 112 | # This file is produced when the main "version.py update" command is run. That 113 | # command copies this file to all sub-packages which contain 114 | # setup.py. Configuration is maintain in version.yaml at the project's top 115 | # level. 116 | 117 | def get_versions(): 118 | return tag_version_data(raw_versions(), \"\"\"%s\"\"\") 119 | 120 | def raw_versions(): 121 | return json.loads(\"\"\" 122 | %s 123 | \"\"\") 124 | """ 125 | 126 | def get_config_file(version_file="version.yaml"): 127 | version_path = os.path.join(os.path.dirname( 128 | os.path.abspath(__file__)), version_file) 129 | 130 | return yaml.load(open(version_path, "rt").read(), Loader=yaml.SafeLoader), version_path 131 | 132 | 133 | def get_versions(version_file="version.yaml"): 134 | result, version_path = get_config_file(version_file) 135 | version_data = result["version_data"] 136 | 137 | return tag_version_data(version_data), version_path 138 | 139 | def escape_string(instr): 140 | return instr.replace('"""', r'\"\"\"') 141 | 142 | TEMPLATES = [] 143 | 144 | 145 | def update_templates(version_data): 146 | version_data["debian_ts"] = arrow.utcnow().format( 147 | 'ddd, D MMM YYYY h:mm:ss Z') 148 | for path in TEMPLATES: 149 | if not path.endswith(".in"): 150 | continue 151 | 152 | target = path[:-3] 153 | with open(target, "wt") as outfd: 154 | outfd.write(open(path, "rt").read() % version_data) 155 | 156 | 157 | def update_version_files(args): 158 | data, version_path = get_config_file(args.version_file) 159 | version_data = data["version_data"] 160 | if args.version: 161 | version_data["version"] = args.version 162 | 163 | if args.post: 164 | version_data["post"] = args.post 165 | 166 | if args.rc: 167 | version_data["rc"] = args.rc 168 | 169 | if args.codename: 170 | version_data["codename"] = args.codename 171 | 172 | # Write the updated version_data into the file. 173 | with open(version_path, "wt") as fd: 174 | fd.write(yaml.safe_dump(data, default_flow_style=False)) 175 | 176 | # Should not happen but just in case... 177 | contents = _VERSION_TEMPLATE % ( 178 | escape_string(args.version_file), 179 | escape_string(json.dumps(version_data, indent=4))) + _VERSION_CODE 180 | 181 | # Now copy the static version files to all locations. 182 | for path in data["dependent_versions"]: 183 | current_dir = os.path.abspath(os.path.dirname( 184 | os.path.abspath(__file__))) 185 | version_path = os.path.abspath(os.path.join(current_dir, path)) 186 | if not os.path.relpath(version_path, current_dir): 187 | raise TypeError("Dependent version path is outside tree.") 188 | 189 | with open(version_path, "wt") as fd: 190 | fd.write(contents) 191 | 192 | update_templates(version_data) 193 | 194 | 195 | def update(args): 196 | if (args.version is None and 197 | args.post is None and 198 | args.rc is None and 199 | args.codename is None): 200 | raise AttributeError("You must set something in this release.") 201 | 202 | update_version_files(args) 203 | 204 | 205 | def main(): 206 | parser = argparse.ArgumentParser() 207 | parser.add_argument( 208 | "--version_file", default="version.yaml", 209 | help="Version configuration file.") 210 | 211 | subparsers = parser.add_subparsers(help='sub-command help', dest='command') 212 | update_parser = subparsers.add_parser("update", help="Update the version") 213 | 214 | update_parser.add_argument( 215 | "--version", help="Set to this new version.") 216 | 217 | update_parser.add_argument( 218 | "--post", help="Set to this new post release.") 219 | 220 | update_parser.add_argument( 221 | "--rc", help="Set to this new release candidate.") 222 | 223 | update_parser.add_argument( 224 | "--codename", help="Set to this new codename.") 225 | 226 | 227 | subparsers.add_parser("version", help="Report the current version.") 228 | 229 | args = parser.parse_args() 230 | if args.command == "update": 231 | update(args) 232 | 233 | elif args.command == "version": 234 | version_data, version_path = get_versions(args.version_file) 235 | print("Scanning %s:\n%s" % (version_path, version_data)) 236 | 237 | 238 | if __name__ == "__main__": 239 | main() 240 | -------------------------------------------------------------------------------- /version.yaml: -------------------------------------------------------------------------------- 1 | dependent_versions: 2 | - _version.py 3 | - pyaff4/_version.py 4 | version_data: 5 | post: '0' 6 | rc: '0' 7 | version: '0.32' 8 | --------------------------------------------------------------------------------