├── requirements.txt
├── structs
    ├── constants.py
    ├── __init__.py
    ├── encryption_parameters.py
    ├── file_index.py
    ├── file.py
    └── file_entry.py
├── README.md
├── xp3reader.py
├── tests.py
├── xp3writer.py
└── xp3.py


/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy


--------------------------------------------------------------------------------
/structs/constants.py:
--------------------------------------------------------------------------------
1 | XP3Signature = b'XP3\x0D\x0A\x20\x0A\x1A\x8B\x67\x01'
2 | XP3FileIndexContinue = 0x80
3 | XP3FileIndexCompressed = 0x01
4 | Xp3FileIndexUncompressed = 0x00
5 | XP3FileIsEncrypted = 1 << 31
6 | 


--------------------------------------------------------------------------------
/structs/__init__.py:
--------------------------------------------------------------------------------
1 | from .constants import XP3Signature
2 | from .file import XP3File
3 | from .file_index import XP3FileIndex
4 | from .file_entry import XP3FileEntry, XP3FileEncryption, XP3FileTime, XP3FileAdler, XP3FileSegments, XP3FileInfo
5 | from .encryption_parameters import encryption_parameters
6 | 


--------------------------------------------------------------------------------
/structs/encryption_parameters.py:
--------------------------------------------------------------------------------
1 | encryption_parameters = {
2 |     # Master key, secondary key, XOR the first byte, segment name (for packing)
3 |     'none':            (0x00000000, 0x00, False, b'eliF'),
4 |     'neko_vol1':       (0x1548E29C, 0xD7, False, b'eliF'),
5 |     'neko_vol1_steam': (0x44528B87, 0x23, False, b'eliF'),
6 |     'neko_vol0':       (0x1548E29C, 0xD7, True,  b'neko'),
7 |     'neko_vol0_steam': (0x44528B87, 0x23, True,  b'neko')
8 | }
9 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | KiriKiri .XP3 archive repacking and extraction tool
 2 | 
 3 | Extracts an .XP3 archive to a directory of files, and
 4 | packs a directory of files into an .XP3 archive, including any
 5 | subdirectory structure.
 6 | 
 7 | Uses Numpy (if available) to speed up decryption/encryption.
 8 | 
 9 | Original script by Edward Keyes, ed-at-insani-dot-org, 2006-07-08, http://www.insani.org/tools/
10 | 
11 | and SmilingWolf, https://bitbucket.org/SmilingWolf/xp3tools-updated
12 | 
13 | Python 3 rewrite, Awakening


--------------------------------------------------------------------------------
/xp3reader.py:
--------------------------------------------------------------------------------
 1 | from io import BytesIO
 2 | from structs import XP3Signature, XP3FileIndex, XP3File
 3 | 
 4 | 
 5 | class XP3Reader:
 6 |     def __init__(self, buffer, silent: bool = False, use_numpy: bool = True):
 7 |         if isinstance(buffer, bytes):
 8 |             buffer = BytesIO(buffer)
 9 | 
10 |         self.buffer = buffer
11 |         self.silent = silent
12 |         self.use_numpy = use_numpy
13 | 
14 |         if XP3Signature != self.buffer.read(len(XP3Signature)):
15 |             raise AssertionError('Is not an XP3 file')
16 | 
17 |         if not silent:
18 |             print('Reading the file index', end='')
19 |         self.file_index = XP3FileIndex.read_from(self.buffer)
20 |         if not silent:
21 |             print(', found {} file(s)'.format(len(self.file_index.entries)))
22 | 
23 |     def close(self):
24 |         self.buffer.close()
25 | 
26 |     def __enter__(self):
27 |         return self
28 | 
29 |     def __exit__(self, exc_type, exc_val, exc_tb):
30 |         self.close()
31 | 
32 |     @property
33 |     def is_encrypted(self):
34 |         for file in self:
35 |             if file.is_encrypted:
36 |                 return True
37 |         return False
38 | 
39 |     # File access
40 | 
41 |     def __getitem__(self, item):
42 |         """Access a file by it's internal file path or position in file index"""
43 |         return XP3File(self.file_index[item], self.buffer, self.silent, self.use_numpy)
44 | 
45 |     def open(self, item):
46 |         return self.__getitem__(item)
47 | 


--------------------------------------------------------------------------------
/structs/file_index.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import zlib
  3 | import struct
  4 | from .file_entry import XP3FileEntry
  5 | from io import BytesIO
  6 | from .constants import XP3Signature, XP3FileIndexContinue, XP3FileIndexCompressed, Xp3FileIndexUncompressed
  7 | 
  8 | 
  9 | class peek:
 10 |     """
 11 |         Context manager, goes to position in the buffer and goes back
 12 |         Usage example::
 13 |             buffer.tell() # 0
 14 |             with peek(buffer, position) as peek_buffer:
 15 |                 peek_buffer.read(4)
 16 |             buffer.tell() # 0
 17 |     """
 18 | 
 19 |     def __init__(self, input_buffer, position):
 20 |         self.input_buffer = input_buffer
 21 |         self.position = position
 22 | 
 23 |     def __enter__(self):
 24 |         self.initial_position = self.input_buffer.tell()
 25 |         self.input_buffer.seek(self.position)
 26 |         return self.input_buffer
 27 | 
 28 |     def __exit__(self, exc_type, exc_val, exc_tb):
 29 |         self.input_buffer.seek(self.initial_position)
 30 | 
 31 | 
 32 | class XP3FileIndex:
 33 |     def __init__(self, entries: list, buffer=None):
 34 |         self.entries = entries
 35 |         self.path_index = {entry.file_path: index for index, entry in enumerate(entries)}
 36 |         self.buffer = buffer
 37 | 
 38 |     @classmethod
 39 |     def from_entries(cls, entries: list, buffer=None):
 40 |         return cls(entries, buffer)
 41 | 
 42 |     @classmethod
 43 |     def read_from(cls, buffer):
 44 |         """Constructor to instantiate class from buffer"""
 45 |         index = cls.read_index(buffer)
 46 |         entries = []
 47 |         with BytesIO(index) as index_buffer:
 48 |             while index_buffer.tell() < len(index):
 49 |                 entries.append(XP3FileEntry.read_from(index_buffer))
 50 | 
 51 |         return cls.from_entries(entries, buffer)
 52 | 
 53 |     def extract(self, to=''):
 54 |         """Dump file index from buffer"""
 55 | 
 56 |         with peek(self.buffer, len(XP3Signature)):
 57 |             index = self.read_index(self.buffer)
 58 | 
 59 |         dirname = os.path.dirname(to)
 60 |         if not os.path.exists(dirname):
 61 |             os.makedirs(dirname)
 62 | 
 63 |         with open(to, 'wb') as out:
 64 |             out.write(index)
 65 | 
 66 |         return self
 67 | 
 68 |     @staticmethod
 69 |     def read_index(buffer):
 70 |         """Reads file index from buffer"""
 71 |         offset, = struct.unpack('<1Q', buffer.read(8))
 72 |         if not offset:
 73 |             raise AssertionError('File index offset is missing')
 74 |         buffer.seek(offset, 0)
 75 | 
 76 |         flag, = struct.unpack('<B', buffer.read(1))
 77 |         if flag == XP3FileIndexContinue:  # Index is in another castle
 78 |             offset, = struct.unpack('<8x1Q', buffer.read(16))
 79 |             buffer.seek(offset, 0)
 80 |             flag, = struct.unpack('<B', buffer.read(1))
 81 | 
 82 |         if flag == XP3FileIndexCompressed:
 83 |             compressed_size, uncompressed_size = struct.unpack('<2Q', buffer.read(16))
 84 |             index = buffer.read(compressed_size)
 85 |             index = zlib.decompress(index)
 86 |             if len(index) != uncompressed_size:
 87 |                 raise AssertionError('Index size mismatch')
 88 |         elif flag == Xp3FileIndexUncompressed:
 89 |             uncompressed_size, = struct.unpack('<Q', buffer.read(8))
 90 |             index = buffer.read(uncompressed_size)
 91 |         else:
 92 |             raise AssertionError('Unexpected index flag {}'.format(flag))
 93 | 
 94 |         return index
 95 | 
 96 |     def to_bytes(self):
 97 |         uncompressed_index = b''.join([entry.to_bytes() for entry in self.entries])
 98 |         compressed_index = zlib.compress(uncompressed_index, level=9)
 99 |         uncompressed_size = len(uncompressed_index)
100 |         compressed_size = len(compressed_index)
101 |         if compressed_size + 1 + 8 + 8 < uncompressed_size + 1 + 8:  # Account for header overhead
102 |             return struct.pack('<BQQ', XP3FileIndexCompressed, compressed_size, uncompressed_size) + compressed_index
103 |         else:
104 |             return struct.pack('<BQ', Xp3FileIndexUncompressed, uncompressed_size) + uncompressed_index
105 | 
106 |     def __iter__(self):
107 |         yield from self.entries
108 | 
109 |     def __getitem__(self, item):
110 |         """Access file entries by index position or their file path"""
111 |         if isinstance(item, int):
112 |             return self.entries[item]
113 |         elif isinstance(item, str):
114 |             return self.entries[self.path_index[item]]
115 |         else:
116 |             raise TypeError
117 | 
118 |     def __repr__(self):
119 |         return "<XP3FileIndex, {} entry(ies)>".format(len(self.entries))
120 | 


--------------------------------------------------------------------------------
/tests.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import unittest
  3 | import datetime
  4 | import tempfile
  5 | from xp3 import XP3, XP3Reader, XP3Writer
  6 | 
  7 | 
  8 | class Encryption(unittest.TestCase):
  9 |     """Encryption test with Numpy and pure Python XORing"""
 10 | 
 11 |     def encrypt_and_decrypt(self, data, encryption_type, use_numpy):
 12 |         with XP3Writer(silent=True, use_numpy=use_numpy) as xp3:
 13 |             xp3.add('dummy_file', data, encryption_type)
 14 |             archive = xp3.pack_up()
 15 | 
 16 |         with XP3Reader(archive, silent=True, use_numpy=use_numpy) as xp3:
 17 |             self.assertTrue(xp3.is_encrypted)
 18 |             file = xp3.open('dummy_file')
 19 |             self.assertTrue(file.is_encrypted)
 20 |             self.assertEqual('dummy_file', file.file_path)
 21 |             self.assertEqual(data, file.read(encryption_type=encryption_type))
 22 | 
 23 |     def with_numpy(self, data):
 24 |         # Crash test early if Numpy is not present
 25 |         import numpy
 26 |         del numpy
 27 |         self.encrypt_and_decrypt(data, 'neko_vol0', True)
 28 | 
 29 |     def with_python(self, data):
 30 |         self.encrypt_and_decrypt(data, 'neko_vol0', False)
 31 | 
 32 |     def test_numpy_uncompressed(self):
 33 |         self.with_numpy(b'dummy_data')
 34 | 
 35 |     def test_numpy_compressed(self):
 36 |         self.with_numpy(b'111111111111')
 37 | 
 38 |     def test_python_uncompressed(self):
 39 |         self.with_python(b'dummy_data')
 40 | 
 41 |     def test_python_compressed(self):
 42 |         self.with_python(b'111111111111')
 43 | 
 44 | 
 45 | class FolderReadAndWrite(unittest.TestCase):
 46 |     """Read and write from and into file"""
 47 | 
 48 |     dummy_data = (
 49 |         ('dummyfile1', b'dummydata1'),
 50 |         ('dummyfile2', b'dummydata2')
 51 |     )
 52 | 
 53 |     def test(self):
 54 |         with tempfile.TemporaryDirectory() as datadir, tempfile.TemporaryDirectory() as xp3dir:
 55 |             for filename, filedata in self.dummy_data:
 56 |                 with open(os.path.join(datadir, filename), 'wb') as file:
 57 |                     file.write(filedata)
 58 | 
 59 |             xp3_path = os.path.join(xp3dir, 'data.xp3')
 60 |             with XP3(xp3_path, mode='w', silent=True) as xp3:
 61 |                 xp3.add_folder(datadir)
 62 | 
 63 |             with XP3(xp3_path, mode='r', silent=True) as xp3:
 64 |                 file1 = xp3.open('dummyfile1')
 65 |                 file2 = xp3.open('dummyfile2')
 66 |                 file1_name, file1_data = self.dummy_data[0]
 67 |                 file2_name, file2_data = self.dummy_data[1]
 68 |                 self.assertEqual(file1_name, file1.file_path)
 69 |                 self.assertEqual(file2_name, file2.file_path)
 70 |                 self.assertEqual(file1_data, file1.read())
 71 |                 self.assertEqual(file2_data, file2.read())
 72 |                 xp3.extract(os.path.join(xp3dir, 'out'))  # Extract the archive to folder
 73 |                 xp3.file_index.extract(os.path.join(xp3dir, 'data.xp3.index'))  # Dump file index
 74 | 
 75 |             # Check the extracted files
 76 |             for filename, filedata in self.dummy_data:
 77 |                 with open(os.path.join(xp3dir, 'out', filename), 'rb') as file:
 78 |                     self.assertEqual(filedata, file.read())
 79 | 
 80 | 
 81 | class MemoryReadAndWrite(unittest.TestCase):
 82 |     """In memory read and write"""
 83 |     dummy_data = (
 84 |         ('dummy_file_1', b'dummydata1', round(datetime.datetime.utcnow().timestamp() * 1000), False),  # convert timestamp to milliseconds
 85 |         ('dummy_file_2', b'dummydata2', 2000, False),
 86 |         ('dummy_file_3', b'111111111111', 3000, True),  # should be compressed
 87 |         ('dummy_file_4', b'11111111111', 3000, False)  # compressed and uncompressed sizes should match, do not compress
 88 |     )
 89 | 
 90 |     def test(self):
 91 |         with XP3Writer(silent=True) as xp3:
 92 |             for filepath, data, timestamp, compressed in self.dummy_data:
 93 |                 xp3.add(filepath, data, None, timestamp=timestamp)
 94 |             archive = xp3.pack_up()
 95 | 
 96 |         with XP3Reader(archive, silent=True) as xp3:
 97 | 
 98 |             for index, (filepath, data, timestamp, compressed) in enumerate(self.dummy_data):
 99 |                 file = xp3.open(index)
100 |                 self.assertEqual(filepath, file.file_path)
101 |                 self.assertEqual(data, file.read())
102 |                 self.assertEqual(timestamp // 1000, file.time.timestamp)
103 |                 self.assertEqual(compressed, file.segm.segments[0].is_compressed)
104 | 
105 | 
106 | class DuplicateWrite(unittest.TestCase):
107 |     """Make sure that duplicates can not be added into archive"""
108 | 
109 |     def test(self):
110 |         with XP3Writer(silent=True) as xp3:
111 |             xp3.add('duplicate_file', b'12345', None)
112 |             with self.assertRaises(FileExistsError):
113 |                 xp3.add('duplicate_file', b'12345', None)
114 | 
115 | 
116 | if __name__ == '__main__':
117 |     unittest.main()
118 | 


--------------------------------------------------------------------------------
/structs/file.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | from io import BytesIO
  3 | from array import array
  4 | import zlib
  5 | from .encryption_parameters import encryption_parameters
  6 | from .file_entry import XP3FileEntry
  7 | try:
  8 |     from numpy import frombuffer, uint8, bitwise_and, bitwise_xor, right_shift, concatenate
  9 |     numpy = True
 10 | except ModuleNotFoundError:
 11 |     numpy = False
 12 | 
 13 | 
 14 | class XP3DecryptionError(Exception):
 15 |     pass
 16 | 
 17 | 
 18 | class XP3File(XP3FileEntry):
 19 |     """Wrapper around file entry with buffer access to be able to read the file"""
 20 | 
 21 |     def __init__(self, index_entry: XP3FileEntry, buffer, silent, use_numpy):
 22 |         super(XP3File, self).__init__(
 23 |             encryption=index_entry.encryption,
 24 |             time=index_entry.time,
 25 |             adlr=index_entry.adlr,
 26 |             segm=index_entry.segm,
 27 |             info=index_entry.info
 28 |         )
 29 |         self.buffer = buffer
 30 |         self.silent = silent
 31 |         self.use_numpy = use_numpy
 32 | 
 33 |     def read(self, encryption_type='none', raw=False):
 34 |         """Reads the file from buffer and return it's data"""
 35 |         for segment in self.segm:
 36 |             self.buffer.seek(segment.offset)
 37 |             data = self.buffer.read(segment.compressed_size)
 38 | 
 39 |             if segment.is_compressed:
 40 |                 data = zlib.decompress(data)
 41 |             if len(data) != segment.uncompressed_size:
 42 |                 raise AssertionError(len(data), segment.uncompressed_size)
 43 | 
 44 |             if self.is_encrypted:
 45 |                 file_buffer = BytesIO(data)
 46 |                 if encryption_type in ('none', None) and not raw:
 47 |                     raise XP3DecryptionError('File is encrypted and no encryption type was specified')
 48 |                 self.xor(file_buffer, self.adler32, encryption_type, self.use_numpy)
 49 |                 data = file_buffer.getvalue()
 50 |                 file_buffer.close()
 51 |         return data
 52 | 
 53 |     def extract(self, to='', name=None, encryption_type='none', raw=False):
 54 |         """
 55 |         Reads the data and saves the file to specified folder,
 56 |         if no location is specified, unpacks into folder with archive name (data.xp3, unpacks into data folder)
 57 |         """
 58 |         file = self.read(encryption_type=encryption_type, raw=raw)
 59 |         if zlib.adler32(file) != self.adler32 and not self.silent:
 60 |             print('! Checksum error')
 61 | 
 62 |         if not to:
 63 |             # Use archive name as output folder if it's not explicitly specified
 64 |             basename = os.path.basename(self.buffer.name)
 65 |             to = os.path.splitext(basename)[0]
 66 |         if not name:
 67 |             name = self.file_path
 68 |         to = os.path.join(to, name)
 69 |         dirname = os.path.dirname(to)
 70 |         if not os.path.exists(dirname):
 71 |             os.makedirs(dirname)
 72 | 
 73 |         with open(to, 'wb') as output:
 74 |             output.write(file)
 75 | 
 76 |     @staticmethod
 77 |     def xor(output_buffer, adler32: int, encryption_type: str, use_numpy: bool = True):
 78 |         """XOR the data, uses numpy if available"""
 79 |         master_key, secondary_key, xor_the_first_byte, _ = encryption_parameters[encryption_type]
 80 |         # Read the encrypted data from buffer
 81 |         output_buffer.seek(0)
 82 |         data = output_buffer.read()
 83 | 
 84 |         # Use numpy if available
 85 |         if numpy and use_numpy:
 86 |             # Calculate the XOR key
 87 |             adler_key = bitwise_xor(adler32, master_key)
 88 |             xor_key = bitwise_and(bitwise_xor(bitwise_xor(bitwise_xor(right_shift(adler_key, 24), right_shift(adler_key, 16)), right_shift(adler_key, 8)), adler_key), 0xFF)
 89 |             if not xor_key:
 90 |                 xor_key = secondary_key
 91 | 
 92 |             data = frombuffer(data, dtype=uint8)
 93 | 
 94 |             if xor_the_first_byte:
 95 |                 first_byte_key = bitwise_and(adler_key, 0xFF)
 96 |                 if not first_byte_key:
 97 |                     first_byte_key = bitwise_and(master_key, 0xFF)
 98 |                 # Split the first byte into separate array
 99 |                 first = frombuffer(data[:1], dtype=uint8)
100 |                 rest = frombuffer(data[1:], dtype=uint8)
101 |                 # XOR the first byte
102 |                 first = bitwise_xor(first, first_byte_key)
103 |                 # Concatenate the array back
104 |                 data = concatenate((first, rest))
105 | 
106 |             data = bitwise_xor(data, xor_key)
107 |         else:
108 |             adler_key = adler32 ^ master_key
109 |             xor_key = (adler_key >> 24 ^ adler_key >> 16 ^ adler_key >> 8 ^ adler_key) & 0xFF
110 |             if not xor_key:
111 |                 xor_key = secondary_key
112 | 
113 |             data = array('B', data)
114 | 
115 |             if xor_the_first_byte:
116 |                 first_byte_key = adler_key & 0xFF
117 |                 if not first_byte_key:
118 |                     first_byte_key = master_key & 0xFF
119 |                 data[0] ^= first_byte_key
120 | 
121 |             # XOR the data
122 |             for index in range(len(data)):
123 |                 data[index] ^= xor_key
124 | 
125 |         # Overwrite the buffer with decrypted/encrypted data
126 |         output_buffer.seek(0)
127 |         output_buffer.write(data.tobytes())
128 | 


--------------------------------------------------------------------------------
/xp3writer.py:
--------------------------------------------------------------------------------
  1 | import zlib
  2 | import struct
  3 | import hashlib
  4 | from io import BytesIO
  5 | from structs import XP3FileIndex, XP3FileEncryption, XP3FileTime, XP3FileAdler, XP3FileSegments, XP3FileInfo, XP3File, \
  6 |     XP3FileEntry, XP3Signature, encryption_parameters
  7 | 
  8 | 
  9 | class XP3Writer:
 10 |     def __init__(self, buffer: BytesIO = None, silent: bool = False, use_numpy: bool = True):
 11 |         """
 12 |         :param buffer: Buffer object to write data to
 13 |         :param silent: Supress prints
 14 |         :param use_numpy: Use Numpy for XORing if available
 15 |         """
 16 |         if not buffer:
 17 |             buffer = BytesIO()
 18 |         self.buffer = buffer
 19 |         self.file_entries = []
 20 |         self.silent = silent
 21 |         self.use_numpy = use_numpy
 22 |         buffer.seek(0)
 23 |         buffer.write(XP3Signature)
 24 |         buffer.write(struct.pack('<Q', 0))  # File index offset placeholder
 25 |         self.packed_up = False
 26 |         self._filenames = []
 27 | 
 28 |     def __enter__(self):
 29 |         return self
 30 | 
 31 |     def __exit__(self, exc_type, exc_val, exc_tb):
 32 |         if not self.packed_up:
 33 |             self.pack_up()
 34 |         self.buffer.close()
 35 | 
 36 |     def add(self, internal_filepath: str, file: bytes, encryption_type: str = None, timestamp: int = 0):
 37 |         """
 38 |         Add a file to the archive
 39 |         :param internal_filepath: Internal file path
 40 |         :param file: File to add
 41 |         :param encryption_type: Encryption type to encrypt with
 42 |         :param timestamp: Timestamp (in milliseconds) to save
 43 |         """
 44 |         if self.packed_up:
 45 |             raise Exception('Archive is already packed up')
 46 |         if internal_filepath in self._filenames:
 47 |             raise FileExistsError
 48 | 
 49 |         self._filenames.append(internal_filepath)
 50 |         file_entry, file = self._create_file_entry(
 51 |             internal_filepath=internal_filepath,
 52 |             uncompressed_data=file,
 53 |             offset=self.buffer.tell(),
 54 |             encryption_type=encryption_type,
 55 |             timestamp=timestamp)
 56 |         self.file_entries.append(file_entry)
 57 |         if not self.silent:
 58 |             print('| Packing {} ({} -> {} bytes)'.format(internal_filepath,
 59 |                                                          file_entry.segm.uncompressed_size,
 60 |                                                          file_entry.segm.compressed_size))
 61 |         self.buffer.write(file)
 62 | 
 63 |     def pack_up(self) -> bytes:
 64 |         """
 65 |         Write the file index to the archive, returns the resulting archive if it can
 66 |         (if already packed, just returns the archive)
 67 |         """
 68 |         if self.packed_up:
 69 |             if hasattr(self.buffer, 'getvalue'):
 70 |                 return self.buffer.getvalue()
 71 | 
 72 |         # Write the file index
 73 |         file_index = XP3FileIndex.from_entries(self.file_entries).to_bytes()
 74 |         file_index_offset = self.buffer.tell()
 75 |         self.buffer.write(file_index)
 76 | 
 77 |         # Go back to the header and write the offset
 78 |         self.buffer.seek(len(XP3Signature))
 79 |         self.buffer.write(struct.pack('<Q', file_index_offset))
 80 | 
 81 |         # Mark as packed up and return the resulting archive
 82 |         self.packed_up = True
 83 |         # Flush the buffer explicitly, had a test fail because index was missing
 84 |         self.buffer.flush()
 85 |         if hasattr(self.buffer, 'getvalue'):
 86 |             return self.buffer.getvalue()
 87 | 
 88 |     def _create_file_entry(self, internal_filepath, uncompressed_data, offset, encryption_type: str = None,
 89 |                            timestamp: int = 0) -> (XP3FileEntry, bytes):
 90 |         """
 91 |         Create a file entry for a file
 92 |         :param internal_filepath: Internal file path
 93 |         :param uncompressed_data: File to create entry for
 94 |         :param offset: Position in the buffer to put into the segment data
 95 |         :param encryption_type: Encryption type to use
 96 |         :param timestamp Timestamp (in milliseconds)
 97 |         :return XP3FileEntry object and compressed or uncompressed file (to write into buffer)
 98 |         """
 99 | 
100 |         adlr = XP3FileAdler.from_data(uncompressed_data)
101 | 
102 |         is_encrypted = False if encryption_type in ('none', None) else True
103 |         if is_encrypted:
104 |             uncompressed_data = self.xor(uncompressed_data, adlr.value, encryption_type, self.use_numpy)
105 |             _, _, _, name = encryption_parameters[encryption_type]
106 |             encryption = XP3FileEncryption(adlr.value, internal_filepath, name)
107 |             path_hash = hashlib.md5(internal_filepath.lower().encode('utf-16le')).hexdigest()
108 |         else:
109 |             encryption = path_hash = None
110 | 
111 |         uncompressed_size = len(uncompressed_data)
112 |         compressed_data = zlib.compress(uncompressed_data, level=9)
113 |         compressed_size = len(compressed_data)
114 | 
115 |         if compressed_size >= uncompressed_size:
116 |             data = uncompressed_data
117 |             compressed_size = uncompressed_size
118 |             is_compressed = False
119 |         else:
120 |             data = compressed_data
121 |             is_compressed = True
122 | 
123 |         time = XP3FileTime(timestamp)
124 |         info = XP3FileInfo(is_encrypted=is_encrypted,
125 |                            uncompressed_size=uncompressed_size,
126 |                            compressed_size=compressed_size,
127 |                            file_path=internal_filepath if not is_encrypted else path_hash
128 |                            )
129 | 
130 |         segment = XP3FileSegments.segment(
131 |             is_compressed=is_compressed,
132 |             offset=offset,
133 |             uncompressed_size=uncompressed_size,
134 |             compressed_size=compressed_size
135 |         )
136 |         segm = XP3FileSegments([segment])
137 | 
138 |         file_entry = XP3FileEntry(encryption=encryption, time=time, adlr=adlr, segm=segm, info=info)
139 | 
140 |         return file_entry, data
141 | 
142 |     @staticmethod
143 |     def xor(data, adler32, encryption_type, use_numpy):
144 |         with BytesIO() as buffer:
145 |             buffer.write(data)
146 |             XP3File.xor(buffer, adler32, encryption_type, use_numpy)
147 |             return buffer.getvalue()
148 | 


--------------------------------------------------------------------------------
/xp3.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | # KiriKiri .XP3 archive repacking and extraction tool
  4 | #
  5 | # Extracts an .XP3 archive to a directory of files, and
  6 | # packs a directory of files into an .XP3 archive, including any
  7 | # subdirectory structure.
  8 | #
  9 | # Uses Numpy (if available) to speed up decryption/encryption.
 10 | #
 11 | # Original script by Edward Keyes, ed-at-insani-dot-org, 2006-07-08, http://www.insani.org/tools/
 12 | # and SmilingWolf, https://bitbucket.org/SmilingWolf/xp3tools-updated
 13 | # Python 3 rewrite, Awakening
 14 | 
 15 | 
 16 | import os
 17 | from xp3reader import XP3Reader
 18 | from xp3writer import XP3Writer
 19 | 
 20 | 
 21 | class XP3(XP3Reader, XP3Writer):
 22 |     def __init__(self, target, mode='r', silent=False):
 23 |         self.mode = mode
 24 | 
 25 |         if self._is_readmode:
 26 |             if isinstance(target, str):
 27 |                 if not os.path.isfile(target):
 28 |                     raise FileNotFoundError
 29 |                 target = open(target, 'rb')
 30 |             XP3Reader.__init__(self, target, silent)
 31 |         elif self._is_writemode:
 32 |             if isinstance(target, str):
 33 |                 dir = os.path.dirname(target)
 34 |                 if dir and not os.path.exists(dir):
 35 |                     os.makedirs(dir)
 36 |                 target = open(target, 'wb')
 37 |             XP3Writer.__init__(self, target, silent)
 38 |         else:
 39 |             raise ValueError('Invalid operation mode')
 40 | 
 41 |     @property
 42 |     def _is_readmode(self):
 43 |         return True if self.mode == 'r' else False
 44 | 
 45 |     @property
 46 |     def _is_writemode(self):
 47 |         return True if self.mode == 'w' else False
 48 | 
 49 |     def extract(self, to='', encryption_type='none'):
 50 |         """Extract all files in the archive to specified folder"""
 51 |         if not self._is_readmode:
 52 |             raise Exception('Archive is not open in reading mode')
 53 | 
 54 |         for file in self:
 55 |             try:
 56 |                 if not self.silent:
 57 |                     print('| Extracting {} ({} -> {} bytes)'.format(file.file_path,
 58 |                                                                     file.info.compressed_size,
 59 |                                                                     file.info.uncompressed_size))
 60 |                 file.extract(to=to, encryption_type=encryption_type)
 61 |             except OSError:  # Usually because of long file names
 62 |                 if not self.silent:
 63 |                     print('! Problem writing {}'.format(file.file_path))
 64 |         return self
 65 | 
 66 |     def add_folder(self, path, flatten: bool = False, encryption_type: str = None, save_timestamps: bool = False):
 67 |         if not self._is_writemode:
 68 |             raise Exception('Archive is not open in writing mode')
 69 |         if not self.silent:
 70 |             print('Packing {}'.format(path))
 71 |         for dirpath, dirs, filenames in os.walk(path):
 72 |             # Strip off the base directory and possible slash
 73 |             internal_root = dirpath[len(path) + 1:]
 74 |             # and make sure we're using forward slash as a separator
 75 |             internal_root = internal_root.split(os.sep)
 76 |             internal_root = '/'.join(internal_root)
 77 | 
 78 |             for filename in filenames:
 79 |                 internal_filepath = internal_root + '/' + filename \
 80 |                                     if internal_root and not flatten \
 81 |                                     else filename
 82 |                 self.add_file(os.path.join(dirpath, filename), internal_filepath, encryption_type, save_timestamps)
 83 | 
 84 |     def add_file(self, path, internal_filepath: str = None, encryption_type: str = None, save_timestamps: bool = False):
 85 |         """
 86 |         :param path: Path to file
 87 |         :param internal_filepath: Internal archive path to save file under (if not specified, file name is used)
 88 |         :param encryption_type: Encryption type to use
 89 |         :param save_timestamps: Save the file creating time into archive or not
 90 |         """
 91 |         if not self._is_writemode:
 92 |             raise Exception('Archive is not open in writing mode')
 93 | 
 94 |         if not os.path.exists(path):
 95 |             raise FileNotFoundError
 96 | 
 97 |         with open(path, 'rb') as buffer:
 98 |             data = buffer.read()
 99 |             if not internal_filepath:
100 |                 internal_filepath = os.path.basename(buffer.name)
101 | 
102 |         timestamp = 0 if not save_timestamps else round(os.path.getctime(path) * 1000)
103 |         super().add(internal_filepath, data, encryption_type, timestamp)
104 | 
105 |     def __exit__(self, exc_type, exc_val, exc_tb):
106 |         """Write the file index in the archive as we leave the context manager"""
107 |         if self._is_writemode:
108 |             if not self.packed_up:
109 |                 self.pack_up()
110 |             self.buffer.close()
111 | 
112 | 
113 | if __name__ == '__main__':
114 |     import argparse
115 |     from structs.encryption_parameters import encryption_parameters
116 | 
117 |     def input_filepath(path: str) -> str:
118 |         if not os.path.exists(os.path.realpath(path)):
119 |             raise argparse.ArgumentError
120 |         return path
121 | 
122 | 
123 |     parser = argparse.ArgumentParser(description='KiriKiri .XP3 archive repacking and extraction tool')
124 |     parser.add_argument('-mode', '-m', choices=['e', 'r', 'extract', 'repack'], default='e', help='Operation mode')
125 |     parser.add_argument('-silent', '-s', action='store_true', default=False)
126 |     parser.add_argument('-flatten', '-f', action='store_true', default=False,
127 |                         help='Ignore the subdirectories and pack the archive as if all files are in the root folder')
128 |     parser.add_argument('--dump-index', '-i', action='store_true', help='Dump the file index of an archive')
129 |     parser.add_argument('-encryption', '-e', choices=encryption_parameters.keys(), default='none',
130 |                         help='Specify the encryption method')
131 |     parser.add_argument('input', type=input_filepath, help='File to unpack or folder to pack')
132 |     parser.add_argument('output', help='Output folder to unpack into or output file to pack into')
133 |     args = parser.parse_args()
134 | 
135 |     if args.mode in ('e', 'extract'):
136 |         with XP3(args.input, 'r', args.silent) as xp3:
137 |             if args.dump_index:
138 |                 xp3.file_index.extract(args.output)
139 |             else:
140 |                 xp3.extract(args.output, args.encryption)
141 |     elif args.mode in ('r', 'repack'):
142 |         with XP3(args.output, 'w', args.silent) as xp3:
143 |             xp3.add_folder(args.input, args.flatten, args.encryption)
144 | 


--------------------------------------------------------------------------------
/structs/file_entry.py:
--------------------------------------------------------------------------------
  1 | import zlib
  2 | import struct
  3 | from io import BufferedReader
  4 | from collections import namedtuple
  5 | from datetime import datetime
  6 | from .constants import XP3FileIsEncrypted
  7 | 
  8 | 
  9 | class XP3FileEncryption:
 10 |     encryption_chunk = struct.Struct('<QIH')
 11 | 
 12 |     def __init__(self, adler32: int, file_path: str, name: bytes = b'eliF'):
 13 |         """
 14 |         :param adler32: Adler-32 checksum of a file
 15 |         :param file_path: Internal file path
 16 |         :param name: Chunk header
 17 |         """
 18 |         self.name = name
 19 |         self.adler32 = adler32
 20 |         self.file_path = file_path
 21 | 
 22 |     @classmethod
 23 |     def read_from(cls, buffer: BufferedReader, name: bytes = b'eliF'):
 24 |         start = buffer.tell() + 8
 25 |         size, adler32, file_path_length = cls.encryption_chunk.unpack(buffer.read(14))
 26 |         file_path, = struct.unpack('<' + (str(file_path_length * 2) + 's') + 'xx',
 27 |                                    buffer.read(2 + file_path_length * 2))
 28 |         file_path = file_path.decode('utf-16le')
 29 | 
 30 |         if buffer.tell() != start + size:  # chunk size doesn't include the size itself
 31 |             raise AssertionError('Buffer position {}, expected {}'.format(buffer.tell(),
 32 |                                                                           start + size))
 33 |         return cls(adler32, file_path, name)
 34 | 
 35 |     def to_bytes(self):
 36 |         size = 4 + 2 + (len(self.file_path) * 2) + 2
 37 |         data = struct.pack('<QIH', size, self.adler32, len(self.file_path))
 38 |         file_path = self.file_path.encode('utf-16le') + b'\x00\x00'
 39 |         return self.name + data + file_path
 40 | 
 41 |     def __repr__(self):
 42 |         return "<XP3FileEncryption adler32={}, file_path='{}'>".format(self.adler32, self.file_path)
 43 | 
 44 | 
 45 | class XP3FileTime:
 46 |     time_chunk = struct.Struct('<QQ')
 47 | 
 48 |     def __init__(self, timestamp: int = 0):
 49 |         self.timestamp = timestamp
 50 | 
 51 | 
 52 |     @classmethod
 53 |     def read_from(cls, buffer: BufferedReader):
 54 |         size, timestamp = cls.time_chunk.unpack(buffer.read(16))
 55 |         if size != 8:
 56 |             raise AssertionError
 57 |         return cls(timestamp // 1000)
 58 | 
 59 |     def to_bytes(self):
 60 |         return b'time' + self.time_chunk.pack(8, self.timestamp)
 61 | 
 62 | 
 63 | class XP3FileSegments:
 64 |     segment = namedtuple('Segment', 'is_compressed, offset, uncompressed_size, compressed_size')
 65 |     _header = struct.Struct('<Q')
 66 |     _segment = struct.Struct('<?xxxQQQ')
 67 | 
 68 |     def __init__(self, segments: list):
 69 |         self.segments = segments
 70 | 
 71 |     @classmethod
 72 |     def read_from(cls, buffer: BufferedReader):
 73 |         size, = cls._header.unpack(buffer.read(8))
 74 |         number_of_segments = size // 28  # 28 bytes per segment
 75 |         segments = [cls.segment(*cls._segment.unpack(buffer.read(28)))
 76 |                     for _ in range(number_of_segments)]
 77 | 
 78 |         return cls(segments)
 79 | 
 80 |     @property
 81 |     def uncompressed_size(self):
 82 |         return sum([segment.uncompressed_size for segment in self.segments])
 83 | 
 84 |     @property
 85 |     def compressed_size(self):
 86 |         return sum([segment.compressed_size for segment in self.segments])
 87 | 
 88 |     def to_bytes(self):
 89 |         size = len(self.segments) * 28
 90 |         header = b'segm' + self._header.pack(size)
 91 |         segments = [self._segment.pack(*segment) for segment in self.segments]
 92 |         return header + b''.join(segments)
 93 | 
 94 |     def __iter__(self):
 95 |         yield from self.segments
 96 | 
 97 |     def __getitem__(self, item):
 98 |         return self.segments[item]
 99 | 
100 |     def __repr__(self):
101 |         return '<XP3FileSegments {} segment(s)>'\
102 |             .format(len(self.segments))
103 | 
104 | 
105 | class XP3FileInfo:
106 |     info_chunk = struct.Struct('<QIQQH')
107 | 
108 |     def __init__(self, is_encrypted: bool, uncompressed_size: int, compressed_size: int, file_path: str):
109 |         self.is_encrypted = is_encrypted
110 |         self.uncompressed_size = uncompressed_size
111 |         self.compressed_size = compressed_size
112 |         self.file_path = file_path
113 | 
114 |     @classmethod
115 |     def read_from(cls, buffer: BufferedReader):
116 |         start = buffer.tell() + 8
117 |         size, flags, uncompressed_size, compressed_size, file_path_length \
118 |             = cls.info_chunk.unpack(buffer.read(30))
119 |         encrypted = bool(flags & XP3FileIsEncrypted)
120 | 
121 |         # 2 bytes per character and 2 bytes null-terminator
122 |         file_path, = struct.unpack('<' + (str(file_path_length * 2) + 's') + 'xx',
123 |                                    buffer.read(2 + file_path_length * 2))
124 |         file_path = file_path.decode('utf-16le')
125 | 
126 |         if buffer.tell() != start + size:
127 |             raise AssertionError('Buffer position {}, expected {}'.format(buffer.tell(),
128 |                                                                           start + size))
129 |         return cls(encrypted, uncompressed_size, compressed_size, file_path)
130 | 
131 |     def to_bytes(self):
132 |         size = 4 + 8 + 8 + 2 + (len(self.file_path) * 2) + 2
133 |         flags = XP3FileIsEncrypted if self.is_encrypted else 0
134 | 
135 |         return b'info' \
136 |                + self.info_chunk.pack(size, flags, self.uncompressed_size, self.compressed_size, len(self.file_path)) \
137 |                + self.file_path.encode('utf-16le') + b'\x00\x00'
138 | 
139 |     def __repr__(self):
140 |         return "<XP3FileInfo encrypted={}, uncompressed_size={}, compressed_size={}, file_path='{}'"\
141 |             .format(self.is_encrypted, self.uncompressed_size, self.compressed_size, self.file_path)
142 | 
143 | 
144 | class XP3FileAdler:
145 |     adler32_chunk = struct.Struct('<QI')
146 | 
147 |     def __init__(self, adler32: int):
148 |         self.value = adler32
149 | 
150 |     @classmethod
151 |     def read_from(cls, buffer: BufferedReader):
152 |         size, adler32, = cls.adler32_chunk.unpack(buffer.read(12))
153 |         if size != 4:  # adler value size should always be 4 bytes
154 |             raise AssertionError
155 |         return cls(adler32)
156 | 
157 |     @classmethod
158 |     def from_data(cls, data: bytes):
159 |         return cls(zlib.adler32(data))
160 | 
161 |     def to_bytes(self):
162 |         return b'adlr' + self.adler32_chunk.pack(4, self.value)
163 | 
164 |     def __repr__(self):
165 |         return "<XP3FileAdler adler32={}>".format(self.value)
166 | 
167 | 
168 | class XP3FileEntry:
169 |     file_chunk = struct.Struct('<Q')
170 | 
171 |     def __init__(self, time: XP3FileTime, adlr: XP3FileAdler, segm: XP3FileSegments, info: XP3FileInfo,
172 |                  encryption: XP3FileEncryption = None):
173 |         self.encryption = encryption
174 |         self.time = time
175 |         self.adlr = adlr
176 |         self.segm = segm
177 |         self.info = info
178 | 
179 |         if encryption:
180 |             if adlr.value != encryption.adler32:
181 |                 raise AssertionError('Checksum values in adlr chunk and encryption chunk do not match')
182 | 
183 |     @classmethod
184 |     def read_from(cls, buffer: BufferedReader):
185 |         encryption = None
186 |         time = None
187 |         adlr = None
188 |         segm = None
189 |         info = None
190 | 
191 |         name = buffer.read(4)
192 |         if name != b'File':  # first chunk should be 'File', if not most likely an encryption chunk
193 |             encryption = XP3FileEncryption.read_from(buffer, name)
194 |             if buffer.read(4) != b'File':
195 |                 raise AssertionError
196 |         start = buffer.tell()
197 |         size, = cls.file_chunk.unpack(buffer.read(8))
198 |         end = start + size
199 |         while buffer.tell() < end:
200 |             name = buffer.read(4)
201 |             if name == b'time':
202 |                 time = XP3FileTime.read_from(buffer)
203 |             elif name == b'adlr':
204 |                 adlr = XP3FileAdler.read_from(buffer)
205 |             elif name == b'segm':
206 |                 segm = XP3FileSegments.read_from(buffer)
207 |             elif name == b'info':
208 |                 info = XP3FileInfo.read_from(buffer)
209 | 
210 |         if not adlr or not segm or not info:
211 |             raise AssertionError
212 |         elif not time:  # time chunk is not always present
213 |             time = XP3FileTime()  # create an empty placeholder
214 | 
215 |         return cls(adlr=adlr, segm=segm, info=info, time=time, encryption=encryption)
216 | 
217 |     @property
218 |     def adler32(self):
219 |         return self.adlr.value
220 | 
221 |     @property
222 |     def is_encrypted(self):
223 |         return bool(self.encryption)
224 | 
225 |     @property
226 |     def file_path(self):
227 |         if self.is_encrypted and self.encryption:
228 |             return self.encryption.file_path
229 |         else:
230 |             return self.info.file_path
231 | 
232 |     def to_bytes(self):
233 |         entry = self.time.to_bytes() \
234 |             + self.adlr.to_bytes() \
235 |             + self.segm.to_bytes() \
236 |             + self.info.to_bytes()
237 |         header = struct.pack('<4sQ', b'File', len(entry))
238 |         encryption = self.encryption.to_bytes() if self.is_encrypted else b''
239 | 
240 |         return encryption + header + entry
241 | 
242 |     def __repr__(self):
243 |         return "<XP3FileEntry file_path='{}', size={}, encrypted={}, timestamp={}>"\
244 |             .format(self.file_path, self.info.uncompressed_size, self.is_encrypted, datetime.utcfromtimestamp(self.time.timestamp))
245 | 


--------------------------------------------------------------------------------