├── .gitignore ├── readme.md └── cisoplus.py /.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | !*.md 4 | !*.py 5 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # CISO Plus 2 | ## Converting PSP ISO file to compressed CSO format 3 | 4 | The code is mainly based on two projects: 5 | [phyber/ciso](https://github.com/phyber/ciso) 6 | [barneygale/iso9660](https://github.com/barneygale/iso9660) 7 | 8 | But, I made some modifications and added extra features of what [CisoPlus](http://cisoplus.pspgen.com/) has: 9 | 10 | 1. Threshold: Only the blocks of which compression ratio is below the specific threshold will be compressed. 11 | 2. Do not compress multimedia files: Leave PMF and AT3 files alone. 12 | 3. Vaccum files for system upgrading: Fill these files with blank data (the iso file is untouched), so it can save even more space. 13 | -------------------------------------------------------------------------------- /cisoplus.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2.7 2 | 3 | from __future__ import print_function 4 | import os 5 | import struct 6 | import sys 7 | import zlib 8 | import urllib 9 | import pprint 10 | import array 11 | from itertools import chain 12 | import os.path 13 | 14 | try: 15 | from cStringIO import StringIO 16 | except ImportError: 17 | from StringIO import StringIO 18 | 19 | CISO_MAGIC = 0x4F534943 # CISO 20 | CISO_HEADER_SIZE = 0x18 # 24 21 | CISO_BLOCK_SIZE = 0x800 # 2048 22 | CISO_HEADER_FMT = ' 0: 83 | p = {} 84 | l1 = self._unpack('B') 85 | l2 = self._unpack('B') 86 | p['ex_loc'] = self._unpack('i') 246 | self._pvd['path_table_opt_m_loc'] = self._unpack('>i') 247 | _, self._root = self._unpack_record() #root directory record 248 | self._pvd['volume_set_identifer'] = self._unpack_string(128) 249 | self._pvd['publisher_identifier'] = self._unpack_string(128) 250 | self._pvd['data_preparer_identifier'] = self._unpack_string(128) 251 | self._pvd['application_identifier'] = self._unpack_string(128) 252 | self._pvd['copyright_file_identifier'] = self._unpack_string(38) 253 | self._pvd['abstract_file_identifier'] = self._unpack_string(36) 254 | self._pvd['bibliographic_file_identifier'] = self._unpack_string(37) 255 | self._pvd['volume_datetime_created'] = self._unpack_vd_datetime() 256 | self._pvd['volume_datetime_modified'] = self._unpack_vd_datetime() 257 | self._pvd['volume_datetime_expires'] = self._unpack_vd_datetime() 258 | self._pvd['volume_datetime_effective'] = self._unpack_vd_datetime() 259 | self._pvd['file_structure_version'] = self._unpack('B') 260 | 261 | ## 262 | ## Unpack a directory record (a listing of a file or folder) 263 | ## 264 | 265 | def _unpack_record(self, read=0): 266 | l0 = self._unpack('B') 267 | 268 | if l0 == 0: 269 | return read+1, None 270 | 271 | l1 = self._unpack('B') 272 | 273 | d = dict() 274 | d['ex_loc'] = self._unpack_both('I') 275 | d['ex_len'] = self._unpack_both('I') 276 | d['datetime'] = self._unpack_dir_datetime() 277 | d['flags'] = self._unpack('B') 278 | d['interleave_unit_size'] = self._unpack('B') 279 | d['interleave_gap_size'] = self._unpack('B') 280 | d['volume_sequence'] = self._unpack_both('h') 281 | 282 | l2 = self._unpack('B') 283 | d['name'] = self._unpack_string(l2).split(';')[0] 284 | if d['name'] == '\x00': 285 | d['name'] = '' 286 | 287 | if l2 % 2 == 0: 288 | self._unpack('B') 289 | 290 | t = 34 + l2 - (l2 % 2) 291 | 292 | e = l0-t 293 | if e>0: 294 | extra = self._unpack_raw(e) 295 | 296 | return read+l0, d 297 | 298 | #Assuming d is a directory record, this generator yields its children 299 | def _unpack_dir_children(self, d): 300 | sector = d['ex_loc'] 301 | read = 0 302 | self._get_sector(sector, 2048) 303 | 304 | read, r_self = self._unpack_record(read) 305 | read, r_parent = self._unpack_record(read) 306 | 307 | while read < r_self['ex_len']: #Iterate over files in the directory 308 | if read % 2048 == 0: 309 | sector += 1 310 | self._get_sector(sector, 2048) 311 | read, data = self._unpack_record(read) 312 | 313 | if data == None: #end of directory listing 314 | to_read = 2048 - (read % 2048) 315 | self._unpack_raw(to_read) 316 | read += to_read 317 | else: 318 | yield data 319 | 320 | #Search for one child amongst the children 321 | def _search_dir_children(self, d, term): 322 | for e in self._unpack_dir_children(d): 323 | if e['name'] == term: 324 | return e 325 | 326 | raise ISO9660IOError(term) 327 | ## 328 | ## Datatypes 329 | ## 330 | 331 | def _unpack_raw(self, l): 332 | return self._buff.read(l) 333 | 334 | #both-endian 335 | def _unpack_both(self, st): 336 | a = self._unpack('<'+st) 337 | b = self._unpack('>'+st) 338 | assert a == b 339 | return a 340 | 341 | def _unpack_string(self, l): 342 | return self._buff.read(l).rstrip(' ') 343 | 344 | def _unpack(self, st): 345 | if st[0] not in ('<','>'): 346 | st = '<' + st 347 | d = struct.unpack(st, self._buff.read(struct.calcsize(st))) 348 | if len(st) == 2: 349 | return d[0] 350 | else: 351 | return d 352 | 353 | def _unpack_vd_datetime(self): 354 | return self._unpack_raw(17) #TODO 355 | 356 | def _unpack_dir_datetime(self): 357 | return self._unpack_raw(7) #TODO 358 | 359 | def get_terminal_size(fd=sys.stdout.fileno()): 360 | try: 361 | import fcntl, termios 362 | hw = struct.unpack("hh", fcntl.ioctl( 363 | fd, termios.TIOCGWINSZ, '1234')) 364 | except: 365 | try: 366 | hw = (os.environ['LINES'], os.environ['COLUMNS']) 367 | except: 368 | hw = (25, 80) 369 | return hw 370 | 371 | (console_height, console_width) = get_terminal_size() 372 | 373 | def seek_and_read(f, pos, size): 374 | f.seek(pos, os.SEEK_SET) 375 | return f.read(size) 376 | 377 | def parse_header_info(header_data): 378 | (magic, header_size, total_bytes, block_size, 379 | ver, align) = header_data 380 | if magic == CISO_MAGIC: 381 | ciso = { 382 | 'magic': magic, 383 | 'magic_str': ''.join( 384 | [chr(magic >> i & 0xFF) for i in (0,8,16,24)]), 385 | 'header_size': header_size, 386 | 'total_bytes': total_bytes, 387 | 'block_size': block_size, 388 | 'ver': ver, 389 | 'align': align, 390 | 'total_blocks': int(total_bytes / block_size), 391 | } 392 | ciso['index_size'] = (ciso['total_blocks'] + 1) * 4 393 | else: 394 | raise Exception("Not a CISO file.") 395 | return ciso 396 | 397 | def update_progress(progress): 398 | barLength = console_width - len("Progress: 100% []") - 1 399 | block = int(round(barLength*progress)) + 1 400 | text = "\rProgress: [{blocks}] {percent:.0f}%".format( 401 | blocks="#" * block + "-" * (barLength - block), 402 | percent=progress * 100) 403 | sys.stdout.write(text) 404 | sys.stdout.flush() 405 | 406 | def decompress_cso(infile, outfile): 407 | with open(outfile, 'wb') as fout: 408 | with open(infile, 'rb') as fin: 409 | data = seek_and_read(fin, 0, CISO_HEADER_SIZE) 410 | header_data = struct.unpack(CISO_HEADER_FMT, data) 411 | ciso = parse_header_info(header_data) 412 | 413 | # Print some info before we start 414 | print("Decompressing '{}' to '{}'".format( 415 | infile, outfile)) 416 | for k, v in ciso.items(): 417 | print("{}: {}".format(k, v)) 418 | 419 | # Get the block index 420 | block_index = [struct.unpack(" percent_cnt: 460 | update_progress((block / (ciso['total_blocks'] + 1))) 461 | percent_cnt = percent 462 | # close infile 463 | # close outfile 464 | return True 465 | 466 | def check_file_size(f): 467 | f.seek(0, os.SEEK_END) 468 | file_size = f.tell() 469 | ciso = { 470 | 'magic': CISO_MAGIC, 471 | 'ver': 1, 472 | 'block_size': CISO_BLOCK_SIZE, 473 | 'total_bytes': file_size, 474 | 'total_blocks': int(file_size / CISO_BLOCK_SIZE), 475 | 'align': 0, 476 | } 477 | f.seek(0, os.SEEK_SET) 478 | return ciso 479 | 480 | def write_cso_header(f, ciso): 481 | f.write(struct.pack(CISO_HEADER_FMT, 482 | ciso['magic'], 483 | CISO_HEADER_SIZE, 484 | ciso['total_bytes'], 485 | ciso['block_size'], 486 | ciso['ver'], 487 | ciso['align'] 488 | )) 489 | 490 | def write_block_index(f, block_index): 491 | for index, block in enumerate(block_index): 492 | try: 493 | f.write(struct.pack('> ciso['align'] 554 | 555 | # Read raw data 556 | raw_data = fin.read(ciso['block_size']) 557 | raw_data_size = len(raw_data) 558 | 559 | # Compress block 560 | # Compressed data will have the gzip header on it, we strip that. 561 | if nocom_array[block] == 1: 562 | writable_data = raw_data 563 | # Plain block marker 564 | block_index[block] |= 0x80000000 565 | # Next index 566 | write_pos += raw_data_size 567 | else: 568 | if nocom_array[block] == 2: 569 | # block += 1 570 | # print(block) 571 | raw_data = struct.pack('{0}x'.format(raw_data_size)) 572 | compressed_data = zlib.compress(raw_data, compression_level)[2:] 573 | compressed_size = len(compressed_data) 574 | 575 | if compressed_size > (raw_data_size * COM_THRESHOLD): 576 | writable_data = raw_data 577 | # Plain block marker 578 | block_index[block] |= 0x80000000 579 | # Next index 580 | write_pos += raw_data_size 581 | else: 582 | writable_data = compressed_data 583 | # Next index 584 | write_pos += compressed_size 585 | 586 | # Write data 587 | fout.write(writable_data) 588 | 589 | # Progress bar 590 | percent = int(round((block / (ciso['total_blocks'] + 1)) * 100)) 591 | if percent > percent_cnt: 592 | update_progress((block / (ciso['total_blocks'] + 1))) 593 | percent_cnt = percent 594 | 595 | # end for block 596 | # last position (total size) 597 | block_index[block+1] = write_pos >> ciso['align'] 598 | 599 | # write header and index block 600 | print("Writing block index") 601 | fout.seek(CISO_HEADER_SIZE, os.SEEK_SET) 602 | write_block_index(fout, block_index) 603 | # end open(infile) 604 | 605 | if __name__ == '__main__': 606 | if len(sys.argv) < 3: 607 | sys.stderr.write('{0} isofile csofile\n'.format(os.path.basename(sys.argv[0]))) 608 | raise SystemExit 609 | infile = sys.argv[1] 610 | outfile = sys.argv[2] 611 | 612 | 613 | iso = ISO9660(infile) 614 | # for i in iso.tree(): 615 | # print(i) 616 | 617 | print('Scanning media files...') 618 | iso.scan_media_file() 619 | # mediatree = IntervalTree(iso._media_pos, 0, 1, 620 | # min(i[0] for i in iso._media_pos), 621 | # max(i[1] for i in iso._media_pos)) 622 | # mediatree.pprint(2) 623 | for i in iso._media_file: 624 | print(i) 625 | 626 | # startlen = [] 627 | # for i in vaccum_list: 628 | # s_l = iso.get_file_pos(i) 629 | # if s_l is not None: 630 | # startlen.append(s_l) 631 | # 632 | # if len(startlen): 633 | # with open(infile, 'r+b') as wfh: 634 | # for start, length in startlen: 635 | # wfh.seek(start) 636 | # for i in xrange(0, length/1024): 637 | # wfh.write(struct.pack('1024x')) 638 | # wfh.write(struct.pack('{0}x'.format(length % 1024))) 639 | 640 | compression_level = 9 641 | compress_iso(infile, outfile, compression_level, 642 | iso._media_block, 643 | iso._vaccum_block) 644 | 645 | --------------------------------------------------------------------------------