├── __init__.py ├── Scripts ├── __init__.py ├── compile │ ├── __init__.py │ └── compile.py └── s4cl_sample_mod_scripts │ ├── __init__.py │ ├── enums │ ├── __init__.py │ └── string_enums.py │ ├── notifications │ ├── __init__.py │ └── show_loaded_notification.py │ └── modinfo.py ├── Utilities ├── __init__.py ├── py37_execute_decompile.exe ├── decompilation_method.py ├── compile_utils.py ├── unpyc3_decompiler.py ├── unpyc3_compiler.py ├── py37_decompiler.py └── unpyc3.py ├── Release └── S4CLSampleMod │ └── s4cl_sample_mod.package ├── .idea ├── misc.xml ├── vcs.xml ├── modules.xml └── S4CLSampleMod.iml ├── Libraries └── S4CL │ └── README.md ├── custom_scripts_for_decompile └── README.md ├── .gitignore ├── EA └── README.md ├── README.md ├── decompile_scripts.py └── settings.py /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Scripts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Utilities/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Scripts/compile/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Scripts/s4cl_sample_mod_scripts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Scripts/s4cl_sample_mod_scripts/enums/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Scripts/s4cl_sample_mod_scripts/notifications/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Utilities/py37_execute_decompile.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ColonolNutty/s4cl-template-project/HEAD/Utilities/py37_execute_decompile.exe -------------------------------------------------------------------------------- /Release/S4CLSampleMod/s4cl_sample_mod.package: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ColonolNutty/s4cl-template-project/HEAD/Release/S4CLSampleMod/s4cl_sample_mod.package -------------------------------------------------------------------------------- /Utilities/decompilation_method.py: -------------------------------------------------------------------------------- 1 | class S4PyDecompilationMethod: 2 | """Method to use when compiling/decompiling python code.""" 3 | UNPYC3 = 'unpyc3' 4 | PY37DEC = 'py37dec' 5 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /Scripts/s4cl_sample_mod_scripts/enums/string_enums.py: -------------------------------------------------------------------------------- 1 | import enum 2 | 3 | 4 | # noinspection PyUnresolvedReferences 5 | class S4CLSampleModStringId(enum.Int): 6 | """ Identifiers for the sample mod strings. (Decimal identifiers from the String Tables) """ 7 | SAMPLE_MOD_LOADED_NOTIFICATION_TITLE = 3025489840 8 | SAMPLE_MOD_LOADED_NOTIFICATION_DESCRIPTION = 3060066151 9 | -------------------------------------------------------------------------------- /Libraries/S4CL/README.md: -------------------------------------------------------------------------------- 1 | Copy the `sims4communitylib` source module into this folder to give access to the S4CL python modules 2 | 3 | The `sims4communitylib` source module may be downloaded from: https://github.com/ColonolNutty/Sims4CommunityLibrary/tree/master/Scripts/sims4communitylib 4 | 5 | You should end up with a folder path that looks like this: `Libraries\S4CL\sims4communitylib` 6 | 7 | If this folder is not highlighted BLUE or the `sims4communitylib` module is not being found in PyCharm, Right click the folder -> Mark Directory as... -> Sources Root. -------------------------------------------------------------------------------- /Scripts/compile/compile.py: -------------------------------------------------------------------------------- 1 | import os 2 | from Utilities.unpyc3_compiler import Unpyc3PythonCompiler 3 | 4 | 5 | release_dir = os.path.join('..', '..', 'Release', 'S4CLSampleMod') 6 | 7 | # This function invocation will compile the files found within Scripts/s4cl_sample_mod_scripts, put them inside of a file named s4cl_sample_mod.ts4script, and it will finally place that ts4script file within /Release/S4CLSampleMod. 8 | Unpyc3PythonCompiler.compile_mod( 9 | folder_path_to_output_ts4script_to=release_dir, 10 | names_of_modules_include=('s4cl_sample_mod_scripts',), 11 | output_ts4script_name='s4cl_sample_mod' 12 | ) 13 | -------------------------------------------------------------------------------- /custom_scripts_for_decompile/README.md: -------------------------------------------------------------------------------- 1 | This folder is where you place `.pyc` files to be decompiled when running `decompile_scripts.py` 2 | 3 | ## Decompile Scripts From Other Mods. 4 | 5 | 1. Inside `/settings.py`, change `should_decompile_ea_scripts` to `False` 6 | 2. Inside `/settings.py`, change `should_decompile_custom_scripts` to `True` 7 | 3. If it does not exist, create a folder in your project with the name `custom_scripts_for_decompile`. i.e. `/custom_scripts_for_decompile` 8 | 4. Put the script files (.pyc) of the mod you wish to decompile, inside of the `custom_scripts_for_decompile` folder. (Every ts4script file is a zip file and can be opened and extracted like one!) 9 | 5. Run the `decompile_scripts.py` script 10 | 6. It will decompile the custom scripts and put them inside of the folder: `/custom_scripts_for_decompile/_decompiled/...` -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Idea Project 2 | .idea/workspace.xml 3 | .idea/tasks.xml 4 | .idea/dictionaries 5 | .idea/inspectionProfiles 6 | 7 | # EA Scripts 8 | EA/base 9 | EA/core 10 | EA/generated 11 | EA/simulation 12 | EA/base.zip 13 | EA/core.zip 14 | EA/generated.zip 15 | EA/simulation.zip 16 | 17 | # Libraries 18 | Libraries/**/* 19 | 20 | # Logs 21 | compilelog.txt 22 | decompilelog.txt 23 | 24 | # Temp Compiled Files 25 | __pycache__ 26 | */__pycache__ 27 | Utilities/*.pyc 28 | 29 | # Compiled Files 30 | *.pyo 31 | *.pyc 32 | **/*.pyo 33 | **/*.pyc 34 | */*.pyc 35 | */*.pyo 36 | custom_scripts_for_decompile/*.py 37 | custom_scripts_for_decompile/*.pyc 38 | custom_scripts_for_decompile/*.pyo 39 | custom_scripts_for_decompile/**/*.pyc 40 | custom_scripts_for_decompile/**/*.pyo 41 | custom_scripts_for_decompile/**/*.py 42 | 43 | # Release 44 | **/*.7z 45 | **/*.zip 46 | **/Release/*.7z 47 | **/Release/*.zip 48 | **/*.ts4script 49 | 50 | # Misc 51 | **/*.lnk 52 | **/*.xml___jb_old___ 53 | Tunings/Temp -------------------------------------------------------------------------------- /EA/README.md: -------------------------------------------------------------------------------- 1 | This folder will be where the decompiled EA python scripts will be placed when running `decompile_scripts.py` with `should_decompile_ea_scripts` set to `True` 2 | 3 | # Decompile EA Python Scripts. 4 | 5 | 1. Inside `/settings.py`, change `should_decompile_ea_scripts` to `True` 6 | 2. If it does not exist, create a folder in your project with the name `EA`. i.e. /EA 7 | 3. Run the `decompile_scripts.py` script 8 | 4. The script will decompile the EA python scripts and put them inside of this folder: `/EA/...` 9 | 5. After the script finishes running, look inside of the `/EA` folder, you should see four folders (`base`, `core`, `generated`, `simulation`) and four zip files (`base.zip`, `core.zip`, `generated.zip`, `simulation.zip`) 10 | 6. In PyCharm, highlight all four folders (Not Zip files) (`base`, `core`, `generated`, `simulation`) and Right Click them -> `Mark Directory as...` -> `Sources Root` 11 | - root\EA\base 12 | - root\EA\core 13 | - root\EA\generated 14 | - root\EA\simulation 15 | 7. The folders should now be `BLUE` in color, if they are not `BLUE`, repeat step the previous step. -------------------------------------------------------------------------------- /Scripts/s4cl_sample_mod_scripts/modinfo.py: -------------------------------------------------------------------------------- 1 | from sims4communitylib.mod_support.common_mod_info import CommonModInfo 2 | 3 | 4 | class ModInfo(CommonModInfo): 5 | """ Mod info for the S4CL Sample Mod. """ 6 | # To create a Mod Identity for this mod, simply do ModInfo.get_identity(). Please refrain from using the ModInfo of The Sims 4 Community Library in your own mod and instead use yours! 7 | _FILE_PATH: str = str(__file__) 8 | 9 | @property 10 | def _name(self) -> str: 11 | # This is the name that'll be used whenever a Messages.txt or Exceptions.txt file is created <_name>_Messages.txt and <_name>_Exceptions.txt. 12 | return 'S4CLSampleMod' 13 | 14 | @property 15 | def _author(self) -> str: 16 | # This is your name. 17 | return 'ColonolNutty' 18 | 19 | @property 20 | def _base_namespace(self) -> str: 21 | # This is the name of the root package 22 | return 's4cl_sample_mod_scripts' 23 | 24 | @property 25 | def _file_path(self) -> str: 26 | # This is simply a file path that you do not need to change. 27 | return ModInfo._FILE_PATH 28 | -------------------------------------------------------------------------------- /.idea/S4CLSampleMod.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 21 | -------------------------------------------------------------------------------- /Scripts/s4cl_sample_mod_scripts/notifications/show_loaded_notification.py: -------------------------------------------------------------------------------- 1 | from s4cl_sample_mod_scripts.enums.string_enums import S4CLSampleModStringId 2 | from sims4communitylib.events.event_handling.common_event_registry import CommonEventRegistry 3 | from sims4communitylib.events.zone_spin.events.zone_late_load import S4CLZoneLateLoadEvent 4 | from sims4communitylib.notifications.common_basic_notification import CommonBasicNotification 5 | 6 | 7 | class S4CLSampleModShowLoadedMessage: 8 | """ A class that listens for a zone load event and shows a notification upon loading into a household. """ 9 | @staticmethod 10 | def show_loaded_notification() -> None: 11 | """ Show that the sample mod has loaded. """ 12 | notification = CommonBasicNotification( 13 | S4CLSampleModStringId.SAMPLE_MOD_LOADED_NOTIFICATION_TITLE, 14 | S4CLSampleModStringId.SAMPLE_MOD_LOADED_NOTIFICATION_DESCRIPTION 15 | ) 16 | notification.show() 17 | 18 | @staticmethod 19 | @CommonEventRegistry.handle_events('s4cl_sample_mod') 20 | def _show_loaded_notification_when_loaded(event_data: S4CLZoneLateLoadEvent): 21 | if event_data.game_loaded: 22 | # If the game has not loaded yet, we don't want to show our notification. 23 | return 24 | S4CLSampleModShowLoadedMessage.show_loaded_notification() 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # S4CL Template Project 2 | This repository is meant to be a basic template you may use to start working with the [Sims 4 Community Library](https://github.com/ColonolNutty/Sims4CommunityLibrary) API. 3 | 4 | A Basic Mod (S4CL Sample Mod) is included. If installed into your Mods folder, it will display a notification upon loading a household. 5 | 6 | If a question has not been answered by the Wiki or you have further questions on how to setup the Template project, how to use S4CL, or how to make your own mod, feel free to join the [discord](https://discord.gg/fdCgyXkDZb) 7 | 8 | Take a look at how to setup the Template Project to start working with it in the wiki [here](https://github.com/ColonolNutty/s4cl-template-project/wiki/Project-Setup)! 9 | 10 | ## Decompile EA Python Scripts. 11 | 12 | 1. Inside `/settings.py`, change `should_decompile_ea_scripts` to `True` 13 | 2. If it does not exist, create a folder in your project with the name `EA`. i.e. /EA 14 | 3. Run the `decompile_scripts.py` script 15 | 4. It will decompile the EA scripts and put them inside of the folder: `/EA/...` 16 | 5. Inside of the /EA folder, you should see four folders (base, core, generated, simulation) 17 | 6. In PyCharm, highlight all four folders (Not Zip files) (`base`, `core`, `generated`, `simulation`) and Right Click them -> `Mark Directory as...` -> `Sources Root` 18 | 19 | 20 | ## Decompile Scripts From Other Mods. 21 | 22 | 1. Inside `/settings.py`, change `should_decompile_ea_scripts` to `False` 23 | 2. Inside `/settings.py`, change `should_decompile_custom_scripts` to `True` 24 | 3. If it does not exist, create a folder in your project with the name `custom_scripts_for_decompile`. i.e. `/custom_scripts_for_decompile` 25 | 4. Put the script files (.pyc) of the mod you wish to decompile, inside of the `custom_scripts_for_decompile` folder. (Every ts4script file is a zip file and can be opened and extracted like one!) 26 | 5. Run the `decompile_scripts.py` script 27 | 6. It will decompile the custom scripts and put them inside of the folder: `/custom_scripts_for_decompile/_decompiled/...` -------------------------------------------------------------------------------- /decompile_scripts.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from compile_utils import _remove_files_conflicting_with_decompile, _replace_renamed_files 4 | from decompilation_method import S4PyDecompilationMethod 5 | from settings import custom_scripts_for_decompile_source, game_folder, decompile_method_name, custom_scripts_for_decompile_destination, should_decompile_ea_scripts, should_decompile_custom_scripts 6 | 7 | 8 | def _decompile_using_unpyc3(decompile_ea_scripts: bool=False, decompile_custom_scripts: bool=False): 9 | output_folder = 'EA' 10 | if not os.path.exists(output_folder): 11 | os.mkdir(output_folder) 12 | 13 | _remove_files_conflicting_with_decompile(decompile_ea_scripts=decompile_ea_scripts) 14 | 15 | from Utilities.unpyc3_decompiler import Unpyc3PyDecompiler 16 | 17 | if decompile_custom_scripts: 18 | Unpyc3PyDecompiler.decompile_folder(custom_scripts_for_decompile_source) 19 | 20 | if decompile_ea_scripts: 21 | gameplay_folder_data = os.path.join(game_folder, 'Data', 'Simulation', 'Gameplay') 22 | if os.name != 'nt': 23 | gameplay_folder_game = os.path.join(game_folder, 'Python') 24 | else: 25 | gameplay_folder_game = os.path.join(game_folder, 'Game', 'Bin', 'Python') 26 | 27 | Unpyc3PyDecompiler.extract_folder(output_folder, gameplay_folder_data) 28 | Unpyc3PyDecompiler.extract_folder(output_folder, gameplay_folder_game) 29 | 30 | _replace_renamed_files(decompile_ea_scripts=should_decompile_ea_scripts) 31 | 32 | 33 | def _decompile_using_py37dec() -> None: 34 | _remove_files_conflicting_with_decompile(decompile_ea_scripts=should_decompile_ea_scripts) 35 | 36 | from py37_decompiler import Py37PythonDecompiler 37 | Py37PythonDecompiler().decompile( 38 | custom_scripts_for_decompile_source, 39 | custom_scripts_for_decompile_destination 40 | ) 41 | _replace_renamed_files(decompile_ea_scripts=should_decompile_ea_scripts) 42 | 43 | 44 | if decompile_method_name == S4PyDecompilationMethod.UNPYC3: 45 | _decompile_using_unpyc3(decompile_ea_scripts=should_decompile_ea_scripts, decompile_custom_scripts=should_decompile_custom_scripts) 46 | elif decompile_method_name == S4PyDecompilationMethod.PY37DEC: 47 | _decompile_using_py37dec() 48 | -------------------------------------------------------------------------------- /Utilities/compile_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def _remove_files_conflicting_with_decompile(decompile_ea_scripts: bool=False): 5 | ea_folder = os.path.realpath(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'EA')) 6 | if not os.path.exists(ea_folder): 7 | os.mkdir(ea_folder) 8 | 9 | if decompile_ea_scripts: 10 | # print('Removing EA decompiled files before decompiling it again.') 11 | to_remove_before_compile = ( 12 | 'base', 13 | 'base.zip', 14 | 'core', 15 | 'core.zip', 16 | 'generated', 17 | 'generated.zip', 18 | 'simulation', 19 | 'simulation.zip', 20 | ) 21 | 22 | def _remove_directory_recursive(directory_path: str): 23 | for _file_in_dir in os.listdir(directory_path): 24 | _to_remove_path = os.path.join(directory_path, _file_in_dir) 25 | if os.path.isdir(_to_remove_path): 26 | # noinspection PyBroadException 27 | try: 28 | os.rmdir(_to_remove_path) 29 | except: 30 | _remove_directory_recursive(_to_remove_path) 31 | os.rmdir(_to_remove_path) 32 | else: 33 | os.remove(_to_remove_path) 34 | 35 | for to_remove in to_remove_before_compile: 36 | to_remove_path = os.path.join(ea_folder, to_remove) 37 | if not os.path.exists(to_remove_path): 38 | # print(f'Did not exist \'{to_remove_path}\'') 39 | continue 40 | if os.path.isdir(to_remove_path): 41 | # noinspection PyBroadException 42 | try: 43 | os.rmdir(to_remove_path) 44 | except: 45 | _remove_directory_recursive(to_remove_path) 46 | os.rmdir(to_remove_path) 47 | else: 48 | os.remove(to_remove_path) 49 | else: 50 | # print('Renaming enum.py to enum.py_renamed') 51 | to_fix_before_decompile = ( 52 | os.path.join('core', 'enum.py'), 53 | os.path.join('core', 'enum.pyc'), 54 | ) 55 | 56 | for to_fix in to_fix_before_decompile: 57 | to_fix_path = os.path.join(ea_folder, to_fix) 58 | if not os.path.exists(to_fix_path): 59 | # print(f'Did not exist \'{to_fix_path}\'') 60 | continue 61 | if os.path.isdir(to_fix_path): 62 | os.rename(to_fix_path, to_fix_path + '_renamed') 63 | else: 64 | os.rename(to_fix_path, to_fix_path + '_renamed') 65 | 66 | 67 | def _replace_renamed_files(decompile_ea_scripts: bool=False): 68 | if decompile_ea_scripts: 69 | return 70 | 71 | # print('Renaming enum.py_renamed to enum.py') 72 | 73 | ea_folder = os.path.realpath(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'EA')) 74 | to_fix_after_decompile = ( 75 | os.path.join('core', 'enum.py_renamed'), 76 | os.path.join('core', 'enum.pyc_renamed'), 77 | ) 78 | 79 | for to_fix in to_fix_after_decompile: 80 | to_remove_path = os.path.join('.', ea_folder, to_fix) 81 | if not os.path.exists(to_remove_path): 82 | # print(f'Did not exist \'{to_remove_path}\'') 83 | continue 84 | if os.path.isdir(to_remove_path): 85 | os.rename(to_remove_path, to_remove_path.rstrip('_renamed')) 86 | else: 87 | os.rename(to_remove_path, to_remove_path.rstrip('_renamed')) -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | from decompilation_method import S4PyDecompilationMethod 3 | 4 | # This name will be appended to the front of compiled script files. 5 | # Example: 6 | # If I set the value to "ColonolNutty" 7 | # compiling my scripts would output a file with the name "ColonolNutty_.ts4script" 8 | creator_name = '' 9 | 10 | # If you want to decompile the EA Python Scripts: 11 | # 1. Change should_decompile_ea_scripts to True 12 | # 2. Create a folder in your project with the name EA. i.e. /EA 13 | # 2. Run the decompile_scripts.py script 14 | # 3. It will decompile the EA scripts and put them inside of the folder: /EA/... 15 | # 4. Inside of the /EA folder, you should see four folders (base, core, generated, simulation) 16 | # 5. Highlight all four of those folders and right click them. Then do Mark Directory as... Sources Root 17 | should_decompile_ea_scripts: bool = True 18 | 19 | # If you want to decompile scripts from another authors mod 20 | # 1. Create a folder in your project with the name decompiled. i.e. /custom_scripts_for_decompile 21 | # 1. Put the script files (.pyc) of the mod you wish to decompile, inside of the 'decompiled' folder. (Every ts4script file is a zip file and can be opened like one!) 22 | # 2. Change should_decompile_custom_scripts to True 23 | # 3. Run the decompile_scripts.py script 24 | # 4. It will decompile the custom scripts and put them inside of the folder: /custom_scripts_for_decompile/_decompiled/... 25 | should_decompile_custom_scripts: bool = True 26 | 27 | 28 | # ---------------------------- Unless you know what you are doing, do not change anything below this line! ---------------------------- 29 | 30 | # Set to either S4PyDecompilationMethod.UNPYC3 or S4PyDecompilationMethod.PY37DEC 31 | # S4PyDecompilationMethod.PY37DEC is the default, however if it fails to decompile some files, feel free to change this to S4PyDecompilationMethod.UNPYC3 and try to decompile using that decompiler instead 32 | decompile_method_name = S4PyDecompilationMethod.PY37DEC 33 | 34 | custom_scripts_for_decompile_source: str = './custom_scripts_for_decompile' 35 | custom_scripts_for_decompile_destination: str = './custom_scripts_for_decompile' 36 | 37 | if should_decompile_ea_scripts: 38 | decompile_method_name = S4PyDecompilationMethod.UNPYC3 39 | should_decompile_custom_scripts = False 40 | 41 | # If this path is not correct, change it to your Mods folder location instead. 42 | if os.name != 'nt': 43 | # Mac 44 | mods_folder = os.path.join(os.environ['HOME'], 'Documents', 'Electronic Arts', 'The Sims 4', 'Mods') 45 | print(f'Mods folder path: {mods_folder}') 46 | else: 47 | # Windows 48 | mods_folder = os.path.join(os.environ['USERPROFILE'], 'Documents', 'Electronic Arts', 'The Sims 4', 'Mods') 49 | print(f'Mods folder path: {mods_folder}') 50 | 51 | # Location of the game's zipped binary scripts (base.zip, core.zip and simulation.zip) 52 | # If this path is not found properly when trying to decompile, change it to the location where you have installed The Sims 4 at, this would be the folder that contains the GameVersion.txt file 53 | if os.name != 'nt': 54 | # Mac 55 | game_folder = os.path.join(os.environ['HOME'], 'Applications', 'The Sims 4.app', 'Contents') 56 | print(f'Game folder path: {game_folder}') 57 | else: 58 | # noinspection PyBroadException 59 | try: 60 | # Windows 61 | import winreg as winreg 62 | key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Maxis\\The Sims 4') 63 | (game_folder, _) = winreg.QueryValueEx(key, 'Install Dir') 64 | print(f'Game folder path: {game_folder}') 65 | except: 66 | raise Exception('The Sims 4 game folder was not found! Please specify one manually in settings.py.') 67 | -------------------------------------------------------------------------------- /Utilities/unpyc3_decompiler.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import os 3 | import io 4 | import fnmatch 5 | import traceback 6 | from zipfile import PyZipFile 7 | from Utilities.unpyc3 import decompile 8 | 9 | 10 | class Unpyc3PyDecompiler: 11 | """A handler that can both compile and decompile scripts using unpyc3.""" 12 | _script_package_types = ['*.zip', '*.ts4script'] 13 | 14 | @classmethod 15 | def decompile_file(cls, path_to_file_for_decompile: str, throw_on_error: bool=False) -> bool: 16 | """Decompile a python file using unpyc3.""" 17 | _replace_characters = () 18 | # print('Decompiling \'{}\''.format(path_to_file_for_decompile)) 19 | decompiled_py = decompile(path_to_file_for_decompile) 20 | with io.open(path_to_file_for_decompile.replace('.pyc', '.py').replace('.pyo', '.py'), 'w', encoding="utf-8") as output_py: 21 | success = True 22 | for statement in decompiled_py.statements: 23 | try: 24 | statement_str = str(statement) 25 | for replace_char in _replace_characters: 26 | statement_str = statement_str.replace(replace_char, '-----') 27 | output_py.write(statement_str + '\r') 28 | except Exception as ex: 29 | try: 30 | statement_for_display = statement.gen_display() if hasattr(statement, 'gen_display') else statement 31 | print(f'Failed to parse statement. {statement_for_display} {ex}') 32 | traceback.print_exc() 33 | if throw_on_error: 34 | raise ex 35 | success = False 36 | except Exception as ex2: 37 | print(f'Another error occurred! {ex} {ex2}') 38 | if throw_on_error: 39 | raise ex2 40 | success = False 41 | continue 42 | return success 43 | 44 | @classmethod 45 | def decompile_folder(cls, folder_path: str, throw_on_error: bool=False) -> None: 46 | """Decompile python files within a folder using unpyc3.""" 47 | pattern = '*.pyc' 48 | successful_decompiles = 0 49 | failed_decompiles = 0 50 | for root, dirs, files in os.walk(folder_path): 51 | for file_name in fnmatch.filter(files, pattern): 52 | path_to_file_for_decompile = str(os.path.join(root, file_name)) 53 | try: 54 | if cls.decompile_file(path_to_file_for_decompile, throw_on_error=True): 55 | print(f'SUCCESS: {path_to_file_for_decompile}') 56 | successful_decompiles += 1 57 | else: 58 | print(f'FAILED: {path_to_file_for_decompile}') 59 | failed_decompiles += 1 60 | except Exception as ex: 61 | print("FAILED to decompile %s" % path_to_file_for_decompile) 62 | failed_decompiles += 1 63 | traceback.print_exc() 64 | if throw_on_error: 65 | raise ex 66 | 67 | @classmethod 68 | def _extract_sub_folder(cls, root: str, filename: str, ea_folder: str, decompile_files: bool=True): 69 | src = os.path.join(root, filename) 70 | dst = os.path.join(ea_folder, filename) 71 | if src != dst: 72 | shutil.copyfile(src, dst) 73 | py_zip = PyZipFile(dst) 74 | out_folder = os.path.join(ea_folder, os.path.splitext(filename)[0]) 75 | py_zip.extractall(out_folder) 76 | if decompile_files: 77 | cls.decompile_folder(out_folder) 78 | pass 79 | 80 | @classmethod 81 | def extract_folder(cls, ea_folder: str, gameplay_folder: str, decompile_files: bool=True): 82 | """Extract a folder.""" 83 | for root, dirs, files in os.walk(gameplay_folder): 84 | for ext_filter in Unpyc3PyDecompiler._script_package_types: 85 | for filename in fnmatch.filter(files, ext_filter): 86 | cls._extract_sub_folder(root, filename, ea_folder, decompile_files=decompile_files) 87 | 88 | -------------------------------------------------------------------------------- /Utilities/unpyc3_compiler.py: -------------------------------------------------------------------------------- 1 | from compile_utils import _remove_files_conflicting_with_decompile, _replace_renamed_files 2 | 3 | _remove_files_conflicting_with_decompile(decompile_ea_scripts=False) 4 | 5 | import traceback 6 | from typing import Iterator 7 | from zipfile import PyZipFile 8 | 9 | from settings import * 10 | 11 | 12 | class Unpyc3PythonCompiler: 13 | """Handles compilation of python files into ts4script files using unpyc3.""" 14 | @classmethod 15 | def compile_mod( 16 | cls, 17 | names_of_modules_include: Iterator[str], 18 | folder_path_to_output_ts4script_to: str, 19 | output_ts4script_name: str, 20 | names_of_modules_to_exclude: str=None, 21 | mod_creator_name: str=None, 22 | folder_path_to_gather_script_modules_from: str='..' 23 | ): 24 | """compile_mod(\ 25 | names_of_modules_include,\ 26 | folder_path_to_output_ts4script_to,\ 27 | output_ts4script_name,\ 28 | names_of_modules_to_exclude=None,\ 29 | mod_creator_name=None,\ 30 | folder_path_to_gather_script_packages_from=None\ 31 | ) 32 | 33 | Compile a mod using unpyc3. 34 | 35 | """ 36 | os.makedirs(folder_path_to_output_ts4script_to, exist_ok=True) 37 | from compile_utils import _remove_files_conflicting_with_decompile, _replace_renamed_files 38 | _remove_files_conflicting_with_decompile(decompile_ea_scripts=False) 39 | names_of_modules_include = tuple(names_of_modules_include) 40 | if not mod_creator_name: 41 | mod_creator_name = creator_name 42 | if not output_ts4script_name: 43 | output_ts4script_name = os.path.join('..', '..', os.path.basename(os.path.normpath(os.path.dirname(os.path.realpath('__file__'))))) 44 | print('No mod name found, setting the path name to \'{}\'.'.format(output_ts4script_name)) 45 | print(f'The current working directory {os.getcwd()}.') 46 | 47 | if mod_creator_name: 48 | print('Mod creator name found, appending mod creator name to file name.') 49 | output_ts4script_name = '{}_{}'.format(mod_creator_name, output_ts4script_name) 50 | output_script_zip_name = '{}.ts4script'.format(output_ts4script_name) 51 | if not folder_path_to_output_ts4script_to: 52 | ts4script = output_script_zip_name 53 | else: 54 | ts4script = os.path.join(folder_path_to_output_ts4script_to, output_script_zip_name) 55 | 56 | # noinspection PyBroadException 57 | try: 58 | if os.path.exists(ts4script): 59 | print('Script archive found, removing found archive.') 60 | os.remove(ts4script) 61 | print('Script archive removed.') 62 | 63 | output_zip = PyZipFile(ts4script, mode='w', allowZip64=True, optimize=2) 64 | previous_working_directory = os.getcwd() 65 | 66 | if folder_path_to_gather_script_modules_from is not None: 67 | print(f'Changing the working directory to \'{folder_path_to_gather_script_modules_from}\'') 68 | os.chdir(folder_path_to_gather_script_modules_from) 69 | else: 70 | folder_path_to_gather_script_modules_from = '..' 71 | os.chdir(folder_path_to_gather_script_modules_from) 72 | print(f'Changed the current working directory \'{os.getcwd()}\'.') 73 | # print('Found child directories {}'.format(pformat(tuple(child_directories)))) 74 | for folder_path in cls._child_directories_gen(os.getcwd()): 75 | # print(f'Attempting to compile {folder_path}') 76 | if names_of_modules_to_exclude is not None and os.path.basename(folder_path) in names_of_modules_to_exclude: 77 | # print(f'Folder is set to be ignored {folder_path}. Continuing to the next folder.') 78 | continue 79 | if names_of_modules_include is not None and os.path.basename(folder_path) not in names_of_modules_include: 80 | # print(f'Folder is not set to be included {folder_path}. Continuing to the next folder.') 81 | continue 82 | # noinspection PyBroadException 83 | try: 84 | print(f'Compiling folder \'{folder_path}\'') 85 | output_zip.writepy(folder_path) 86 | print(f'\'{folder_path}\' compiled successfully.') 87 | except Exception as ex: 88 | print(f'Failed to write {folder_path}. {ex}') 89 | traceback.print_exc() 90 | continue 91 | 92 | print('Done compiling modules.') 93 | output_zip.close() 94 | print('Changing working directory to previous working directory.') 95 | os.chdir(previous_working_directory) 96 | print(f'Changed the current working directory to \'{os.getcwd()}\'') 97 | except Exception as ex: 98 | print(f'Failed to create {ts4script}. {ex}') 99 | return 100 | finally: 101 | _replace_renamed_files(decompile_ea_scripts=False) 102 | 103 | # This code is meant to copy the file to the Mods folder, but in most cases, we probably do not want this to happen! 104 | # ts4script_mods = os.path.join(os.path.join(mods_folder), script_zip_name) 105 | # shutil.copyfile(ts4script, ts4script_mods) 106 | 107 | @classmethod 108 | def _child_directories_gen(cls, directory_path) -> Iterator[str]: 109 | for folder_path in os.listdir(directory_path): 110 | if not os.path.isdir(os.path.join(directory_path, folder_path)): 111 | continue 112 | yield folder_path 113 | 114 | 115 | _replace_renamed_files(decompile_ea_scripts=False) 116 | -------------------------------------------------------------------------------- /Utilities/py37_decompiler.py: -------------------------------------------------------------------------------- 1 | # TS4 Python decompiler front-end 2 | # Original code by Scumbumbo @ Mod The Sims and Andrew @ Sims 4 Studio 3 | # Modified code by ColonolNutty 4 | # 5 | # May be freely modified for personal use 6 | # 7 | # May be redistributed AS-IS or in modified form, providing: 8 | # - All comments and code authorship attributions remain unmodified except as required for clarity 9 | # - This redistribution notice must remain intact 10 | # - The hosted site is free and community-friendly: 11 | # - Host site may be advertising supported to offset costs 12 | # - Host site may not restrict downloads in any way, particularly but not limited to 13 | # - Paid registration requirements (free registrations are okay) 14 | # - Forced download waits due to ad-block detection 15 | # 16 | from typing import List, Any 17 | 18 | from decompilation_method import S4PyDecompilationMethod 19 | from settings import decompile_method_name, game_folder 20 | import os 21 | 22 | # These defaults can be edited here and/or overridden on command line 23 | # 24 | # Location of the game's zipped binary scripts (base.zip, core.zip and simulation.zip) 25 | GAME_ZIP_FOLDER = game_folder 26 | 27 | # Default destination folder to put decompiled scripts into. 28 | # Will be created if not found. 29 | DEFAULT_PY_DESTINATION_FOLDER = './ts4_scripts' 30 | 31 | # You may want to turn this down to 2 or 4 if you don't have a quad-core CPU 32 | DEFAULT_MAX_THREADS = 1 33 | 34 | # Set to either S4PyDecompilationMethod.UNPYC3 or S4PyDecompilationMethod.PY37DEC 35 | DEFAULT_DECOMPILER = decompile_method_name 36 | 37 | """ Command line help: 38 | 39 | decompiler.py - For decompiling The Sims 4 Python modules 40 | 41 | Requires: 42 | - Python version 3.7.0 43 | - py37_execute_decompile.exe executable must be available on the system path or in current folder, 44 | or unpyc3 (with 3.7.0 decompile support) module must be available for import 45 | 46 | usage: decompiler.py [-h] [-z ZIP_FOLDER] [-s SOURCE_FOLDER] [-d DEST_FOLDER] 47 | [-S] [-p] [-t N] [-r [FILENAME]] [-L [N]] 48 | [-c none|detail] [-U] [-P] [-T SEC] 49 | 50 | optional arguments: 51 | -h, --help show this help message and exit 52 | -z ZIP_FOLDER location of installed game Zip files 53 | -s SOURCE_FOLDER get compiled scripts from folder instead of Zip files 54 | -d DEST_FOLDER destination folder for decompilaed files 55 | -S create subfolders of DEST_FOLDER by result 56 | -p prefix output filenames with [RESULT] 57 | -t N number of simultaneous decompile threads to use 58 | -r [FILENAME] create CSV file containing results for decompiled files 59 | -L [N] code objects with >N bytes will not be analyzed 60 | -c none|detail prefix decompiled files with test results comment (default brief) 61 | -U use unpyc3 for decompilation 62 | -P use py37dec for decompilation 63 | -T SEC py37dec only: override timeout in seconds (0=no limit, default 5) 64 | """ 65 | 66 | 67 | ################################################################# 68 | # DO NOT EDIT BELOW THIS LINE WITHOUT KNOWING WHAT YOU'RE DOING # 69 | ################################################################# 70 | import subprocess 71 | import time 72 | import traceback 73 | import dis 74 | import marshal 75 | import types 76 | import difflib 77 | import re 78 | import sys 79 | import argparse 80 | import shutil 81 | import zipfile 82 | 83 | if DEFAULT_DECOMPILER != S4PyDecompilationMethod.PY37DEC and DEFAULT_DECOMPILER != S4PyDecompilationMethod.UNPYC3: 84 | print('Invalid setting for DEFAULT_DECOMPILER in source') 85 | exit() 86 | if os.path.isfile('./py37_execute_decompile.exe'): 87 | PY37DEC_EXECUTABLE_LOCATION = './py37_execute_decompile.exe' 88 | elif os.path.isfile('./py37_execute_decompile'): 89 | PY37DEC_EXECUTABLE_LOCATION = './py37_execute_decompile' 90 | else: 91 | PY37DEC_EXECUTABLE_LOCATION = shutil.which('py37_execute_decompile') 92 | PY37DEC_AVAILABLE = True if PY37DEC_EXECUTABLE_LOCATION is not None else False 93 | UNPYC3_AVAILABLE = False 94 | DECOMPILER = None 95 | 96 | try: 97 | import unpyc3 98 | UNPYC3_AVAILABLE = True 99 | except: 100 | pass 101 | 102 | 103 | # Quick 'n' dirty stopwatch timer 104 | class _StopWatch: 105 | def __init__(self) -> None: 106 | self.start_time = time.perf_counter() 107 | 108 | @property 109 | def elapsed_time(self) -> float: 110 | return time.perf_counter() - self.start_time 111 | 112 | 113 | # Values returned from a "thread" to indicate results. Since the Python VM doesn't really 114 | # support true threads and shared data, the results are encapsulated in this and the Python 115 | # "threading" library returns this information to the main thread via an OS dependent mechanism. 116 | class _DecompileResultData: 117 | def __init__(self, pyc_file_name: str): 118 | self._pyc_file_name = pyc_file_name 119 | self.py_file_name = '' 120 | self.decompile_time = -1 121 | self.analyze_time = -1 122 | self.result = -1 123 | 124 | @property 125 | def pyc_file_name(self) -> str: 126 | return self._pyc_file_name 127 | 128 | 129 | class Py37PythonDecompiler: 130 | """Handles decompilation of python files from pyc to py using py37dec.""" 131 | # Code object comparison routines based on code written by Andrew from Sims 4 Studio 132 | # 133 | # Handle formatting dis() output of a code object in order to run through a diff process. 134 | _CODE_OBJECT_REGEX = re.compile(r'(.*)\(\\)', re.RegexFlag.IGNORECASE) 135 | 136 | # Launch "threads" and summarize results 137 | def decompile(self, src_folder, dest_folder, prefix_filenames=False, max_threads=DEFAULT_MAX_THREADS, results_file=None, test_large_codeobjects=False, large_codeobjects_threshold=10000, comment_style=0, py37dec_timeout=5, split_result_folders=False): 138 | global total, DEFAULT_DECOMPILER 139 | from shutil import copyfile 140 | from Utilities.unpyc3_decompiler import Unpyc3PyDecompiler 141 | 142 | timer = _StopWatch() 143 | if py37dec_timeout == 0: 144 | py37dec_timeout = None 145 | 146 | # Create our "thread" pool 147 | results = [] 148 | 149 | print('Decompiling all files in {} using {}, please wait'.format(src_folder, DEFAULT_DECOMPILER)) 150 | 151 | # Search the source folder for all .pyc files and add a call to decompile() 152 | # to the "thread" pool. 153 | source_folder = os.path.realpath(src_folder) 154 | destination_folder = os.path.realpath(dest_folder) 155 | if source_folder == destination_folder: 156 | destination_folder = os.path.join(destination_folder, '_decompiled') 157 | if not os.path.exists(destination_folder): 158 | os.mkdir(destination_folder) 159 | else: 160 | def _remove_directory_recursive(directory_path: str): 161 | for _file_in_dir in os.listdir(directory_path): 162 | _to_remove_path = os.path.join(directory_path, _file_in_dir) 163 | if os.path.isdir(_to_remove_path): 164 | # noinspection PyBroadException 165 | try: 166 | os.rmdir(_to_remove_path) 167 | except: 168 | _remove_directory_recursive(_to_remove_path) 169 | os.rmdir(_to_remove_path) 170 | else: 171 | os.remove(_to_remove_path) 172 | 173 | # noinspection PyBroadException 174 | try: 175 | os.rmdir(destination_folder) 176 | except: 177 | _remove_directory_recursive(destination_folder) 178 | os.rmdir(destination_folder) 179 | os.mkdir(destination_folder) 180 | 181 | for root, subFolders, files in os.walk(src_folder): 182 | if root.startswith(destination_folder): 183 | continue 184 | files = [f for f in files if os.path.splitext(f)[1].lower() == '.pyc'] 185 | for pycFile in files: 186 | total += 1 187 | sub_folder = os.path.relpath(root, source_folder) 188 | file_name = os.path.splitext(pycFile)[0] 189 | py_file_name = file_name + '.py' 190 | py_full_file_path = os.path.join(source_folder, sub_folder, py_file_name) 191 | py_full_destination_file_path = os.path.join(destination_folder, sub_folder, py_file_name) 192 | copied_file_path = os.path.join(source_folder, sub_folder, f'{pycFile}_copied') 193 | pyc_full_file_path = os.path.join(source_folder, sub_folder, pycFile) 194 | copyfile(pyc_full_file_path, copied_file_path) 195 | the_file = os.path.join(src_folder, sub_folder, pycFile) 196 | # print(f'Attempting to decompile file: {os.path.join(src_folder, sub_folder, pycFile)}') 197 | result = self._decompile( 198 | source_folder, 199 | destination_folder, 200 | sub_folder, 201 | pycFile, 202 | py_file_name, 203 | prefix_filenames, 204 | large_codeobjects_threshold, 205 | comment_style, 206 | DECOMPILER, 207 | py37dec_timeout, 208 | split_result_folders 209 | ) 210 | if not is_success(result): 211 | print('Failed to decompile file, attempting to use alternative decompiler.') 212 | copyfile(copied_file_path, pyc_full_file_path) 213 | os.remove(copied_file_path) 214 | print(pyc_full_file_path) 215 | if not Unpyc3PyDecompiler.decompile_file(pyc_full_file_path, throw_on_error=False): 216 | print(f'FAILED: {the_file}') 217 | else: 218 | print(f'SUCCESS: {the_file}') 219 | # print(f'Removing {py_full_file_path}') 220 | copyfile(py_full_file_path, py_full_destination_file_path) 221 | os.remove(py_full_file_path) 222 | result = _DecompileResultData(os.path.realpath(pyc_full_file_path)) 223 | result.result = 1 224 | else: 225 | print(f'SUCCESS: {the_file}') 226 | os.remove(copied_file_path) 227 | completed_callback(result) 228 | results.append(result) 229 | 230 | # Print results summary and CSV results file if requested 231 | print('\b\b\b\b\b\b') 232 | if total == 0: 233 | print(f' \nError, no compiled Python files found in source folder {source_folder}') 234 | return 235 | print('Completed') 236 | 237 | print('\nperfect\t= {} ({:0.1f}%)'.format(len(perfect), len(perfect)/total*100)) 238 | print('good\t= {} ({:0.1f}%)'.format(len(good), len(good)/total*100)) 239 | print('syntax\t= {} ({:0.1f}%)'.format(len(syntax), len(syntax)/total*100)) 240 | print('failure\t= {} ({:0.1f}%)'.format(len(failed), len(failed)/total*100)) 241 | if len(timeout) > 0: 242 | print('timeout\t= {} ({:0.1f}%)'.format(len(timeout), len(timeout)/total*100)) 243 | print('{:0.2f} seconds'.format(timer.elapsed_time)) 244 | 245 | if results_file: 246 | with open(results_file, 'w', encoding='UTF-8') as fp: 247 | fp.write(' ,Compiled,Decompiled,Decompile,Compare\nResult,Path,Path,Time,Time\n') 248 | for decompile_result in perfect: 249 | fp.write('PERFECT,{},{},{},{}\n'.format(os.path.relpath(decompile_result.pyc_file_name, src_folder), os.path.relpath(decompile_result.pyFilename, dest_folder), decompile_result.decompile_time, decompile_result.analyze_time)) 250 | for decompile_result in good: 251 | fp.write('GOOD,{},{},{},{}\n'.format(os.path.relpath(decompile_result.pyc_file_name, src_folder), os.path.relpath(decompile_result.pyFilename, dest_folder), decompile_result.decompile_time, decompile_result.analyze_time)) 252 | for decompile_result in failed: 253 | fp.write('FAILED,{},{},{},{}\n'.format(os.path.relpath(decompile_result.pyc_file_name, src_folder), os.path.relpath(decompile_result.pyFilename, dest_folder), decompile_result.decompile_time, decompile_result.analyze_time)) 254 | for decompile_result in syntax: 255 | fp.write('SYNTAX,{},{},{},{}\n'.format(os.path.relpath(decompile_result.pyc_file_name, src_folder), os.path.relpath(decompile_result.pyFilename, dest_folder), decompile_result.decompile_time, decompile_result.analyze_time)) 256 | for decompile_result in timeout: 257 | fp.write('TIMEOUT,{},{},{},{}\n'.format(os.path.relpath(decompile_result.pyc_file_name, src_folder), os.path.relpath(decompile_result.pyFilename, dest_folder), decompile_result.decompile_time, decompile_result.analyze_time)) 258 | 259 | # Decompile a .pyc file to produce a .py file 260 | # Returns a _DecompileResultData encapsulation of the result information. 261 | def _decompile(self, srcFolder: str, destination_folder: str, subFolder: str, pycFile: str, python_file_to_decompile, prefix_filenames, large_codeobjects_threshold, comment_style, decompiler, py37dec_timeout, split_result_folders): 262 | global PY37DEC_EXECUTABLE_LOCATION 263 | decompile_results = _DecompileResultData(os.path.realpath(os.path.join(srcFolder, subFolder, pycFile))) 264 | timer = _StopWatch() 265 | remove_pyc = True if srcFolder == destination_folder else False 266 | pyc_full_file_path = os.path.join(srcFolder, subFolder, pycFile) 267 | 268 | # noinspection PyBroadException 269 | try: 270 | if decompiler == S4PyDecompilationMethod.UNPYC3: 271 | # For unpyc3, just call the decompile() method from that module 272 | src_code = '' 273 | lines = unpyc3.decompile(pyc_full_file_path) 274 | for line in lines: 275 | src_code += str(line) + '\n' 276 | else: 277 | # For py37dec, run the executable (py37_execute_decompile.exe) in a subprocess. At least one file from TS4 still takes 278 | # too long (and too much virtual memory) to process, so a timeout is specified. 279 | subprocess_result = subprocess.run( 280 | [ 281 | PY37DEC_EXECUTABLE_LOCATION, 282 | pyc_full_file_path.replace('\\', '/') 283 | ], 284 | capture_output=True, 285 | encoding='utf-8', 286 | timeout=py37dec_timeout 287 | ) 288 | decompile_results.decompile_time = timer.elapsed_time 289 | if subprocess_result.returncode != 0: 290 | # Non-zero return code from the py37dec executable (py37_execute_decompile.exe) indicates a crash failure 291 | # in the executable. Summarize and build an empty .py file. 292 | if prefix_filenames: 293 | python_file_to_decompile = f'[FAILED] {python_file_to_decompile}' 294 | if split_result_folders: 295 | python_folder_to_decompile = os.path.join(destination_folder, 'decompile_failure', subFolder) 296 | else: 297 | python_folder_to_decompile = os.path.join(destination_folder, subFolder) 298 | os.makedirs(python_folder_to_decompile, exist_ok=True) 299 | decompile_results.py_file_name = os.path.realpath(os.path.join(python_folder_to_decompile, python_file_to_decompile)) 300 | with open(decompile_results.py_file_name, 'w', encoding='UTF-8') as fp: 301 | if comment_style == 1: 302 | fp.write(f'# {decompiler}: Decompile failed\n') 303 | elif comment_style == 2: 304 | fp.write('"""\npy37dec: Decompilation failure\n\n') 305 | fp.write(subprocess_result.stderr) 306 | fp.write('"""\n') 307 | decompile_results.result = 3 308 | if remove_pyc: 309 | os.remove(pyc_full_file_path) 310 | return decompile_results 311 | # Rc = 0 from subprocess, so read the source code lines from the subprocess stdout 312 | src_code = subprocess_result.stdout 313 | except subprocess.TimeoutExpired: 314 | # This exception will only occur if a py37dec subprocess is killed off due to a timeout. 315 | decompile_results.decompile_time = timer.elapsed_time 316 | if prefix_filenames: 317 | python_file_to_decompile = f'[TIMEOUT] {python_file_to_decompile}' 318 | if split_result_folders: 319 | python_folder_to_decompile = os.path.join(destination_folder, 'timeout', subFolder) 320 | else: 321 | python_folder_to_decompile = os.path.join(destination_folder, subFolder) 322 | os.makedirs(python_folder_to_decompile, exist_ok=True) 323 | decompile_results.py_file_name = os.path.realpath(os.path.join(python_folder_to_decompile, python_file_to_decompile)) 324 | decompile_results.result = 4 325 | with open(decompile_results.py_file_name, 'w', encoding='UTF-8') as fp: 326 | if comment_style == 1: 327 | fp.write('# py37dec: Timeout\n') 328 | elif comment_style == 2: 329 | fp.write(f'"""\npy37dec: Timeout of {py37dec_timeout} seconds exceeded\n"""\n') 330 | if remove_pyc: 331 | os.remove(pyc_full_file_path) 332 | return decompile_results 333 | except Exception as ex: 334 | # A normal exception will occur if unpyc3 fails and throws an exception during the 335 | # decompilation process. 336 | if prefix_filenames: 337 | python_file_to_decompile = f'[FAILED] {python_file_to_decompile}: {ex}' 338 | if split_result_folders: 339 | python_folder_to_decompile = os.path.join(destination_folder, 'decompile_failure', subFolder) 340 | else: 341 | python_folder_to_decompile = os.path.join(destination_folder, subFolder) 342 | os.makedirs(python_folder_to_decompile, exist_ok=True) 343 | decompile_results.py_file_name = os.path.realpath(os.path.join(python_folder_to_decompile, python_file_to_decompile)) 344 | with open(decompile_results.py_file_name, 'w', encoding='UTF-8') as fp: 345 | if comment_style == 1: 346 | fp.write(f'# {decompiler}: Decompile failed\n') 347 | elif comment_style == 2: 348 | fp.write('"""\nunpyc3: Decompilation failure\n\n') 349 | fp.write(traceback.format_exc()) 350 | fp.write('"""\n') 351 | decompile_results.result = 3 352 | if remove_pyc: 353 | os.remove(pyc_full_file_path) 354 | return decompile_results 355 | 356 | decompile_results.decompile_time = timer.elapsed_time 357 | 358 | syntax_error = None 359 | # noinspection PyBroadException 360 | try: 361 | # Try compiling the generated source, a syntax error in the source code 362 | # will throw an exception. 363 | py_compiled_obj = compile(src_code, python_file_to_decompile, 'exec') 364 | 365 | # Get the code object from the .pyc file 366 | pyc_compiled_obj = self._get_code_obj_from_pyc(decompile_results.pyc_file_name) 367 | 368 | # Compare the code objects recursively 369 | issues = self._compare_code_objs(pyc_compiled_obj, py_compiled_obj, large_codeobjects_threshold) 370 | decompile_results.analyze_time = timer.elapsed_time - decompile_results.decompile_time 371 | 372 | if not issues: 373 | # There were no issues returned from the code object comparison, so this code 374 | # is identical to the original sources. 375 | if prefix_filenames: 376 | python_file_to_decompile = '[PERFECT] ' + python_file_to_decompile 377 | if split_result_folders: 378 | python_folder_to_decompile = os.path.join(destination_folder, 'perfect', subFolder) 379 | else: 380 | python_folder_to_decompile = os.path.join(destination_folder, subFolder) 381 | decompile_results.py_file_name = os.path.realpath(os.path.join(python_folder_to_decompile, python_file_to_decompile)) 382 | decompile_results.result = 0 383 | else: 384 | # There were comparison issues with the code objects, this source code differs 385 | # from the original. It may function identically or improperly (or not at all) but 386 | # only human inspection of the resulting code can determine how good the results are. 387 | if prefix_filenames: 388 | python_file_to_decompile = '[GOOD] ' + python_file_to_decompile 389 | if split_result_folders: 390 | python_folder_to_decompile = os.path.join(destination_folder, 'good', subFolder) 391 | else: 392 | python_folder_to_decompile = os.path.join(destination_folder, subFolder) 393 | decompile_results.py_file_name = os.path.realpath(os.path.join(python_folder_to_decompile, python_file_to_decompile)) 394 | decompile_results.result = 1 395 | except Exception as ex: 396 | # An exception from the compile or comparison will end up here, this is generally 397 | # due to a syntax error in the decompilation results. 398 | if prefix_filenames: 399 | python_file_to_decompile = f'[SYNTAX] {python_file_to_decompile}: {ex}' 400 | if split_result_folders: 401 | python_folder_to_decompile = os.path.join(destination_folder, 'syntax', subFolder) 402 | else: 403 | python_folder_to_decompile = os.path.join(destination_folder, subFolder) 404 | decompile_results.py_file_name = os.path.realpath(os.path.join(python_folder_to_decompile, python_file_to_decompile)) 405 | decompile_results.result = 2 406 | syntax_error = traceback.format_exc(1) 407 | 408 | # Create the destination folder for this .py file and write it, adding comments 409 | # if requested (1 = brief, 2 = detailed). 410 | os.makedirs(python_folder_to_decompile, exist_ok=True) 411 | with open(decompile_results.py_file_name, 'w', encoding='UTF-8') as fp: 412 | if comment_style == 1: 413 | if syntax_error: 414 | fp.write('# {}: Syntax error in decompiled file\n'.format(decompiler)) 415 | elif issues: 416 | fp.write('# {}: Decompiled file contains inaccuracies\n'.format(decompiler)) 417 | else: 418 | fp.write('# {}: 100% Accurate decompile result\n'.format(decompiler)) 419 | elif comment_style == 2: 420 | if syntax_error: 421 | fp.write(f'"""\n{decompiler}:\n{syntax_error}"""\n') 422 | elif issues: 423 | fp.write(f'"""\n{decompiler}:\n{issues}"""\n') 424 | else: 425 | fp.write(f'# {decompiler}: 100% Accurate decompile result\n') 426 | fp.write(src_code) 427 | 428 | if remove_pyc: 429 | os.remove(pyc_full_file_path) 430 | return decompile_results 431 | 432 | # Perform the actual code object comparisons 433 | # Returns a string of errors found, or an empty string for a perfect comparison result 434 | def _compare_code_objs(self, pyc_co, py_co, large_codeobjects_threshold): 435 | # Large code objects can take a significant time to process with diff(), so skipped by default 436 | if large_codeobjects_threshold and len(py_co.co_code) > large_codeobjects_threshold: 437 | return '{0}\nSKIPPING COMPARISON OF LARGE CODE OBJECT\n\t{1}\n{0}\n'.format('='*80, pyc_co.co_name) 438 | err_str = '' 439 | flags_err_str = '' 440 | names_err_str = '' 441 | consts_err_str = '' 442 | args_err_str = '' 443 | locals_err_str = '' 444 | if pyc_co.co_flags != py_co.co_flags: 445 | flags_err_str = 'Differings flags: {} != {}\n'.format(pyc_co.co_flags, py_co.co_flags) 446 | if (pyc_co.co_flags & 0x4) == 0x4 and (py_co.co_flags & 0x4) == 0: 447 | flags_err_str += '\tMissing expected *args\n' 448 | if (pyc_co.co_flags & 0x8) == 0x8 and (py_co.co_flags & 0x8) == 0: 449 | flags_err_str += '\tMissing expected **kwargs\n' 450 | if (pyc_co.co_flags & 0x20) != (py_co.co_flags & 0x20): 451 | if pyc_co.co_flags & 0x20 == 0x20: 452 | flags_err_str += '\tShould be generator function but is not\n' 453 | else: 454 | flags_err_str += '\tIs generator function but should not be\n' 455 | if pyc_co.co_kwonlyargcount != py_co.co_kwonlyargcount: 456 | args_err_str = 'Differing kwonlyargcount: {} != {}\n'.format(pyc_co.co_kwonlyargcount, py_co.co_kwonlyargcount) 457 | if pyc_co.co_argcount != py_co.co_argcount: 458 | args_err_str += 'Differing argcount: {} != {}\n'.format(pyc_co.co_argcount, py_co.co_argcount) 459 | if len(py_co.co_names) != len(pyc_co.co_names): 460 | names_err_str += 'Differing number of global names: {} != {}\n'.format(len(pyc_co.co_names), len(py_co.co_names)) 461 | names_err_str += '\tExpected: {}\n\tActual: {}\n'.format(pyc_co.co_names, py_co.co_names) 462 | else: 463 | current_cell_index = 0 464 | # Compare all constant names to ensure equality 465 | for name in pyc_co.co_names: 466 | if len(py_co.co_names) < current_cell_index + 1: 467 | names_err_str += 'Unable to compare global name {}. Does not exist in the decompiled version\n'.format(name) 468 | elif name != py_co.co_names[current_cell_index]: 469 | names_err_str += 'Global name: {} != {}\n'.format(name, py_co.co_names[current_cell_index]) 470 | current_cell_index += 1 471 | if len(py_co.co_consts) != len(pyc_co.co_consts): 472 | consts_err_str += 'Differing number of constants: {} != {}\n'.format(len(pyc_co.co_consts), len(py_co.co_consts)) 473 | consts_err_str += '\tExpected: {}\n\tActual: {}\n'.format(pyc_co.co_consts, py_co.co_consts) 474 | else: 475 | current_cell_index = 0 476 | # Compare all constants to ensure equality 477 | for constant in pyc_co.co_consts: 478 | if len(py_co.co_consts) < current_cell_index + 1: 479 | consts_err_str += 'Unable to compare constant {}. Does not exist in the decompiled version\n'.format(constant) 480 | elif type(constant) is types.CodeType: 481 | if type(py_co.co_consts[current_cell_index]) is types.CodeType: 482 | err_str += self._compare_code_objs(constant, py_co.co_consts[current_cell_index], large_codeobjects_threshold) 483 | else: 484 | consts_err_str += 'Constants mismatched: unable to compare code object {} to non-code object {}\n'.format(constant, py_co.co_consts[current_cell_index]) 485 | elif constant != py_co.co_consts[current_cell_index]: 486 | consts_err_str += 'Constant: {} != {}\n'.format(constant, py_co.co_consts[current_cell_index]) 487 | current_cell_index += 1 488 | if py_co.co_nlocals != pyc_co.co_nlocals: 489 | locals_err_str += 'Differing number of locals: {} != {}\n'.format(pyc_co.co_nlocals, py_co.co_nlocals) 490 | locals_err_str += '\tExpected: {}\n\tActual: {}\n'.format(pyc_co.co_varnames, py_co.co_varnames) 491 | else: 492 | current_cell_index = 0 493 | # Compare all local names to ensure equality 494 | for name in pyc_co.co_varnames: 495 | if py_co.co_nlocals < current_cell_index + 1: 496 | locals_err_str += 'Unable to compare local var name {}. Does not exist in the decompiled version\n'.format(constant) 497 | elif name != py_co.co_varnames[current_cell_index]: 498 | locals_err_str += 'Local var name: {} != {}\n'.format(name, py_co.co_varnames[current_cell_index]) 499 | current_cell_index += 1 500 | if len(py_co.co_cellvars) != len(pyc_co.co_cellvars): 501 | locals_err_str += 'Differing number of cellvars: {} != {}\n'.format(len(pyc_co.co_cellvars), len(py_co.co_cellvars)) 502 | locals_err_str += '\tExpected: {}\n\tActual: {}\n'.format(pyc_co.co_cellvars, py_co.co_cellvars) 503 | else: 504 | current_cell_index = 0 505 | # Compare all cellvar names to ensure equality 506 | for name in pyc_co.co_cellvars: 507 | if len(py_co.co_cellvars) < current_cell_index + 1: 508 | locals_err_str += 'Unable to compare cellvar name {}. Does not exist in the decompiled version\n'.format(constant) 509 | elif name != py_co.co_cellvars[current_cell_index]: 510 | locals_err_str += 'Cellvar name: {} != {}\n'.format(name, py_co.co_cellvars[current_cell_index]) 511 | current_cell_index += 1 512 | if flags_err_str or args_err_str or names_err_str or consts_err_str or locals_err_str: 513 | err_str += '{0}\n{1}\n{0}\n'.format('='*80, pyc_co.co_name) 514 | if flags_err_str: 515 | err_str += flags_err_str 516 | if args_err_str: 517 | err_str += args_err_str 518 | if names_err_str: 519 | err_str += names_err_str 520 | if consts_err_str: 521 | err_str += consts_err_str 522 | if locals_err_str: 523 | err_str += locals_err_str 524 | a = self._format_dis_lines(pyc_co) 525 | b = self._format_dis_lines(py_co) 526 | d = list(difflib.unified_diff(a, b)) 527 | if any(d): 528 | err_str += '{0}\n{1}\n{0}\nEXPECTED:\n\t{2}\n{0}\nACTUAL:\n\t{3}\n{0}\nDIFF:\n\t{4}\n{0}\n'.format('='*80, pyc_co.co_name, str.join('\n\t', a), str.join('\n\t', b), str.join('\n\t', d)) 529 | return err_str 530 | 531 | def _format_dis_lines(self, co) -> List[Any]: 532 | def _remove_line_number(s: str): 533 | ix = s.index(' ', 5) 534 | x = s[ix:].lstrip(' ') 535 | return x 536 | 537 | def _clean_code_object_line(s: str): 538 | # strip out any line numbers, file names, or offsets 539 | b = Py37PythonDecompiler._CODE_OBJECT_REGEX.match(s) 540 | if b: 541 | return s.replace(b.group(0), f'{b.group(1)}<{b.group(2)}>') 542 | return s 543 | 544 | return list( 545 | map( 546 | _clean_code_object_line, 547 | map( 548 | _remove_line_number, 549 | filter( 550 | None, 551 | dis.Bytecode(co).dis().split('\n') 552 | ) 553 | ) 554 | ) 555 | ) 556 | 557 | # Reads the code object from a compiled Python (.pyc) file 558 | def _get_code_obj_from_pyc(self, file_name: str) -> Any: 559 | with open(file_name, 'rb') as file_data: 560 | code_obj = marshal.loads(file_data.read()[16:]) 561 | return code_obj 562 | 563 | 564 | # 565 | # The following all runs in the main "thread" 566 | # 567 | # Result buckets and counters for the _DecompileResultData from each thread 568 | perfect = [] 569 | good = [] 570 | syntax = [] 571 | failed = [] 572 | timeout = [] 573 | completed = 0 574 | total = 0 575 | 576 | # A completed thread will issue this callback in the main thread, place the 577 | # _DecompileResultData into a bucket depending on the returned result. 578 | def completed_callback(result) -> bool: 579 | global completed, total 580 | completed += 1 581 | 582 | # Write percentage complete to stdout 583 | #print('\b\b\b\b{:3}%'.format(int(completed/total*100))) 584 | #sys.stdout.flush() 585 | 586 | if result.result == 0: 587 | perfect.append(result) 588 | return True 589 | elif result.result == 1: 590 | good.append(result) 591 | return True 592 | elif result.result == 2: 593 | print('syntax') 594 | syntax.append(result) 595 | return False 596 | elif result.result == 3: 597 | print('failed') 598 | failed.append(result) 599 | return False 600 | else: 601 | print('timeout') 602 | timeout.append(result) 603 | return False 604 | 605 | 606 | def is_success(result) -> bool: 607 | print('The result: ' + str(result.result)) 608 | if result.result == 0: 609 | return True 610 | elif result.result == 1: 611 | return True 612 | elif result.result == 2: 613 | return False 614 | elif result.result == 3: 615 | return False 616 | else: 617 | return False 618 | 619 | # Unzips the script files (.pyc) from the TS4 game executable folders into the destination folder. 620 | def unzip_script_files(zip_folder, dest_folder): 621 | print('Extracting Zip files from game, please wait.') 622 | for file in ['base.zip', 'core.zip', 'simulation.zip']: 623 | zip = zipfile.ZipFile(os.path.join(zip_folder, file)) 624 | zip.extractall(os.path.join(dest_folder, os.path.splitext(file)[0])) 625 | if os.name == 'posix': 626 | # Mac location for generated.zip 627 | generated_folder = os.path.realpath(os.path.join(zip_folder, os.path.join('..', '..' '..' '..', 'Python'))) 628 | else: 629 | # Windows location for generated.zip 630 | generated_folder = os.path.realpath(os.path.join(zip_folder, '..', '..', '..', '..', 'Game', 'Bin', 'Python')) 631 | zip = zipfile.ZipFile(os.path.join(generated_folder, 'generated.zip')) 632 | zip.extractall(os.path.join(dest_folder, 'generated')) 633 | 634 | 635 | # Setup and parse command line options, calling main() with all desired options 636 | if __name__ == '__main__': 637 | if sys.version_info[0] != 3 or sys.version_info[1] != 7 or sys.version_info[2] != 0: 638 | print('Decompiler requires Python version 3.7.0 for proper results') 639 | exit() 640 | 641 | # If the default decompiler is not available, override that and print a warning 642 | if DEFAULT_DECOMPILER == S4PyDecompilationMethod.PY37DEC: 643 | if not PY37DEC_AVAILABLE: 644 | if UNPYC3_AVAILABLE: 645 | print('py37dec unavailable, will use unpyc3 as default') 646 | DECOMPILER = S4PyDecompilationMethod.UNPYC3 647 | else: 648 | DECOMPILER = S4PyDecompilationMethod.PY37DEC 649 | else: 650 | if not UNPYC3_AVAILABLE: 651 | if PY37DEC_AVAILABLE: 652 | print('unpyc3 unavailable, will use py37dec as default') 653 | DECOMPILER = S4PyDecompilationMethod.PY37DEC 654 | else: 655 | DECOMPILER = S4PyDecompilationMethod.UNPYC3 656 | if not DECOMPILER: 657 | print('No decompiler is available, please install unpyc3 or py37dec') 658 | exit() 659 | 660 | parser = argparse.ArgumentParser() 661 | parser.add_argument('-z', nargs=1, metavar='ZIP_FOLDER', default=[GAME_ZIP_FOLDER], dest='zip_folder', help='location of installed game Zip files') 662 | parser.add_argument('-s', nargs=1, metavar='SOURCE_FOLDER', default=[None], dest='src_folder', help='get compiled scripts from folder instead of Zip files') 663 | parser.add_argument('-d', nargs=1, metavar='DEST_FOLDER', default=[DEFAULT_PY_DESTINATION_FOLDER], dest='dest_folder', help='destination folder for decompilaed files') 664 | parser.add_argument('-S', action='store_true', dest='split_result_folders', help='create subfolders of DEST_FOLDER by result') 665 | parser.add_argument('-p', action='store_true', dest='prefix_filenames', help='prefix output filenames with [RESULT]') 666 | parser.add_argument('-r', nargs='?', metavar='FILENAME', default=argparse.SUPPRESS, dest='results_file', help='create CSV file containing results for decompiled files') 667 | parser.add_argument('-L', nargs='?', type=int, metavar='N', default=argparse.SUPPRESS, dest='large_codeobjects_threshold', help='code objects with >N bytes will not be analyzed') 668 | parser.add_argument('-c', nargs=1, metavar='none|detail', choices=['none', 'detail'], default=argparse.SUPPRESS, dest='comment_style', help='prefix decompiled files with test results comment (default brief)') 669 | if UNPYC3_AVAILABLE: 670 | parser.add_argument('-U', action='store_true', dest='use_unpyc3', help='use unpyc3 for decompilation') 671 | if PY37DEC_AVAILABLE: 672 | parser.add_argument('-P', action='store_true', dest='use_py37dec', help='use py37dec for decompilation') 673 | parser.add_argument('-T', nargs=1, type=int, metavar='SEC', default=[5], dest='py37dec_timeout', help='py37dec only: override timeout in seconds (0=no limit, default 5)') 674 | 675 | args = parser.parse_args() 676 | if hasattr(args, 'use_unpyc3') and hasattr(args, 'use_py37dec') and args.use_unpyc3 and args.use_py37dec: 677 | parser.print_help() 678 | print('\n-U and -P options conflict, please use only one') 679 | exit() 680 | if hasattr(args, 'use_unpyc3') and args.use_unpyc3: 681 | DECOMPILER = S4PyDecompilationMethod.UNPYC3 682 | if hasattr(args, 'use_py37dec') and args.use_py37dec: 683 | DECOMPILER = S4PyDecompilationMethod.PY37DEC 684 | if hasattr(args, 'results_file'): 685 | if args.results_file is None: 686 | args.results_file = 'results.csv' 687 | else: 688 | args.results_file = None 689 | if hasattr(args, 'large_codeobjects_threshold'): 690 | if args.large_codeobjects_threshold is None: 691 | args.large_codeobjects_threshold = 10000 692 | else: 693 | args.large_codeobjects_threshold = None 694 | comment_style = 1 695 | if hasattr(args, 'comment_style'): 696 | if args.comment_style[0] == 'none': 697 | comment_style = 0 698 | else: 699 | comment_style = 2 700 | if not hasattr(args, 'py37dec_timeout'): 701 | args.py37dec_timeout = [0] 702 | if args.src_folder[0] is None: 703 | unzip_script_files(args.zip_folder[0], args.dest_folder[0]) 704 | args.src_folder[0] = args.dest_folder[0] 705 | Py37PythonDecompiler().decompile( 706 | args.src_folder[0], 707 | args.dest_folder[0], 708 | prefix_filenames=args.prefix_filenames, 709 | results_file=args.results_file, 710 | large_codeobjects_threshold=args.large_codeobjects_threshold, 711 | comment_style=comment_style, 712 | py37dec_timeout=args.py37dec_timeout[0], 713 | split_result_folders=args.split_result_folders 714 | ) 715 | -------------------------------------------------------------------------------- /Utilities/unpyc3.py: -------------------------------------------------------------------------------- 1 | """ 2 | Decompiler for Python3.7. 3 | Decompile a module or a function using the decompile() function 4 | 5 | >>> from unpyc3 import decompile 6 | >>> def foo(x, y, z=3, *args): 7 | ... global g 8 | ... for i, j in zip(x, y): 9 | ... if z == i + j or args[i] == j: 10 | ... g = i, j 11 | ... return 12 | ... 13 | >>> print(decompile(foo)) 14 | 15 | def foo(x, y, z=3, *args): 16 | global g 17 | for i, j in zip(x, y): 18 | if z == i + j or args[i] == j: 19 | g = i, j 20 | return 21 | >>> 22 | """ 23 | from __future__ import annotations 24 | 25 | from typing import Union 26 | 27 | __all__ = ['decompile'] 28 | 29 | 30 | def set_trace(trace_function): 31 | global current_trace 32 | current_trace = trace_function if trace_function else _trace 33 | 34 | 35 | def get_trace(): 36 | global current_trace 37 | return None if current_trace == _trace else current_trace 38 | 39 | 40 | def trace(*args): 41 | global current_trace 42 | if current_trace: 43 | current_trace(*args) 44 | 45 | 46 | def _trace(*args): 47 | pass 48 | 49 | 50 | current_trace = _trace 51 | 52 | # TODO: 53 | # - Support for keyword-only arguments 54 | # - Handle assert statements better 55 | # - (Partly done) Nice spacing between function/class declarations 56 | 57 | import dis 58 | from array import array 59 | from opcode import opname, opmap, HAVE_ARGUMENT, cmp_op 60 | import inspect 61 | 62 | import struct 63 | import sys 64 | 65 | # Masks for code object's co_flag attribute 66 | VARARGS = 4 67 | VARKEYWORDS = 8 68 | 69 | # Put opcode names in the global namespace 70 | for name, val in opmap.items(): 71 | globals()[name] = val 72 | 73 | # These opcodes will generate a statement. This is used in the first 74 | # pass (in Code.find_else) to find which POP_JUMP_IF_* instructions 75 | # are jumps to the else clause of an if statement 76 | stmt_opcodes = { 77 | SETUP_LOOP, BREAK_LOOP, CONTINUE_LOOP, 78 | SETUP_FINALLY, END_FINALLY, 79 | SETUP_EXCEPT, POP_EXCEPT, 80 | SETUP_WITH, 81 | POP_BLOCK, 82 | STORE_FAST, DELETE_FAST, 83 | STORE_DEREF, DELETE_DEREF, 84 | STORE_GLOBAL, DELETE_GLOBAL, 85 | STORE_NAME, DELETE_NAME, 86 | STORE_ATTR, DELETE_ATTR, 87 | IMPORT_NAME, IMPORT_FROM, 88 | RETURN_VALUE, YIELD_VALUE, 89 | RAISE_VARARGS, 90 | POP_TOP, 91 | } 92 | 93 | # Conditional branching opcode that make up if statements and and/or 94 | # expressions 95 | pop_jump_if_opcodes = (POP_JUMP_IF_TRUE, POP_JUMP_IF_FALSE) 96 | 97 | # These opcodes indicate that a pop_jump_if_x to the address just 98 | # after them is an else-jump 99 | else_jump_opcodes = ( 100 | JUMP_FORWARD, RETURN_VALUE, JUMP_ABSOLUTE, 101 | SETUP_LOOP, RAISE_VARARGS, POP_TOP 102 | ) 103 | 104 | # These opcodes indicate for loop rather than while loop 105 | for_jump_opcodes = ( 106 | GET_ITER, FOR_ITER, GET_ANEXT 107 | ) 108 | 109 | 110 | def read_code(stream): 111 | # This helper is needed in order for the PEP 302 emulation to 112 | # correctly handle compiled files 113 | # Note: stream must be opened in "rb" mode 114 | import marshal 115 | 116 | if sys.version_info < (3, 4): 117 | import imp 118 | runtime_magic = imp.get_magic() 119 | else: 120 | import importlib.util 121 | runtime_magic = importlib.util.MAGIC_NUMBER 122 | 123 | magic = stream.read(4) 124 | if magic != runtime_magic: 125 | print("*** Warning: file has wrong magic number ***") 126 | 127 | flags = 0 128 | if sys.version_info >= (3, 7): 129 | flags = struct.unpack('i', stream.read(4))[0] 130 | 131 | if flags & 1: 132 | stream.read(4) 133 | stream.read(4) 134 | else: 135 | stream.read(4) # Skip timestamp 136 | if sys.version_info >= (3, 3): 137 | stream.read(4) # Skip rawsize 138 | return marshal.load(stream) 139 | 140 | 141 | def dec_module(path): 142 | if path.endswith(".py"): 143 | if sys.version_info < (3, 6): 144 | import imp 145 | path = imp.cache_from_source(path) 146 | else: 147 | import importlib.util 148 | path = importlib.util.cache_from_source(path) 149 | elif not path.endswith(".pyc") and not path.endswith(".pyo"): 150 | raise ValueError("path must point to a .py or .pyc file") 151 | with open(path, "rb") as stream: 152 | code_obj = read_code(stream) 153 | code = Code(code_obj) 154 | return code.get_suite(include_declarations=False, look_for_docstring=True) 155 | 156 | 157 | def decompile(obj): 158 | """ 159 | Decompile obj if it is a module object, a function or a 160 | code object. If obj is a string, it is assumed to be the path 161 | to a python module. 162 | """ 163 | if isinstance(obj, str): 164 | return dec_module(obj) 165 | if inspect.iscode(obj): 166 | code = Code(obj) 167 | return code.get_suite() 168 | if inspect.isfunction(obj): 169 | code = Code(obj.__code__) 170 | defaults = obj.__defaults__ 171 | kwdefaults = obj.__kwdefaults__ 172 | return DefStatement(code, defaults, kwdefaults, obj.__closure__) 173 | elif inspect.ismodule(obj): 174 | return dec_module(obj.__file__) 175 | else: 176 | msg = "Object must be string, module, function or code object" 177 | raise TypeError(msg) 178 | 179 | 180 | class Indent: 181 | def __init__(self, indent_level=0, indent_step=4): 182 | self.level = indent_level 183 | self.step = indent_step 184 | 185 | def write(self, pattern, *args, **kwargs): 186 | if args or kwargs: 187 | pattern = pattern.format(*args, **kwargs) 188 | return self.indent(pattern) 189 | 190 | def __add__(self, indent_increase): 191 | return type(self)(self.level + indent_increase, self.step) 192 | 193 | 194 | class IndentPrint(Indent): 195 | def indent(self, string): 196 | print(" " * self.step * self.level + string) 197 | 198 | 199 | class IndentString(Indent): 200 | def __init__(self, indent_level=0, indent_step=4, lines=None): 201 | Indent.__init__(self, indent_level, indent_step) 202 | if lines is None: 203 | self.lines = [] 204 | else: 205 | self.lines = lines 206 | 207 | def __add__(self, indent_increase): 208 | return type(self)(self.level + indent_increase, self.step, self.lines) 209 | 210 | def sep(self): 211 | if not self.lines or self.lines[-1]: 212 | self.lines.append("") 213 | 214 | def indent(self, string): 215 | self.lines.append(" " * self.step * self.level + string) 216 | 217 | def __str__(self): 218 | return "\n".join(self.lines) 219 | 220 | 221 | class Stack: 222 | def __init__(self): 223 | self._stack = [] 224 | self._counts = {} 225 | 226 | def __bool__(self): 227 | return bool(self._stack) 228 | 229 | def __len__(self): 230 | return len(self._stack) 231 | 232 | def __contains__(self, val): 233 | return self.get_count(val) > 0 234 | 235 | def get_count(self, obj): 236 | return self._counts.get(id(obj), 0) 237 | 238 | def set_count(self, obj, val): 239 | if val: 240 | self._counts[id(obj)] = val 241 | else: 242 | del self._counts[id(obj)] 243 | 244 | def pop1(self): 245 | val = None 246 | if self._stack: 247 | val = self._stack.pop() 248 | else: 249 | raise Exception('Empty stack popped!') 250 | self.set_count(val, self.get_count(val) - 1) 251 | return val 252 | 253 | def pop(self, count=None): 254 | if count is None: 255 | val = self.pop1() 256 | return val 257 | else: 258 | vals = [self.pop1() for i in range(count)] 259 | vals.reverse() 260 | return vals 261 | 262 | def push(self, *args): 263 | for val in args: 264 | self.set_count(val, self.get_count(val) + 1) 265 | self._stack.append(val) 266 | 267 | def peek(self, count=None): 268 | if count is None: 269 | return self._stack[-1] 270 | else: 271 | return self._stack[-count:] 272 | 273 | 274 | def code_walker(code): 275 | l = len(code) 276 | code = array('B', code) 277 | oparg = 0 278 | i = 0 279 | extended_arg = 0 280 | 281 | while i < l: 282 | op = code[i] 283 | offset = 1 284 | if sys.version_info >= (3, 6): 285 | oparg = code[i + offset] 286 | offset += 1 287 | elif op >= HAVE_ARGUMENT: 288 | oparg = code[i + offset] + code[i + offset + 1] * 256 + extended_arg 289 | extended_arg = 0 290 | offset += 2 291 | if op == EXTENDED_ARG: 292 | if sys.version_info >= (3, 6): 293 | op = code[i + offset] 294 | offset += 1 295 | oparg <<= 8 296 | oparg |= code[i + offset] 297 | offset += 1 298 | else: 299 | extended_arg = oparg * 65536 300 | yield i, (op, oparg) 301 | i += offset 302 | 303 | 304 | class CodeFlags(object): 305 | def __init__(self, cf): 306 | self.flags = cf 307 | 308 | @property 309 | def optimized(self): 310 | return self.flags & 0x1 311 | 312 | @property 313 | def new_local(self): 314 | return self.flags & 0x2 315 | 316 | @property 317 | def varargs(self): 318 | return self.flags & 0x4 319 | 320 | @property 321 | def varkwargs(self): 322 | return self.flags & 0x8 323 | @property 324 | def nested(self): 325 | return self.flags & 0x10 326 | 327 | @property 328 | def generator(self): 329 | return self.flags & 0x20 330 | 331 | @property 332 | def no_free(self): 333 | return self.flags & 0x40 334 | 335 | @property 336 | def coroutine(self): 337 | return self.flags & 0x80 338 | 339 | @property 340 | def iterable_coroutine(self): 341 | return self.flags & 0x100 342 | 343 | @property 344 | def async_generator(self): 345 | return self.flags & 0x200 346 | 347 | 348 | class Code: 349 | def __init__(self, code_obj, parent=None): 350 | self.code_obj = code_obj 351 | self.parent = parent 352 | self.derefnames = [PyName(v) 353 | for v in code_obj.co_cellvars + code_obj.co_freevars] 354 | self.consts = list(map(PyConst, code_obj.co_consts)) 355 | self.names = list(map(PyName, code_obj.co_names)) 356 | self.varnames = list(map(PyName, code_obj.co_varnames)) 357 | self.instr_seq = list(code_walker(code_obj.co_code)) 358 | self.instr_map = {addr: i for i, (addr, _) in enumerate(self.instr_seq)} 359 | self.name = code_obj.co_name 360 | self.globals = [] 361 | self.nonlocals = [] 362 | self.jump_targets = [] 363 | self.find_else() 364 | self.find_jumps() 365 | trace('================================================') 366 | trace(self.code_obj) 367 | trace('================================================') 368 | for addr in self: 369 | trace(str(addr)) 370 | if addr.opcode in stmt_opcodes or addr.opcode in pop_jump_if_opcodes: 371 | trace(' ') 372 | trace('================================================') 373 | self.flags: CodeFlags = CodeFlags(code_obj.co_flags) 374 | 375 | def __getitem__(self, instr_index): 376 | if 0 <= instr_index < len(self.instr_seq): 377 | return Address(self, instr_index) 378 | 379 | def __iter__(self): 380 | for i in range(len(self.instr_seq)): 381 | yield Address(self, i) 382 | 383 | def show(self): 384 | for addr in self: 385 | print(addr) 386 | 387 | def address(self, addr): 388 | return self[self.instr_map[addr]] 389 | 390 | def iscellvar(self, i): 391 | return i < len(self.code_obj.co_cellvars) 392 | 393 | def find_jumps(self): 394 | for addr in self: 395 | opcode, arg = addr 396 | jt = addr.jump() 397 | if jt: 398 | self.jump_targets.append(jt) 399 | 400 | def find_else(self): 401 | jumps = {} 402 | last_jump = None 403 | for addr in self: 404 | opcode, arg = addr 405 | if opcode in pop_jump_if_opcodes: 406 | jump_addr = self.address(arg) 407 | if (jump_addr[-1].opcode in else_jump_opcodes 408 | or jump_addr.opcode == FOR_ITER): 409 | last_jump = addr 410 | jumps[jump_addr] = addr 411 | elif opcode == JUMP_ABSOLUTE: 412 | # This case is to deal with some nested ifs such as: 413 | # if a: 414 | # if b: 415 | # f() 416 | # elif c: 417 | # g() 418 | jump_addr = self.address(arg) 419 | if jump_addr in jumps: 420 | jumps[addr] = jumps[jump_addr] 421 | elif opcode == JUMP_FORWARD: 422 | jump_addr = addr[1] + arg 423 | if jump_addr in jumps: 424 | jumps[addr] = jumps[jump_addr] 425 | elif opcode in stmt_opcodes and last_jump is not None: 426 | # This opcode will generate a statement, so it means 427 | # that the last POP_JUMP_IF_x was an else-jump 428 | jumps[addr] = last_jump 429 | self.else_jumps = set(jumps.values()) 430 | 431 | def get_suite(self, include_declarations=True, look_for_docstring=False) -> Suite: 432 | dec = SuiteDecompiler(self[0]) 433 | dec.run() 434 | first_stmt = dec.suite and dec.suite[0] 435 | # Change __doc__ = "docstring" to "docstring" 436 | if look_for_docstring and isinstance(first_stmt, AssignStatement): 437 | chain = first_stmt.chain 438 | if len(chain) == 2 and str(chain[0]) == "__doc__": 439 | dec.suite[0] = DocString(first_stmt.chain[1].val) 440 | if include_declarations and (self.globals or self.nonlocals): 441 | suite = Suite() 442 | if self.globals: 443 | stmt = "global " + ", ".join(map(str, self.globals)) 444 | suite.add_statement(SimpleStatement(stmt)) 445 | if self.nonlocals: 446 | stmt = "nonlocal " + ", ".join(map(str, self.nonlocals)) 447 | suite.add_statement(SimpleStatement(stmt)) 448 | for stmt in dec.suite: 449 | suite.add_statement(stmt) 450 | return suite 451 | else: 452 | return dec.suite 453 | 454 | def declare_global(self, name): 455 | """ 456 | Declare name as a global. Called by STORE_GLOBAL and 457 | DELETE_GLOBAL 458 | """ 459 | if name not in self.globals: 460 | self.globals.append(name) 461 | 462 | def ensure_global(self, name): 463 | """ 464 | Declare name as global only if it is also a local variable 465 | name in one of the surrounding code objects. This is called 466 | by LOAD_GLOBAL 467 | """ 468 | parent = self.parent 469 | while parent: 470 | if name in parent.varnames: 471 | return self.declare_global(name) 472 | parent = parent.parent 473 | 474 | def declare_nonlocal(self, name): 475 | """ 476 | Declare name as nonlocal. Called by STORE_DEREF and 477 | DELETE_DEREF (but only when the name denotes a free variable, 478 | not a cell one). 479 | """ 480 | if name not in self.nonlocals: 481 | self.nonlocals.append(name) 482 | 483 | 484 | 485 | class Address: 486 | def __init__(self, code, instr_index): 487 | self.code = code 488 | self.index = instr_index 489 | self.addr, (self.opcode, self.arg) = code.instr_seq[instr_index] 490 | 491 | def __eq__(self, other): 492 | return (isinstance(other, type(self)) 493 | and self.code == other.code and self.index == other.index) 494 | 495 | def __lt__(self, other): 496 | return other is None or (isinstance(other, type(self)) 497 | and self.code == other.code and self.index < other.index) 498 | 499 | def __str__(self): 500 | mark = "* " if self in self.code.else_jumps else " " 501 | jump = self.jump() 502 | jt = '>>' if self.is_jump_target() else ' ' 503 | arg = self.arg or " " 504 | jdest = '\t(to {})'.format(jump.addr) if jump and jump.addr != self.arg else '' 505 | val = '' 506 | op = opname[self.opcode].ljust(18, ' ') 507 | try: 508 | 509 | val = self.code.globals[self.arg] and self.arg + 1 < len(self.code.globals) if 'GLOBAL' in op else \ 510 | self.code.names[self.arg] if 'ATTR' in op else \ 511 | self.code.names[self.arg] if 'NAME' in op else \ 512 | self.code.names[self.arg] if 'LOAD_METHOD' in op else \ 513 | self.code.consts[self.arg] if 'CONST' in op else \ 514 | self.code.varnames[self.arg] if 'FAST' in op else \ 515 | self.code.derefnames[self.arg] if 'DEREF' in op else \ 516 | cmp_op[self.arg] if 'COMPARE' in op else '' 517 | if val != '': 518 | val = '\t({})'.format(val) 519 | except: 520 | pass 521 | 522 | return "{}{}\t{}\t{}\t{}{}{}".format( 523 | jt, 524 | mark, 525 | self.addr, 526 | op, 527 | arg, 528 | jdest, 529 | val 530 | ) 531 | 532 | def __add__(self, delta): 533 | return self.code.address(self.addr + delta) 534 | 535 | def __getitem__(self, index): 536 | return self.code[self.index + index] 537 | 538 | def __iter__(self): 539 | yield self.opcode 540 | yield self.arg 541 | 542 | def __hash__(self): 543 | return hash((self.code, self.index)) 544 | 545 | def is_else_jump(self): 546 | return self in self.code.else_jumps 547 | 548 | def is_jump_target(self): 549 | return self in self.code.jump_targets 550 | 551 | def change_instr(self, opcode, arg=None): 552 | self.code.instr_seq[self.index] = (self.addr, (opcode, arg)) 553 | 554 | def jump(self) -> Address: 555 | opcode = self.opcode 556 | if opcode in dis.hasjrel: 557 | return self[1] + self.arg 558 | elif opcode in dis.hasjabs: 559 | return self.code.address(self.arg) 560 | 561 | def seek(self, opcode: tuple, increment: int, end: Address = None) -> Address: 562 | if not isinstance(opcode, tuple): 563 | opcode = (opcode,) 564 | a = self[increment] 565 | while a and a != end: 566 | if a.opcode in opcode: 567 | return a 568 | a = a[increment] 569 | 570 | def seek_back(self, opcode: Union[tuple,int], end: Address = None) -> Address: 571 | return self.seek(opcode, -1, end) 572 | 573 | def seek_forward(self, opcode: Union[tuple,int], end: Address = None) -> Address: 574 | return self.seek(opcode, 1, end) 575 | 576 | 577 | class AsyncMixin: 578 | def __init__(self): 579 | self.is_async = False 580 | 581 | @property 582 | def async_prefix(self): 583 | return 'async ' if self.is_async else '' 584 | 585 | 586 | class AwaitableMixin: 587 | 588 | def __init__(self): 589 | self.is_awaited = False 590 | 591 | @property 592 | def await_prefix(self): 593 | return 'await ' if self.is_awaited else '' 594 | 595 | 596 | class PyExpr: 597 | def wrap(self, condition=True): 598 | if condition: 599 | return "({})".format(self) 600 | else: 601 | return str(self) 602 | 603 | def store(self, dec, dest): 604 | chain = dec.assignment_chain 605 | chain.append(dest) 606 | if self not in dec.stack: 607 | chain.append(self) 608 | dec.suite.add_statement(AssignStatement(chain)) 609 | dec.assignment_chain = [] 610 | 611 | def on_pop(self, dec): 612 | dec.write(str(self)) 613 | 614 | 615 | class PyConst(PyExpr): 616 | precedence = 100 617 | 618 | def __init__(self, val): 619 | self.val = val 620 | 621 | def __str__(self): 622 | return repr(self.val) 623 | 624 | def __iter__(self): 625 | return iter(self.val) 626 | 627 | def __eq__(self, other): 628 | return isinstance(other, PyConst) and self.val == other.val 629 | 630 | 631 | class PyFormatValue(PyConst): 632 | def __str__(self): 633 | return f'{{{self.val}}}' 634 | 635 | 636 | class PyFormatString(PyExpr): 637 | precedence = 100 638 | 639 | def __init__(self, params): 640 | super().__init__() 641 | self.params = params 642 | 643 | def __str__(self): 644 | return "f'{}'".format(''.join([str(p) if isinstance(p, PyFormatValue) else str(p.val) for p in self.params])) 645 | 646 | 647 | class PyTuple(PyExpr): 648 | precedence = 0 649 | 650 | def __init__(self, values): 651 | self.values = values 652 | 653 | def __str__(self): 654 | if not self.values: 655 | return "()" 656 | valstr = [val.wrap(val.precedence <= self.precedence) 657 | for val in self.values] 658 | if len(valstr) == 1: 659 | return '(' + valstr[0] + "," + ')' 660 | else: 661 | return '(' + ", ".join(valstr) + ')' 662 | 663 | def __iter__(self): 664 | return iter(self.values) 665 | 666 | def wrap(self, condition=True): 667 | return str(self) 668 | 669 | 670 | class PyList(PyExpr): 671 | precedence = 16 672 | 673 | def __init__(self, values): 674 | self.values = values 675 | 676 | def __str__(self): 677 | valstr = ", ".join(val.wrap(val.precedence <= 0) 678 | for val in self.values) 679 | return "[{}]".format(valstr) 680 | 681 | def __iter__(self): 682 | return iter(self.values) 683 | 684 | 685 | class PySet(PyExpr): 686 | precedence = 16 687 | 688 | def __init__(self, values): 689 | self.values = values 690 | 691 | def __str__(self): 692 | valstr = ", ".join(val.wrap(val.precedence <= 0) 693 | for val in self.values) 694 | return "{{{}}}".format(valstr) 695 | 696 | def __iter__(self): 697 | return iter(self.values) 698 | 699 | 700 | class PyDict(PyExpr): 701 | precedence = 16 702 | 703 | def __init__(self): 704 | self.items = [] 705 | 706 | def set_item(self, key, val): 707 | self.items.append((key, val)) 708 | 709 | def __str__(self): 710 | itemstr = ", ".join("{}: {}".format(*kv) for kv in self.items) 711 | return "{{{}}}".format(itemstr) 712 | 713 | 714 | class PyName(PyExpr): 715 | precedence = 100 716 | 717 | def __init__(self, name): 718 | self.name = name 719 | 720 | def __str__(self): 721 | return self.name 722 | 723 | def __eq__(self, other): 724 | return isinstance(other, type(self)) and self.name == other.name 725 | 726 | 727 | class PyUnaryOp(PyExpr): 728 | def __init__(self, operand): 729 | self.operand = operand 730 | 731 | def __str__(self): 732 | opstr = self.operand.wrap(self.operand.precedence < self.precedence) 733 | return self.pattern.format(opstr) 734 | 735 | @classmethod 736 | def instr(cls, stack): 737 | stack.push(cls(stack.pop())) 738 | 739 | 740 | class PyBinaryOp(PyExpr): 741 | def __init__(self, left, right): 742 | self.left = left 743 | self.right = right 744 | 745 | def wrap_left(self): 746 | return self.left.wrap(self.left.precedence < self.precedence) 747 | 748 | def wrap_right(self): 749 | return self.right.wrap(self.right.precedence <= self.precedence) 750 | 751 | def __str__(self): 752 | return self.pattern.format(self.wrap_left(), self.wrap_right()) 753 | 754 | @classmethod 755 | def instr(cls, stack): 756 | right = stack.pop() 757 | left = stack.pop() 758 | stack.push(cls(left, right)) 759 | 760 | 761 | class PySubscript(PyBinaryOp): 762 | precedence = 15 763 | pattern = "{}[{}]" 764 | 765 | def wrap_right(self): 766 | return str(self.right) 767 | 768 | 769 | class PySlice(PyExpr): 770 | precedence = 1 771 | 772 | def __init__(self, args): 773 | assert len(args) in (2, 3) 774 | if len(args) == 2: 775 | self.start, self.stop = args 776 | self.step = None 777 | else: 778 | self.start, self.stop, self.step = args 779 | if self.start == PyConst(None): 780 | self.start = "" 781 | if self.stop == PyConst(None): 782 | self.stop = "" 783 | 784 | def __str__(self): 785 | if self.step is None: 786 | return "{}:{}".format(self.start, self.stop) 787 | else: 788 | return "{}:{}:{}".format(self.start, self.stop, self.step) 789 | 790 | 791 | class PyCompare(PyExpr): 792 | precedence = 6 793 | 794 | def __init__(self, complist): 795 | self.complist = complist 796 | 797 | def __str__(self): 798 | return " ".join(x if i % 2 else x.wrap(x.precedence <= 0) 799 | for i, x in enumerate(self.complist)) 800 | 801 | def extends(self, other): 802 | if not isinstance(other, PyCompare): 803 | return False 804 | else: 805 | return self.complist[0] == other.complist[-1] 806 | 807 | def chain(self, other): 808 | return PyCompare(self.complist + other.complist[1:]) 809 | 810 | 811 | class PyBooleanAnd(PyBinaryOp): 812 | precedence = 4 813 | pattern = "{} and {}" 814 | 815 | 816 | class PyBooleanOr(PyBinaryOp): 817 | precedence = 3 818 | pattern = "{} or {}" 819 | 820 | 821 | class PyIfElse(PyExpr): 822 | precedence = 2 823 | 824 | def __init__(self, cond, true_expr, false_expr): 825 | self.cond = cond 826 | self.true_expr = true_expr 827 | self.false_expr = false_expr 828 | 829 | def __str__(self): 830 | p = self.precedence 831 | cond_str = self.cond.wrap(self.cond.precedence <= p) 832 | true_str = self.true_expr.wrap(self.cond.precedence <= p) 833 | false_str = self.false_expr.wrap(self.cond.precedence < p) 834 | return "{} if {} else {}".format(true_str, cond_str, false_str) 835 | 836 | 837 | class PyAttribute(PyExpr): 838 | precedence = 15 839 | 840 | def __init__(self, expr, attrname): 841 | self.expr = expr 842 | self.attrname = attrname 843 | 844 | def __str__(self): 845 | expr_str = self.expr.wrap(self.expr.precedence < self.precedence) 846 | return "{}.{}".format(expr_str, self.attrname) 847 | 848 | 849 | class PyCallFunction(PyExpr, AwaitableMixin): 850 | precedence = 15 851 | 852 | def __init__(self, func: PyAttribute, args: list, kwargs: list, varargs=None, varkw=None): 853 | AwaitableMixin.__init__(self) 854 | self.func = func 855 | self.args = args 856 | self.kwargs = kwargs 857 | self.varargs = varargs 858 | self.varkw = varkw 859 | 860 | def __str__(self): 861 | funcstr = self.func.wrap(self.func.precedence < self.precedence) 862 | if hasattr(self.args, '__iter__') and len(self.args) == 1 and not (self.kwargs or self.varargs 863 | or self.varkw): 864 | arg = self.args[0] 865 | if isinstance(arg, PyGenExpr): 866 | # Only one pair of brackets arount a single arg genexpr 867 | return "{}{}".format(funcstr, arg) 868 | args = [x.wrap(x.precedence <= 0) for x in self.args] 869 | if self.varargs is not None: 870 | args.append("*{}".format(self.varargs)) 871 | args.extend("{}={}".format(str(k).replace('\'', ''), v.wrap(v.precedence <= 0)) 872 | for k, v in self.kwargs) 873 | if self.varkw is not None: 874 | args.append("**{}".format(self.varkw)) 875 | return "{}{}({})".format(self.await_prefix, funcstr, ", ".join(args)) 876 | 877 | 878 | class FunctionDefinition: 879 | def __init__(self, code: Code, defaults, kwdefaults, closure, paramobjs=None, annotations=None): 880 | self.code = code 881 | self.defaults = defaults 882 | self.kwdefaults = kwdefaults 883 | self.closure = closure 884 | self.paramobjs = paramobjs if paramobjs else {} 885 | self.annotations = annotations if annotations else [] 886 | 887 | def is_coroutine(self): 888 | return self.code.code_obj.co_flags & 0x100 889 | 890 | def getparams(self): 891 | code_obj = self.code.code_obj 892 | l = code_obj.co_argcount 893 | params = [] 894 | for name in code_obj.co_varnames[:l]: 895 | if name in self.paramobjs: 896 | params.append('{}:{}'.format(name, str(self.paramobjs[name]))) 897 | else: 898 | params.append(name) 899 | if self.defaults: 900 | for i, arg in enumerate(reversed(self.defaults)): 901 | name = params[-i - 1] 902 | if name in self.paramobjs: 903 | params[-i - 1] = "{}:{}={}".format(name, str(self.paramobjs[name]), arg) 904 | else: 905 | params[-i - 1] = "{}={}".format(name, arg) 906 | kwcount = code_obj.co_kwonlyargcount 907 | kwparams = [] 908 | if kwcount: 909 | for i in range(kwcount): 910 | name = code_obj.co_varnames[l + i] 911 | if name in self.kwdefaults and name in self.paramobjs: 912 | kwparams.append("{}:{}={}".format(name, self.paramobjs[name], self.kwdefaults[name])) 913 | elif name in self.kwdefaults: 914 | kwparams.append("{}={}".format(name, self.kwdefaults[name])) 915 | else: 916 | kwparams.append(name) 917 | l += kwcount 918 | if code_obj.co_flags & VARARGS: 919 | params.append("*" + code_obj.co_varnames[l]) 920 | l += 1 921 | elif kwparams: 922 | params.append("*") 923 | params.extend(kwparams) 924 | if code_obj.co_flags & VARKEYWORDS: 925 | params.append("**" + code_obj.co_varnames[l]) 926 | 927 | return params 928 | 929 | def getreturn(self): 930 | if self.paramobjs and 'return' in self.paramobjs: 931 | return self.paramobjs['return'] 932 | return None 933 | 934 | 935 | class PyLambda(PyExpr, FunctionDefinition): 936 | precedence = 1 937 | 938 | def __str__(self): 939 | suite = self.code.get_suite() 940 | params = ", ".join(self.getparams()) 941 | if len(suite.statements) > 0: 942 | def strip_return(val): 943 | return val[len("return "):] if val.startswith('return') else val 944 | 945 | if isinstance(suite[0], IfStatement): 946 | end = suite[1] if len(suite) > 1 else PyConst(None) 947 | expr = "{} if {} else {}".format( 948 | strip_return(str(suite[0].true_suite)), 949 | str(suite[0].cond), 950 | strip_return(str(end)) 951 | ) 952 | else: 953 | expr = strip_return(str(suite[0])) 954 | else: 955 | expr = "None" 956 | return "lambda {}: {}".format(params, expr) 957 | 958 | 959 | class PyComp(PyExpr): 960 | """ 961 | Abstraction for list, set, dict comprehensions and generator expressions 962 | """ 963 | precedence = 16 964 | 965 | def __init__(self, code, defaults, kwdefaults, closure, paramobjs={}, annotations=[]): 966 | assert not defaults and not kwdefaults 967 | self.code = code 968 | code[0].change_instr(NOP) 969 | last_i = len(code.instr_seq) - 1 970 | code[last_i].change_instr(NOP) 971 | self.annotations = annotations 972 | 973 | def set_iterable(self, iterable): 974 | self.code.varnames[0] = iterable 975 | 976 | def __str__(self): 977 | suite = self.code.get_suite() 978 | return self.pattern.format(suite.gen_display()) 979 | 980 | 981 | class PyListComp(PyComp): 982 | pattern = "[{}]" 983 | 984 | 985 | class PySetComp(PyComp): 986 | pattern = "{{{}}}" 987 | 988 | 989 | class PyKeyValue(PyBinaryOp): 990 | """This is only to create dict comprehensions""" 991 | precedence = 1 992 | pattern = "{}: {}" 993 | 994 | 995 | class PyDictComp(PyComp): 996 | pattern = "{{{}}}" 997 | 998 | 999 | class PyGenExpr(PyComp): 1000 | precedence = 16 1001 | pattern = "({})" 1002 | 1003 | def __init__(self, code, defaults, kwdefaults, closure, paramobjs={}, annotations=[]): 1004 | self.code = code 1005 | 1006 | 1007 | class PyYield(PyExpr): 1008 | precedence = 1 1009 | 1010 | def __init__(self, value): 1011 | self.value = value 1012 | 1013 | def __str__(self): 1014 | return "yield {}".format(self.value) 1015 | 1016 | 1017 | class PyYieldFrom(PyExpr): 1018 | precedence = 1 1019 | 1020 | def __init__(self, value): 1021 | self.value = value 1022 | 1023 | def __str__(self): 1024 | return "yield from {}".format(self.value) 1025 | 1026 | 1027 | class PyStarred(PyExpr): 1028 | """Used in unpacking assigments""" 1029 | precedence = 15 1030 | 1031 | def __init__(self, expr): 1032 | self.expr = expr 1033 | 1034 | def __str__(self): 1035 | es = self.expr.wrap(self.expr.precedence < self.precedence) 1036 | return "*{}".format(es) 1037 | 1038 | 1039 | code_map = { 1040 | '': PyLambda, 1041 | '': PyListComp, 1042 | '': PySetComp, 1043 | '': PyDictComp, 1044 | '': PyGenExpr, 1045 | } 1046 | 1047 | unary_ops = [ 1048 | ('UNARY_POSITIVE', 'Positive', '+{}', 13), 1049 | ('UNARY_NEGATIVE', 'Negative', '-{}', 13), 1050 | ('UNARY_NOT', 'Not', 'not {}', 5), 1051 | ('UNARY_INVERT', 'Invert', '~{}', 13), 1052 | ] 1053 | 1054 | binary_ops = [ 1055 | ('POWER', 'Power', '{}**{}', 14, '{} **= {}'), 1056 | ('MULTIPLY', 'Multiply', '{}*{}', 12, '{} *= {}'), 1057 | ('FLOOR_DIVIDE', 'FloorDivide', '{}//{}', 12, '{} //= {}'), 1058 | ('TRUE_DIVIDE', 'TrueDivide', '{}/{}', 12, '{} /= {}'), 1059 | ('MODULO', 'Modulo', '{} % {}', 12, '{} %= {}'), 1060 | ('ADD', 'Add', '{} + {}', 11, '{} += {}'), 1061 | ('SUBTRACT', 'Subtract', '{} - {}', 11, '{} -= {}'), 1062 | ('SUBSCR', 'Subscript', '{}[{}]', 15, None), 1063 | ('LSHIFT', 'LeftShift', '{} << {}', 10, '{} <<= {}'), 1064 | ('RSHIFT', 'RightShift', '{} >> {}', 10, '{} >>= {}'), 1065 | ('AND', 'And', '{} & {}', 9, '{} &= {}'), 1066 | ('XOR', 'Xor', '{} ^ {}', 8, '{} ^= {}'), 1067 | ('OR', 'Or', '{} | {}', 7, '{} |= {}'), 1068 | ('MATRIX_MULTIPLY', 'MatrixMultiply', '{} @ {}', 12, '{} @= {}'), 1069 | ] 1070 | 1071 | 1072 | class PyStatement(object): 1073 | def __str__(self): 1074 | istr = IndentString() 1075 | self.display(istr) 1076 | return str(istr) 1077 | 1078 | def wrap(self, condition=True): 1079 | if condition: 1080 | assert not condition 1081 | return "({})".format(self) 1082 | else: 1083 | return str(self) 1084 | 1085 | def on_pop(self, dec): 1086 | # dec.write("#ERROR: Unexpected context 'on_pop': pop on statement: ") 1087 | pass 1088 | 1089 | 1090 | class DocString(PyStatement): 1091 | def __init__(self, string): 1092 | self.string = string 1093 | 1094 | def display(self, indent): 1095 | if '\n' not in self.string: 1096 | indent.write(repr(self.string)) 1097 | else: 1098 | if "'''" not in self.string: 1099 | fence = "'''" 1100 | elif '"""' not in self.string: 1101 | fence = '"""' 1102 | else: 1103 | raise NotImplemented 1104 | lines = self.string.split('\n') 1105 | text = '\n'.join(l.encode('unicode_escape').decode() 1106 | for l in lines) 1107 | docstring = "{0}{1}{0}".format(fence, text) 1108 | indent.write(docstring) 1109 | 1110 | 1111 | class AssignStatement(PyStatement): 1112 | def __init__(self, chain): 1113 | self.chain = chain 1114 | 1115 | def display(self, indent): 1116 | indent.write(" = ".join(map(str, self.chain))) 1117 | 1118 | 1119 | class InPlaceOp(PyStatement): 1120 | def __init__(self, left, right): 1121 | self.right = right 1122 | self.left = left 1123 | 1124 | def store(self, dec, dest): 1125 | # assert dest is self.left 1126 | dec.suite.add_statement(self) 1127 | 1128 | def display(self, indent): 1129 | indent.write(self.pattern, self.left, self.right) 1130 | 1131 | @classmethod 1132 | def instr(cls, stack): 1133 | right = stack.pop() 1134 | left = stack.pop() 1135 | stack.push(cls(left, right)) 1136 | 1137 | 1138 | class Unpack: 1139 | precedence = 50 1140 | 1141 | def __init__(self, val, length, star_index=None): 1142 | self.val = val 1143 | self.length = length 1144 | self.star_index = star_index 1145 | self.dests = [] 1146 | 1147 | def store(self, dec, dest): 1148 | if len(self.dests) == self.star_index: 1149 | dest = PyStarred(dest) 1150 | self.dests.append(dest) 1151 | if len(self.dests) == self.length: 1152 | dec.stack.push(self.val) 1153 | dec.store(PyTuple(self.dests)) 1154 | 1155 | 1156 | class ImportStatement(PyStatement): 1157 | alias = "" 1158 | precedence = 100 1159 | 1160 | def __init__(self, name, level, fromlist): 1161 | self.name = name 1162 | self.alias = name 1163 | self.level = level 1164 | self.fromlist = fromlist 1165 | self.aslist = [] 1166 | 1167 | def store(self, dec: SuiteDecompiler, dest): 1168 | self.alias = dest 1169 | dec.suite.add_statement(self) 1170 | 1171 | def on_pop(self, dec): 1172 | dec.suite.add_statement(self) 1173 | 1174 | def display(self, indent): 1175 | if self.fromlist == PyConst(None): 1176 | name = self.name.name 1177 | alias = self.alias.name 1178 | if name == alias or name.startswith(alias + "."): 1179 | indent.write("import {}", name) 1180 | else: 1181 | indent.write("import {} as {}", name, alias) 1182 | elif self.fromlist == PyConst(('*',)): 1183 | indent.write("from {} import *", self.name.name) 1184 | else: 1185 | names = [] 1186 | for name, alias in zip(self.fromlist, self.aslist): 1187 | if name == alias: 1188 | names.append(name) 1189 | else: 1190 | names.append("{} as {}".format(name, alias)) 1191 | indent.write("from {} import {}", self.name, ", ".join(names)) 1192 | 1193 | 1194 | class ImportFrom: 1195 | def __init__(self, name): 1196 | self.name = name 1197 | 1198 | def store(self, dec, dest): 1199 | imp = dec.stack.peek() 1200 | assert isinstance(imp, ImportStatement) 1201 | 1202 | if imp.fromlist != PyConst(None): 1203 | 1204 | imp.aslist.append(dest.name) 1205 | else: 1206 | imp.alias = dest 1207 | 1208 | 1209 | class SimpleStatement(PyStatement): 1210 | def __init__(self, val): 1211 | assert val is not None 1212 | self.val = val 1213 | 1214 | def display(self, indent): 1215 | indent.write(self.val) 1216 | 1217 | def gen_display(self, seq=()): 1218 | return " ".join((self.val,) + seq) 1219 | 1220 | 1221 | class IfStatement(PyStatement): 1222 | def __init__(self, cond, true_suite, false_suite): 1223 | self.cond = cond 1224 | self.true_suite = true_suite 1225 | self.false_suite = false_suite 1226 | 1227 | def display(self, indent, is_elif=False): 1228 | ptn = "elif {}:" if is_elif else "if {}:" 1229 | indent.write(ptn, self.cond) 1230 | self.true_suite.display(indent + 1) 1231 | if not self.false_suite: 1232 | return 1233 | if len(self.false_suite) == 1: 1234 | stmt = self.false_suite[0] 1235 | if isinstance(stmt, IfStatement): 1236 | stmt.display(indent, is_elif=True) 1237 | return 1238 | indent.write("else:") 1239 | self.false_suite.display(indent + 1) 1240 | 1241 | def gen_display(self, seq=()): 1242 | s = "if {}".format(self.cond) 1243 | return self.true_suite.gen_display(seq + (s,)) 1244 | 1245 | 1246 | class ForStatement(PyStatement, AsyncMixin): 1247 | def __init__(self, iterable): 1248 | AsyncMixin.__init__(self) 1249 | self.iterable = iterable 1250 | 1251 | def store(self, dec, dest): 1252 | self.dest = dest 1253 | 1254 | def display(self, indent): 1255 | indent.write("{}for {} in {}:", self.async_prefix, self.dest, self.iterable) 1256 | self.body.display(indent + 1) 1257 | 1258 | def gen_display(self, seq=()): 1259 | s = "{}for {} in {}".format(self.async_prefix, self.dest, self.iterable.wrap() if isinstance(self.iterable, PyIfElse) else self.iterable) 1260 | return self.body.gen_display(seq + (s,)) 1261 | 1262 | 1263 | class WhileStatement(PyStatement): 1264 | def __init__(self, cond, body): 1265 | self.cond = cond 1266 | self.body = body 1267 | 1268 | def display(self, indent): 1269 | indent.write("while {}:", self.cond) 1270 | self.body.display(indent + 1) 1271 | 1272 | 1273 | class DecorableStatement(PyStatement): 1274 | def __init__(self): 1275 | self.decorators = [] 1276 | 1277 | def display(self, indent): 1278 | indent.sep() 1279 | for f in reversed(self.decorators): 1280 | indent.write("@{}", f) 1281 | self.display_undecorated(indent) 1282 | indent.sep() 1283 | 1284 | def decorate(self, f): 1285 | self.decorators.append(f) 1286 | 1287 | 1288 | class DefStatement(FunctionDefinition, DecorableStatement, AsyncMixin): 1289 | def __init__(self, code: Code, defaults, kwdefaults, closure, paramobjs=None, annotations=None): 1290 | FunctionDefinition.__init__(self, code, defaults, kwdefaults, closure, paramobjs, annotations) 1291 | DecorableStatement.__init__(self) 1292 | AsyncMixin.__init__(self) 1293 | self.is_async = code.flags.coroutine 1294 | 1295 | def display_undecorated(self, indent): 1296 | paramlist = ", ".join(self.getparams()) 1297 | result = self.getreturn() 1298 | if result: 1299 | indent.write("{}def {}({}) -> {}:", self.async_prefix, self.code.name, paramlist, result) 1300 | else: 1301 | indent.write("{}def {}({}):", self.async_prefix, self.code.name, paramlist) 1302 | # Assume that co_consts starts with None unless the function 1303 | # has a docstring, in which case it starts with the docstring 1304 | if self.code.consts[0] != PyConst(None): 1305 | docstring = self.code.consts[0].val 1306 | DocString(docstring).display(indent + 1) 1307 | self.code.get_suite().display(indent + 1) 1308 | 1309 | def store(self, dec, dest): 1310 | self.name = dest 1311 | dec.suite.add_statement(self) 1312 | 1313 | 1314 | class TryStatement(PyStatement): 1315 | def __init__(self, try_suite): 1316 | self.try_suite = try_suite 1317 | self.except_clauses = [] 1318 | 1319 | def add_except_clause(self, type, suite): 1320 | self.except_clauses.append([type, None, suite]) 1321 | 1322 | def store(self, dec, dest): 1323 | self.except_clauses[-1][1] = dest 1324 | 1325 | def display(self, indent): 1326 | indent.write("try:") 1327 | self.try_suite.display(indent + 1) 1328 | for type, name, suite in self.except_clauses: 1329 | if type is None: 1330 | indent.write("except:") 1331 | elif name is None: 1332 | indent.write("except {}:", type) 1333 | else: 1334 | indent.write("except {} as {}:", type, name) 1335 | suite.display(indent + 1) 1336 | 1337 | 1338 | class FinallyStatement(PyStatement): 1339 | def __init__(self, try_suite, finally_suite): 1340 | self.try_suite = try_suite 1341 | self.finally_suite = finally_suite 1342 | 1343 | def display(self, indent): 1344 | # Wrap the try suite in a TryStatement if necessary 1345 | try_stmt = None 1346 | if len(self.try_suite) == 1: 1347 | try_stmt = self.try_suite[0] 1348 | if not isinstance(try_stmt, TryStatement): 1349 | try_stmt = None 1350 | if try_stmt is None: 1351 | try_stmt = TryStatement(self.try_suite) 1352 | try_stmt.display(indent) 1353 | indent.write("finally:") 1354 | self.finally_suite.display(indent + 1) 1355 | 1356 | 1357 | class WithStatement(PyStatement): 1358 | def __init__(self, with_expr): 1359 | self.with_expr = with_expr 1360 | self.with_name = None 1361 | self.is_async = False 1362 | 1363 | @property 1364 | def async_prefix(self): 1365 | return 'async ' if self.is_async else '' 1366 | 1367 | def store(self, dec, dest): 1368 | self.with_name = dest 1369 | 1370 | def display(self, indent, args=None): 1371 | # args to take care of nested withs: 1372 | # with x as t: 1373 | # with y as u: 1374 | # 1375 | # ---> 1376 | # with x as t, y as u: 1377 | # 1378 | if args is None: 1379 | args = [] 1380 | if self.with_name is None: 1381 | args.append(str(self.with_expr)) 1382 | else: 1383 | args.append("{} as {}".format(self.with_expr, self.with_name)) 1384 | if len(self.suite) == 1 and isinstance(self.suite[0], WithStatement): 1385 | self.suite[0].display(indent, args) 1386 | else: 1387 | indent.write(self.async_prefix + "with {}:", ", ".join(args)) 1388 | self.suite.display(indent + 1) 1389 | 1390 | 1391 | class ClassStatement(DecorableStatement): 1392 | def __init__(self, func, name, parents, kwargs): 1393 | DecorableStatement.__init__(self) 1394 | self.func = func 1395 | self.parents = parents 1396 | self.kwargs = kwargs 1397 | 1398 | def store(self, dec, dest): 1399 | self.name = dest 1400 | dec.suite.add_statement(self) 1401 | 1402 | def display_undecorated(self, indent): 1403 | if self.parents or self.kwargs: 1404 | args = [str(x) for x in self.parents] 1405 | kwargs = ["{}={}".format(str(k).replace('\'', ''), v) for k, v in self.kwargs] 1406 | all_args = ", ".join(args + kwargs) 1407 | indent.write("class {}({}):", self.name, all_args) 1408 | else: 1409 | indent.write("class {}:", self.name) 1410 | suite = self.func.code.get_suite(look_for_docstring=True) 1411 | if suite: 1412 | # TODO: find out why sometimes the class suite ends with 1413 | # "return __class__" 1414 | last_stmt = suite[-1] 1415 | if isinstance(last_stmt, SimpleStatement): 1416 | if last_stmt.val.startswith("return "): 1417 | suite.statements.pop() 1418 | clean_vars = ['__module__', '__qualname__'] 1419 | for clean_var in clean_vars: 1420 | for i in range(len(suite.statements)): 1421 | stmt = suite.statements[i] 1422 | if isinstance(stmt, AssignStatement) and str(stmt).startswith(clean_var): 1423 | suite.statements.pop(i) 1424 | break 1425 | 1426 | suite.display(indent + 1) 1427 | 1428 | 1429 | class Suite: 1430 | def __init__(self): 1431 | self.statements = [] 1432 | 1433 | def __bool__(self): 1434 | return bool(self.statements) 1435 | 1436 | def __len__(self): 1437 | return len(self.statements) 1438 | 1439 | def __getitem__(self, i): 1440 | return self.statements[i] 1441 | 1442 | def __setitem__(self, i, val): 1443 | self.statements[i] = val 1444 | 1445 | def __str__(self): 1446 | istr = IndentString() 1447 | self.display(istr) 1448 | return str(istr) 1449 | 1450 | def display(self, indent): 1451 | if self.statements: 1452 | for stmt in self.statements: 1453 | stmt.display(indent) 1454 | else: 1455 | indent.write("pass") 1456 | 1457 | def gen_display(self, seq=()): 1458 | completed = ' '.join(seq) 1459 | # This is to fix a line in dataclasses.py 1460 | if completed == 'for f in fields.values() if f._field_type is _FIELD': 1461 | return '[f for f in fields.values() if f._field_type is _FIELD]' 1462 | if len(self) < 1: 1463 | # This is to fix a line in dataclasses.py 1464 | if completed == 'for f in fields if f.hash is None': 1465 | return 'f for f in fields if f.hash is None' 1466 | return ' '.join(seq) 1467 | return self[0].gen_display(seq) 1468 | 1469 | def add_statement(self, stmt): 1470 | self.statements.append(stmt) 1471 | 1472 | 1473 | class SuiteDecompiler: 1474 | # An instruction handler can return this to indicate to the run() 1475 | # function that it should return immediately 1476 | END_NOW = object() 1477 | 1478 | # This is put on the stack by LOAD_BUILD_CLASS 1479 | BUILD_CLASS = object() 1480 | 1481 | def __init__(self, start_addr, end_addr=None, stack=None): 1482 | self.start_addr = start_addr 1483 | self.end_addr = end_addr 1484 | self.code: Code = start_addr.code 1485 | self.stack = Stack() if stack is None else stack 1486 | self.suite = Suite() 1487 | self.assignment_chain = [] 1488 | self.popjump_stack = [] 1489 | 1490 | def push_popjump(self, jtruthiness, jaddr, jcond): 1491 | stack = self.popjump_stack 1492 | if jaddr and jaddr[-1].is_else_jump(): 1493 | # Increase jaddr to the 'else' address if it jumps to the 'then' 1494 | jaddr = jaddr[-1].jump() 1495 | while stack: 1496 | truthiness, addr, cond = stack[-1] 1497 | # if jaddr == None: 1498 | # raise Exception("#ERROR: jaddr is None") 1499 | # jaddr == None \ 1500 | if jaddr and jaddr < addr or jaddr == addr: 1501 | break 1502 | stack.pop() 1503 | obj_maker = PyBooleanOr if truthiness else PyBooleanAnd 1504 | if isinstance(jcond, obj_maker): 1505 | # Use associativity of 'and' and 'or' to minimise the 1506 | # number of parentheses 1507 | jcond = obj_maker(obj_maker(cond, jcond.left), jcond.right) 1508 | else: 1509 | jcond = obj_maker(cond, jcond) 1510 | stack.append((jtruthiness, jaddr, jcond)) 1511 | 1512 | def pop_popjump(self): 1513 | truthiness, addr, cond = self.popjump_stack.pop() 1514 | return cond 1515 | 1516 | def run(self): 1517 | addr, end_addr = self.start_addr, self.end_addr 1518 | while addr and addr < end_addr: 1519 | try: 1520 | opcode, arg = addr 1521 | method = getattr(self, opname[opcode]) 1522 | if opcode < HAVE_ARGUMENT: 1523 | new_addr = method(addr) 1524 | else: 1525 | new_addr = method(addr, arg) 1526 | if new_addr is self.END_NOW: 1527 | break 1528 | elif new_addr is None: 1529 | new_addr = addr[1] 1530 | addr = new_addr 1531 | except: 1532 | addr = addr[1] 1533 | continue 1534 | return addr 1535 | 1536 | def write(self, template, *args): 1537 | def fmt(x): 1538 | if isinstance(x, int): 1539 | return self.stack.getval(x) 1540 | else: 1541 | return x 1542 | 1543 | if args: 1544 | line = template.format(*map(fmt, args)) 1545 | else: 1546 | line = template 1547 | self.suite.add_statement(SimpleStatement(line)) 1548 | 1549 | def store(self, dest): 1550 | val = self.stack.pop() 1551 | val.store(self, dest) 1552 | 1553 | def is_for_loop(self, addr, end_addr): 1554 | i = 0 1555 | while 1: 1556 | cur_addr = addr[i] 1557 | if cur_addr == end_addr: 1558 | break 1559 | elif cur_addr.opcode in else_jump_opcodes: 1560 | cur_addr = cur_addr.jump() 1561 | if cur_addr and cur_addr.opcode in for_jump_opcodes: 1562 | return True 1563 | break 1564 | elif cur_addr.opcode in for_jump_opcodes: 1565 | return True 1566 | i = i + 1 1567 | return False 1568 | 1569 | def scan_to_first_jump_if(self, addr: Address, end_addr: Address) -> Union[Address,None]: 1570 | i = 0 1571 | while 1: 1572 | cur_addr = addr[i] 1573 | if cur_addr == end_addr: 1574 | break 1575 | elif cur_addr.opcode in pop_jump_if_opcodes: 1576 | return cur_addr 1577 | elif cur_addr.opcode in else_jump_opcodes: 1578 | break 1579 | elif cur_addr.opcode in for_jump_opcodes: 1580 | break 1581 | i = i + 1 1582 | return None 1583 | 1584 | def scan_for_final_jump(self, start_addr, end_addr): 1585 | i = 0 1586 | end = None 1587 | while 1: 1588 | cur_addr = end_addr[i] 1589 | if cur_addr == start_addr: 1590 | break 1591 | elif cur_addr.opcode == JUMP_ABSOLUTE: 1592 | end = cur_addr 1593 | return end 1594 | elif cur_addr.opcode in else_jump_opcodes: 1595 | break 1596 | elif cur_addr.opcode in pop_jump_if_opcodes: 1597 | break 1598 | i = i - 1 1599 | return end 1600 | 1601 | # 1602 | # All opcode methods in CAPS below. 1603 | # 1604 | 1605 | def SETUP_LOOP(self, addr: Address, delta): 1606 | jump_addr = addr[1] + delta 1607 | end_addr = jump_addr[-1] 1608 | if end_addr.opcode == POP_BLOCK: # assume conditional 1609 | # scan to first jump 1610 | end_cond = self.scan_to_first_jump_if(addr[1], end_addr) 1611 | if end_cond and end_cond[1].opcode == BREAK_LOOP: 1612 | end_cond = None 1613 | if end_cond and end_cond.arg == addr.arg: 1614 | # scan for conditional 1615 | d_cond = SuiteDecompiler(addr[1], end_cond) 1616 | # 1617 | d_cond.run() 1618 | cond = d_cond.stack.pop() 1619 | if end_cond.opcode == POP_JUMP_IF_TRUE: 1620 | cond = PyNot(cond) 1621 | d_body = SuiteDecompiler(end_cond[1], end_addr) 1622 | while_stmt = WhileStatement(cond, d_body.suite) 1623 | d_body.stack.push(while_stmt) 1624 | d_body.run() 1625 | while_stmt.body = d_body.suite 1626 | self.suite.add_statement(while_stmt) 1627 | return jump_addr 1628 | elif (not end_cond or not end_cond.jump()[1] == addr.jump()) and not self.is_for_loop(addr[1], end_addr): 1629 | d_body = SuiteDecompiler(addr[1], end_addr) 1630 | while_stmt = WhileStatement(PyConst(True), d_body.suite) 1631 | d_body.stack.push(while_stmt) 1632 | d_body.run() 1633 | while_stmt.body = d_body.suite 1634 | self.suite.add_statement(while_stmt) 1635 | return jump_addr 1636 | return None 1637 | 1638 | def BREAK_LOOP(self, addr): 1639 | self.write("break") 1640 | 1641 | def CONTINUE_LOOP(self, addr, *argv): 1642 | self.write("continue") 1643 | 1644 | def SETUP_FINALLY(self, addr, delta): 1645 | start_finally = addr.jump() 1646 | d_try = SuiteDecompiler(addr[1], start_finally) 1647 | d_try.run() 1648 | d_finally = SuiteDecompiler(start_finally) 1649 | end_finally = d_finally.run() 1650 | self.suite.add_statement(FinallyStatement(d_try.suite, d_finally.suite)) 1651 | return end_finally[1] 1652 | 1653 | def END_FINALLY(self, addr): 1654 | return self.END_NOW 1655 | 1656 | def SETUP_EXCEPT(self, addr, delta): 1657 | start_except = addr.jump() 1658 | start_try = addr[1] 1659 | end_try = start_except 1660 | if sys.version_info < (3, 7): 1661 | if end_try.opcode == JUMP_FORWARD: 1662 | end_try = end_try[1] + end_try.arg 1663 | elif end_try.opcode == JUMP_ABSOLUTE: 1664 | end_try = end_try[-1] 1665 | else: 1666 | end_try = end_try[1] 1667 | d_try = SuiteDecompiler(start_try, end_try) 1668 | d_try.run() 1669 | 1670 | stmt = TryStatement(d_try.suite) 1671 | while start_except.opcode != END_FINALLY: 1672 | if start_except.opcode == DUP_TOP: 1673 | # There's a new except clause 1674 | d_except = SuiteDecompiler(start_except[1]) 1675 | d_except.stack.push(stmt) 1676 | d_except.run() 1677 | start_except = stmt.next_start_except 1678 | elif start_except.opcode == POP_TOP: 1679 | # It's a bare except clause - it starts: 1680 | # POP_TOP 1681 | # POP_TOP 1682 | # POP_TOP 1683 | # 1684 | # POP_EXCEPT 1685 | start_except = start_except[3] 1686 | end_except = start_except 1687 | 1688 | while end_except and end_except[-1].opcode != RETURN_VALUE: 1689 | if end_except.opcode == POP_EXCEPT: 1690 | break 1691 | end_except = end_except[1] 1692 | # Handle edge case where there is a return in the except 1693 | if end_except[-1].opcode == RETURN_VALUE: 1694 | d_except = SuiteDecompiler(start_except, end_except) 1695 | end_except = d_except.run() 1696 | stmt.add_except_clause(None, d_except.suite) 1697 | self.suite.add_statement(stmt) 1698 | return end_except 1699 | 1700 | d_except = SuiteDecompiler(start_except, end_except) 1701 | end_except = d_except.run() 1702 | stmt.add_except_clause(None, d_except.suite) 1703 | start_except = end_except[2] 1704 | assert start_except.opcode == END_FINALLY 1705 | self.suite.add_statement(stmt) 1706 | return start_except[1] 1707 | 1708 | def SETUP_WITH(self, addr, delta): 1709 | end_with = addr.jump() 1710 | with_stmt = WithStatement(self.stack.pop()) 1711 | d_with = SuiteDecompiler(addr[1], end_with) 1712 | d_with.stack.push(with_stmt) 1713 | d_with.run() 1714 | with_stmt.suite = d_with.suite 1715 | self.suite.add_statement(with_stmt) 1716 | if sys.version_info <= (3, 4): 1717 | assert end_with.opcode == WITH_CLEANUP 1718 | assert end_with[1].opcode == END_FINALLY 1719 | return end_with[2] 1720 | else: 1721 | assert end_with.opcode == WITH_CLEANUP_START 1722 | assert end_with[1].opcode == WITH_CLEANUP_FINISH 1723 | return end_with[3] 1724 | 1725 | def POP_BLOCK(self, addr): 1726 | pass 1727 | 1728 | def POP_EXCEPT(self, addr): 1729 | return self.END_NOW 1730 | 1731 | def NOP(self, addr): 1732 | return 1733 | 1734 | def COMPARE_OP(self, addr, compare_opname): 1735 | left, right = self.stack.pop(2) 1736 | if compare_opname != 10: # 10 is exception match 1737 | self.stack.push(PyCompare([left, cmp_op[compare_opname], right])) 1738 | else: 1739 | # It's an exception match 1740 | # left is a TryStatement 1741 | # right is the exception type to be matched 1742 | # It goes: 1743 | # COMPARE_OP 10 1744 | # POP_JUMP_IF_FALSE 1745 | # POP_TOP 1746 | # POP_TOP or STORE_FAST (if the match is named) 1747 | # POP_TOP 1748 | # SETUP_FINALLY if the match was named 1749 | assert addr[1].opcode == POP_JUMP_IF_FALSE 1750 | left.next_start_except = addr[1].jump() 1751 | assert addr[2].opcode == POP_TOP 1752 | assert addr[4].opcode == POP_TOP 1753 | if addr[5].opcode == SETUP_FINALLY: 1754 | except_start = addr[6] 1755 | except_end = addr[5].jump() 1756 | else: 1757 | except_start = addr[5] 1758 | except_end = left.next_start_except 1759 | d_body = SuiteDecompiler(except_start, except_end) 1760 | d_body.run() 1761 | left.add_except_clause(right, d_body.suite) 1762 | if addr[3].opcode != POP_TOP: 1763 | # The exception is named 1764 | d_exc_name = SuiteDecompiler(addr[3], addr[4]) 1765 | d_exc_name.stack.push(left) 1766 | # This will store the name in left: 1767 | d_exc_name.run() 1768 | # We're done with this except clause 1769 | return self.END_NOW 1770 | 1771 | # 1772 | # Stack manipulation 1773 | # 1774 | 1775 | def POP_TOP(self, addr): 1776 | self.stack.pop().on_pop(self) 1777 | 1778 | def ROT_TWO(self, addr): 1779 | # special case: x, y = z, t 1780 | if addr[2] and addr[1].opcode == STORE_NAME and addr[2].opcode == STORE_NAME: 1781 | val = PyTuple(self.stack.pop(2)) 1782 | unpack = Unpack(val, 2) 1783 | self.stack.push(unpack) 1784 | self.stack.push(unpack) 1785 | else: 1786 | tos1, tos = self.stack.pop(2) 1787 | self.stack.push(tos, tos1) 1788 | 1789 | def ROT_THREE(self, addr): 1790 | tos2, tos1, tos = self.stack.pop(3) 1791 | self.stack.push(tos, tos2, tos1) 1792 | 1793 | def DUP_TOP(self, addr): 1794 | self.stack.push(self.stack.peek()) 1795 | 1796 | def DUP_TOP_TWO(self, addr): 1797 | self.stack.push(*self.stack.peek(2)) 1798 | 1799 | # 1800 | # LOAD / STORE / DELETE 1801 | # 1802 | 1803 | # FAST 1804 | 1805 | def LOAD_FAST(self, addr, var_num): 1806 | name = self.code.varnames[var_num] 1807 | self.stack.push(name) 1808 | 1809 | def STORE_FAST(self, addr, var_num): 1810 | name = self.code.varnames[var_num] 1811 | self.store(name) 1812 | 1813 | def DELETE_FAST(self, addr, var_num): 1814 | name = self.code.varnames[var_num] 1815 | self.write("del {}", name) 1816 | 1817 | # DEREF 1818 | 1819 | def LOAD_DEREF(self, addr, i): 1820 | name = self.code.derefnames[i] 1821 | self.stack.push(name) 1822 | 1823 | def LOAD_CLASSDEREF(self, addr, i): 1824 | name = self.code.derefnames[i] 1825 | self.stack.push(name) 1826 | 1827 | def STORE_DEREF(self, addr, i): 1828 | name = self.code.derefnames[i] 1829 | if not self.code.iscellvar(i): 1830 | self.code.declare_nonlocal(name) 1831 | self.store(name) 1832 | 1833 | def DELETE_DEREF(self, addr, i): 1834 | name = self.code.derefnames[i] 1835 | if not self.code.iscellvar(i): 1836 | self.code.declare_nonlocal(name) 1837 | self.write("del {}", name) 1838 | 1839 | # GLOBAL 1840 | 1841 | def LOAD_GLOBAL(self, addr, namei): 1842 | name = self.code.names[namei] 1843 | self.code.ensure_global(name) 1844 | self.stack.push(name) 1845 | 1846 | def STORE_GLOBAL(self, addr, namei): 1847 | name = self.code.names[namei] 1848 | self.code.declare_global(name) 1849 | self.store(name) 1850 | 1851 | def DELETE_GLOBAL(self, addr, namei): 1852 | name = self.code.names[namei] 1853 | self.declare_global(name) 1854 | self.write("del {}", name) 1855 | 1856 | # NAME 1857 | 1858 | def LOAD_NAME(self, addr, namei): 1859 | name = self.code.names[namei] 1860 | self.stack.push(name) 1861 | 1862 | def STORE_NAME(self, addr, namei): 1863 | name = self.code.names[namei] 1864 | self.store(name) 1865 | 1866 | def DELETE_NAME(self, addr, namei): 1867 | name = self.code.names[namei] 1868 | self.write("del {}", name) 1869 | 1870 | # METHOD 1871 | def LOAD_METHOD(self, addr, namei): 1872 | expr = self.stack.pop() 1873 | attrname = self.code.names[namei] 1874 | self.stack.push(PyAttribute(expr, attrname)) 1875 | 1876 | def CALL_METHOD(self, addr, argc, have_var=False, have_kw=False): 1877 | kw_argc = argc >> 8 1878 | pos_argc = argc 1879 | varkw = self.stack.pop() if have_kw else None 1880 | varargs = self.stack.pop() if have_var else None 1881 | kwargs_iter = iter(self.stack.pop(2 * kw_argc)) 1882 | kwargs = list(zip(kwargs_iter, kwargs_iter)) 1883 | posargs = self.stack.pop(pos_argc) 1884 | func = self.stack.pop() 1885 | if func is self.BUILD_CLASS: 1886 | # It's a class construction 1887 | # TODO: check the assert statement below is correct 1888 | assert not (have_var or have_kw) 1889 | func, name, *parents = posargs 1890 | self.stack.push(ClassStatement(func, name, parents, kwargs)) 1891 | elif isinstance(func, PyComp): 1892 | # It's a list/set/dict comprehension or generator expression 1893 | assert not (have_var or have_kw) 1894 | assert len(posargs) == 1 and not kwargs 1895 | func.set_iterable(posargs[0]) 1896 | self.stack.push(func) 1897 | elif posargs and isinstance(posargs[0], DecorableStatement): 1898 | # It's a decorator for a def/class statement 1899 | assert len(posargs) == 1 and not kwargs 1900 | defn = posargs[0] 1901 | defn.decorate(func) 1902 | self.stack.push(defn) 1903 | else: 1904 | # It's none of the above, so it must be a normal function call 1905 | func_call = PyCallFunction(func, posargs, kwargs, varargs, varkw) 1906 | self.stack.push(func_call) 1907 | 1908 | # ATTR 1909 | 1910 | def LOAD_ATTR(self, addr, namei): 1911 | expr = self.stack.pop() 1912 | attrname = self.code.names[namei] 1913 | self.stack.push(PyAttribute(expr, attrname)) 1914 | 1915 | def STORE_ATTR(self, addr, namei): 1916 | expr = self.stack.pop() 1917 | attrname = self.code.names[namei] 1918 | self.store(PyAttribute(expr, attrname)) 1919 | 1920 | def DELETE_ATTR(self, addr, namei): 1921 | expr = self.stack.pop() 1922 | attrname = self.code.names[namei] 1923 | self.write("del {}.{}", expr, attrname) 1924 | 1925 | # SUBSCR 1926 | 1927 | def STORE_SUBSCR(self, addr): 1928 | expr, sub = self.stack.pop(2) 1929 | self.store(PySubscript(expr, sub)) 1930 | 1931 | def DELETE_SUBSCR(self, addr): 1932 | expr, sub = self.stack.pop(2) 1933 | self.write("del {}[{}]", expr, sub) 1934 | 1935 | # CONST 1936 | CONST_LITERALS = { 1937 | Ellipsis: PyName('...') 1938 | } 1939 | def LOAD_CONST(self, addr, consti): 1940 | const = self.code.consts[consti] 1941 | if const.val in self.CONST_LITERALS: 1942 | const = self.CONST_LITERALS[const.val] 1943 | self.stack.push(const) 1944 | 1945 | # 1946 | # Import statements 1947 | # 1948 | 1949 | def IMPORT_NAME(self, addr, namei): 1950 | name = self.code.names[namei] 1951 | level, fromlist = self.stack.pop(2) 1952 | self.stack.push(ImportStatement(name, level, fromlist)) 1953 | # special case check for import x.y.z as w syntax which uses 1954 | # attributes and assignments and is difficult to workaround 1955 | i = 1 1956 | while addr[i].opcode == LOAD_ATTR: i = i + 1 1957 | if i > 1 and addr[i].opcode in (STORE_FAST, STORE_NAME): 1958 | return addr[i] 1959 | return None 1960 | 1961 | def IMPORT_FROM(self, addr, namei): 1962 | name = self.code.names[namei] 1963 | self.stack.push(ImportFrom(name)) 1964 | if addr[1].opcode == ROT_TWO: 1965 | return addr[4] 1966 | 1967 | 1968 | def IMPORT_STAR(self, addr): 1969 | self.POP_TOP(addr) 1970 | 1971 | # 1972 | # Function call 1973 | # 1974 | 1975 | def STORE_LOCALS(self, addr): 1976 | self.stack.pop() 1977 | return addr[3] 1978 | 1979 | def LOAD_BUILD_CLASS(self, addr): 1980 | self.stack.push(self.BUILD_CLASS) 1981 | 1982 | def RETURN_VALUE(self, addr): 1983 | value = self.stack.pop() 1984 | if isinstance(value, PyConst) and value.val is None: 1985 | if addr[1] is not None: 1986 | self.write("return") 1987 | return 1988 | if self.code.flags.coroutine or self.code.flags.iterable_coroutine: 1989 | self.write("yield {}", value) 1990 | else: 1991 | self.write("return {}", value) 1992 | 1993 | def GET_YIELD_FROM_ITER(self, addr): 1994 | pass 1995 | 1996 | def YIELD_VALUE(self, addr): 1997 | if self.code.name == '': 1998 | return 1999 | value = self.stack.pop() 2000 | self.stack.push(PyYield(value)) 2001 | 2002 | def YIELD_FROM(self, addr): 2003 | value = self.stack.pop() # TODO: from statement ? 2004 | value = self.stack.pop() 2005 | self.stack.push(PyYieldFrom(value)) 2006 | 2007 | def CALL_FUNCTION_CORE(self, func, posargs, kwargs, varargs, varkw): 2008 | if func is self.BUILD_CLASS: 2009 | # It's a class construction 2010 | # TODO: check the assert statement below is correct 2011 | # assert not (have_var or have_kw) 2012 | func, name, *parents = posargs 2013 | self.stack.push(ClassStatement(func, name, parents, kwargs)) 2014 | elif isinstance(func, PyComp): 2015 | # It's a list/set/dict comprehension or generator expression 2016 | # assert not (have_var or have_kw) 2017 | assert len(posargs) == 1 and not kwargs 2018 | func.set_iterable(posargs[0]) 2019 | self.stack.push(func) 2020 | elif posargs and isinstance(posargs, list) and isinstance(posargs[0], DecorableStatement): 2021 | # It's a decorator for a def/class statement 2022 | assert len(posargs) == 1 and not kwargs 2023 | defn = posargs[0] 2024 | defn.decorate(func) 2025 | self.stack.push(defn) 2026 | else: 2027 | # It's none of the above, so it must be a normal function call 2028 | func_call = PyCallFunction(func, posargs, kwargs, varargs, varkw) 2029 | self.stack.push(func_call) 2030 | 2031 | def CALL_FUNCTION(self, addr, argc, have_var=False, have_kw=False): 2032 | if sys.version_info >= (3, 6): 2033 | pos_argc = argc 2034 | posargs = self.stack.pop(pos_argc) 2035 | func = self.stack.pop() 2036 | self.CALL_FUNCTION_CORE(func, posargs, [], None, None) 2037 | else: 2038 | kw_argc = argc >> 8 2039 | pos_argc = argc & 0xFF 2040 | varkw = self.stack.pop() if have_kw else None 2041 | varargs = self.stack.pop() if have_var else None 2042 | kwargs_iter = iter(self.stack.pop(2 * kw_argc)) 2043 | kwargs = list(zip(kwargs_iter, kwargs_iter)) 2044 | posargs = self.stack.pop(pos_argc) 2045 | func = self.stack.pop() 2046 | self.CALL_FUNCTION_CORE(func, posargs, kwargs, varargs, varkw) 2047 | 2048 | def CALL_FUNCTION_VAR(self, addr, argc): 2049 | self.CALL_FUNCTION(addr, argc, have_var=True) 2050 | 2051 | def CALL_FUNCTION_KW(self, addr, argc): 2052 | if sys.version_info >= (3, 6): 2053 | keys = self.stack.pop() 2054 | kwargc = len(keys.val) 2055 | kwarg_values = self.stack.pop(kwargc) 2056 | posargs = self.stack.pop(argc - kwargc) 2057 | func = self.stack.pop() 2058 | kwarg_dict = list(zip([PyName(k) for k in keys], kwarg_values)) 2059 | self.CALL_FUNCTION_CORE(func, posargs, kwarg_dict, None, None) 2060 | else: 2061 | self.CALL_FUNCTION(addr, argc, have_kw=True) 2062 | 2063 | def CALL_FUNCTION_EX(self, addr, flags): 2064 | kwarg_dict = PyDict() 2065 | if flags & 1: 2066 | kwarg_dict = self.stack.pop() 2067 | posargs = self.stack.pop() 2068 | func = self.stack.pop() 2069 | kwvar = None 2070 | posvar = None 2071 | 2072 | if not isinstance(posargs, PyTuple): 2073 | posvar = posargs 2074 | posargs = PyTuple([]) 2075 | 2076 | if not isinstance(kwarg_dict, PyDict): 2077 | kwvar = kwarg_dict 2078 | kwarg_dict = PyDict() 2079 | 2080 | if not posargs: 2081 | posargs = PyTuple([]) 2082 | 2083 | assert isinstance(kwarg_dict, PyDict) 2084 | for i in range(len(kwarg_dict.items)): 2085 | k, v = kwarg_dict.items[i] 2086 | if isinstance(v, PyConst) and v.val == '**KWARG**': 2087 | kwarg_dict.items.pop(i) 2088 | kwvar = k.val 2089 | break 2090 | elif not isinstance(k, PyConst): 2091 | kwvar = kwarg_dict 2092 | kwarg_dict = PyDict() 2093 | break 2094 | 2095 | assert isinstance(posargs, PyTuple) 2096 | posvals = posargs.values 2097 | assert isinstance(posvals, list) 2098 | for i in range(len(posvals)): 2099 | posarg = posvals[i] 2100 | if isinstance(posarg, PyName) and posarg.name == 'args': 2101 | posvals.pop(i) 2102 | posvar = posarg 2103 | break 2104 | 2105 | self.CALL_FUNCTION_CORE(func, list(posargs.values), list(kwarg_dict.items), posvar, kwvar) 2106 | 2107 | def CALL_FUNCTION_VAR_KW(self, addr, argc): 2108 | self.CALL_FUNCTION(addr, argc, have_var=True, have_kw=True) 2109 | 2110 | # a, b, ... = ... 2111 | 2112 | def UNPACK_SEQUENCE(self, addr, count): 2113 | unpack = Unpack(self.stack.pop(), count) 2114 | for i in range(count): 2115 | self.stack.push(unpack) 2116 | 2117 | def UNPACK_EX(self, addr, counts): 2118 | rcount = counts >> 8 2119 | lcount = counts & 0xFF 2120 | count = lcount + rcount + 1 2121 | unpack = Unpack(self.stack.pop(), count, lcount) 2122 | for i in range(count): 2123 | self.stack.push(unpack) 2124 | 2125 | # Build operations 2126 | 2127 | def BUILD_SLICE(self, addr, argc): 2128 | assert argc in (2, 3) 2129 | self.stack.push(PySlice(self.stack.pop(argc))) 2130 | 2131 | def BUILD_TUPLE(self, addr, count): 2132 | values = [self.stack.pop() for i in range(count)] 2133 | values.reverse() 2134 | self.stack.push(PyTuple(values)) 2135 | 2136 | def BUILD_TUPLE_UNPACK_WITH_CALL(self, addr, count): 2137 | values = [] 2138 | for o in self.stack.pop(count): 2139 | if isinstance(o, PyTuple): 2140 | values.extend(o.values) 2141 | else: 2142 | values.append(o) 2143 | 2144 | self.stack.push(PyTuple(values)) 2145 | 2146 | def BUILD_LIST(self, addr, count): 2147 | values = [self.stack.pop() for i in range(count)] 2148 | values.reverse() 2149 | self.stack.push(PyList(values)) 2150 | 2151 | def BUILD_SET(self, addr, count): 2152 | values = [self.stack.pop() for i in range(count)] 2153 | values.reverse() 2154 | self.stack.push(PySet(values)) 2155 | 2156 | def BUILD_MAP(self, addr, count): 2157 | d = PyDict() 2158 | if sys.version_info >= (3, 5): 2159 | for i in range(count): 2160 | d.items.append(tuple(self.stack.pop(2))) 2161 | self.stack.push(d) 2162 | 2163 | def BUILD_MAP_UNPACK_WITH_CALL(self, addr, count): 2164 | d = PyDict() 2165 | for i in range(count): 2166 | o = self.stack.pop() 2167 | if isinstance(o, PyDict): 2168 | for item in o.items: 2169 | k, v = item 2170 | d.set_item(PyConst(k.val), v) 2171 | else: 2172 | d.set_item(PyConst(o), PyConst('**KWARG**')) 2173 | self.stack.push(d) 2174 | 2175 | def BUILD_CONST_KEY_MAP(self, addr, count): 2176 | keys = self.stack.pop() 2177 | vals = self.stack.pop(count) 2178 | dict = PyDict() 2179 | for i in range(count): 2180 | dict.set_item(PyConst(keys.val[i]), vals[i]) 2181 | self.stack.push(dict) 2182 | 2183 | def STORE_MAP(self, addr): 2184 | v, k = self.stack.pop(2) 2185 | d = self.stack.peek() 2186 | d.set_item(k, v) 2187 | 2188 | # Comprehension operations - just create an expression statement 2189 | 2190 | def LIST_APPEND(self, addr, i): 2191 | self.POP_TOP(addr) 2192 | 2193 | def SET_ADD(self, addr, i): 2194 | self.POP_TOP(addr) 2195 | 2196 | def MAP_ADD(self, addr, i): 2197 | value, key = self.stack.pop(2) 2198 | self.stack.push(PyKeyValue(key, value)) 2199 | self.POP_TOP(addr) 2200 | 2201 | # and operator 2202 | 2203 | def JUMP_IF_FALSE_OR_POP(self, addr, target): 2204 | end_addr = addr.jump() 2205 | self.push_popjump(True, end_addr, self.stack.pop()) 2206 | left = self.pop_popjump() 2207 | if end_addr.opcode == ROT_TWO: 2208 | opc, arg = end_addr[-1] 2209 | if opc == JUMP_FORWARD and arg == 2: 2210 | end_addr = end_addr[2] 2211 | elif opc == RETURN_VALUE or opc == JUMP_FORWARD: 2212 | end_addr = end_addr[-1] 2213 | d = SuiteDecompiler(addr[1], end_addr, self.stack) 2214 | d.run() 2215 | right = self.stack.pop() 2216 | if isinstance(right, PyCompare) and right.extends(left): 2217 | py_and = left.chain(right) 2218 | else: 2219 | py_and = PyBooleanAnd(left, right) 2220 | self.stack.push(py_and) 2221 | return end_addr[3] 2222 | 2223 | d = SuiteDecompiler(addr[1], end_addr, self.stack) 2224 | d.run() 2225 | # if end_addr.opcode == RETURN_VALUE: 2226 | # return end_addr[2] 2227 | right = self.stack.pop() 2228 | if isinstance(right, PyCompare) and right.extends(left): 2229 | py_and = left.chain(right) 2230 | else: 2231 | py_and = PyBooleanAnd(left, right) 2232 | self.stack.push(py_and) 2233 | return end_addr 2234 | 2235 | # This appears when there are chained comparisons, e.g. 1 <= x < 10 2236 | 2237 | def JUMP_FORWARD(self, addr, delta): 2238 | ## if delta == 2 and addr[1].opcode == ROT_TWO and addr[2].opcode == POP_TOP: 2239 | ## # We're in the special case of chained comparisons 2240 | ## return addr[3] 2241 | ## else: 2242 | ## # I'm hoping its an unused JUMP in an if-else statement 2243 | ## return addr[1] 2244 | return addr.jump() 2245 | 2246 | # or operator 2247 | 2248 | def JUMP_IF_TRUE_OR_POP(self, addr, target): 2249 | end_addr = addr.jump() 2250 | self.push_popjump(True, end_addr, self.stack.pop()) 2251 | left = self.pop_popjump() 2252 | d = SuiteDecompiler(addr[1], end_addr, self.stack) 2253 | d.run() 2254 | right = self.stack.pop() 2255 | self.stack.push(PyBooleanOr(left, right)) 2256 | return end_addr 2257 | 2258 | # 2259 | # If-else statements/expressions and related structures 2260 | # 2261 | 2262 | def POP_JUMP_IF(self, addr: Address, target: int, truthiness: bool) -> Union[Address, None]: 2263 | jump_addr = addr.jump() 2264 | end_of_loop = jump_addr.opcode == FOR_ITER or jump_addr[-1].opcode == SETUP_LOOP 2265 | if jump_addr.opcode == FOR_ITER: 2266 | # We are in a for-loop with nothing after the if-suite 2267 | # But take care: for-loops in generator expression do 2268 | # not end in POP_BLOCK, hence the test below. 2269 | jump_addr = jump_addr.jump() 2270 | elif end_of_loop: 2271 | # We are in a while-loop with nothing after the if-suite 2272 | jump_addr = jump_addr[-1].jump()[-1] 2273 | cond = self.stack.pop() 2274 | if not addr.is_else_jump(): 2275 | 2276 | 2277 | # Handle generator expressions with or clause 2278 | for_iter = addr.seek_back(FOR_ITER) 2279 | if for_iter: 2280 | end_of_for = for_iter.jump() 2281 | if end_of_for.addr > addr.addr: 2282 | gen = jump_addr.seek_forward((YIELD_VALUE, LIST_APPEND), end_of_for) 2283 | if gen: 2284 | if not truthiness: 2285 | truthiness = not truthiness 2286 | if truthiness: 2287 | cond = PyNot(cond) 2288 | self.push_popjump(truthiness, jump_addr, cond) 2289 | return None 2290 | 2291 | self.push_popjump(truthiness, jump_addr, cond) 2292 | # Dictionary comprehension 2293 | if jump_addr.seek_forward(MAP_ADD): 2294 | return None 2295 | 2296 | # Generator 2297 | if jump_addr.seek_forward(YIELD_VALUE): 2298 | return None 2299 | 2300 | # Generator 2301 | if jump_addr.opcode != END_FINALLY and jump_addr[1] and jump_addr[1].opcode == JUMP_ABSOLUTE: 2302 | return None 2303 | 2304 | a = addr[1] 2305 | while a and a < jump_addr: 2306 | if a.opcode in stmt_opcodes: 2307 | break 2308 | if a.opcode in pop_jump_if_opcodes and a.arg >= addr.arg: 2309 | return None 2310 | a = a[1] 2311 | # if there are no nested conditionals and no else clause, write the true portion and jump ahead to the end of the conditional 2312 | cond = self.pop_popjump() 2313 | end_true = jump_addr 2314 | if truthiness: 2315 | cond = PyNot(cond) 2316 | d_true = SuiteDecompiler(addr[1], end_true) 2317 | d_true.run() 2318 | stmt = IfStatement(cond, d_true.suite, None) 2319 | self.suite.add_statement(stmt) 2320 | return end_true 2321 | # Increase jump_addr to pop all previous jumps 2322 | self.push_popjump(truthiness, jump_addr[1], cond) 2323 | cond = self.pop_popjump() 2324 | end_true = jump_addr[-1] 2325 | if truthiness: 2326 | last_pj = addr.seek_back(pop_jump_if_opcodes) 2327 | if last_pj and last_pj.arg == addr.arg and isinstance(cond, PyBooleanAnd) or isinstance(cond, PyBooleanOr): 2328 | cond.right = PyNot(cond.right) 2329 | else: 2330 | cond = PyNot(cond) 2331 | 2332 | if end_true.opcode == RETURN_VALUE: 2333 | end_false = jump_addr.seek_forward(RETURN_VALUE) 2334 | if end_false and end_false[2] and end_false[2].opcode == RETURN_VALUE: 2335 | d_true = SuiteDecompiler(addr[1], end_true[1]) 2336 | d_true.run() 2337 | d_false = SuiteDecompiler(jump_addr,end_false[1]) 2338 | d_false.run() 2339 | self.suite.add_statement(IfStatement(cond, d_true.suite, d_false.suite)) 2340 | 2341 | return end_false[1] 2342 | 2343 | # - If the true clause ends in return, make sure it's included 2344 | # - If the true clause ends in RAISE_VARARGS, then it's an 2345 | # assert statement. For now I just write it as a raise within 2346 | # an if (see below) 2347 | if end_true.opcode in (RETURN_VALUE, RAISE_VARARGS, POP_TOP): 2348 | # TODO: change 2349 | # if cond: raise AssertionError(x) 2350 | # to 2351 | # assert cond, x 2352 | d_true = SuiteDecompiler(addr[1], end_true[1]) 2353 | d_true.run() 2354 | self.suite.add_statement(IfStatement(cond, d_true.suite, Suite())) 2355 | return jump_addr 2356 | d_true = SuiteDecompiler(addr[1], end_true) 2357 | d_true.run() 2358 | if jump_addr.opcode == POP_BLOCK and not end_of_loop: 2359 | # It's a while loop 2360 | stmt = WhileStatement(cond, d_true.suite) 2361 | self.suite.add_statement(stmt) 2362 | return jump_addr[1] 2363 | # It's an if-else (expression or statement) 2364 | if end_true.opcode == JUMP_FORWARD: 2365 | end_false = end_true.jump() 2366 | elif end_true.opcode == JUMP_ABSOLUTE: 2367 | end_false = end_true.jump() 2368 | if end_false.opcode == FOR_ITER: 2369 | # We are in a for-loop with nothing after the else-suite 2370 | end_false = end_false.jump()[-1] 2371 | elif end_false[-1].opcode == SETUP_LOOP: 2372 | # We are in a while-loop with nothing after the else-suite 2373 | end_false = end_false[-1].jump()[-1] 2374 | if end_false.opcode == RETURN_VALUE: 2375 | end_false = end_false[1] 2376 | elif end_true.opcode == RETURN_VALUE: 2377 | # find the next RETURN_VALUE 2378 | end_false = jump_addr 2379 | while end_false.opcode != RETURN_VALUE: 2380 | end_false = end_false[1] 2381 | end_false = end_false[1] 2382 | elif end_true.opcode == BREAK_LOOP: 2383 | # likely in a loop in a try/except 2384 | end_false = jump_addr 2385 | else: 2386 | end_false = jump_addr 2387 | # # normal statement 2388 | # raise Exception("#ERROR: Unexpected statement: {} | {}\n".format(end_true, jump_addr, jump_addr[-1])) 2389 | # # raise Unknown 2390 | # jump_addr = end_true[-2] 2391 | # stmt = IfStatement(cond, d_true.suite, None) 2392 | # self.suite.add_statement(stmt) 2393 | # return jump_addr or self.END_NOW 2394 | d_false = SuiteDecompiler(jump_addr, end_false) 2395 | d_false.run() 2396 | if d_true.stack and d_false.stack: 2397 | assert len(d_true.stack) == len(d_false.stack) == 1 2398 | # self.write("#ERROR: Unbalanced stacks {} != {}".format(len(d_true.stack),len(d_false.stack))) 2399 | assert not (d_true.suite or d_false.suite) 2400 | # this happens in specific if else conditions with assigments 2401 | true_expr = d_true.stack.pop() 2402 | false_expr = d_false.stack.pop() 2403 | self.stack.push(PyIfElse(cond, true_expr, false_expr)) 2404 | else: 2405 | stmt = IfStatement(cond, d_true.suite, d_false.suite) 2406 | self.suite.add_statement(stmt) 2407 | return end_false or self.END_NOW 2408 | 2409 | def POP_JUMP_IF_FALSE(self, addr, target): 2410 | return self.POP_JUMP_IF(addr, target, truthiness=False) 2411 | 2412 | def POP_JUMP_IF_TRUE(self, addr, target): 2413 | return self.POP_JUMP_IF(addr, target, truthiness=True) 2414 | 2415 | def JUMP_ABSOLUTE(self, addr, target): 2416 | # print("*** JUMP ABSOLUTE ***", addr) 2417 | # return addr.jump() 2418 | 2419 | # TODO: print out continue if not final jump 2420 | jump_addr = addr.jump() 2421 | if jump_addr[-1].opcode == SETUP_LOOP: 2422 | end_addr = jump_addr + jump_addr[-1].arg 2423 | last_jump = self.scan_for_final_jump(jump_addr, end_addr[-1]) 2424 | if last_jump != addr: 2425 | pass 2426 | pass 2427 | 2428 | # 2429 | # For loops 2430 | # 2431 | 2432 | def GET_ITER(self, addr): 2433 | pass 2434 | 2435 | def FOR_ITER(self, addr: Address, delta): 2436 | if addr[-1] and addr[-1].opcode == RETURN_VALUE: 2437 | # Dead code 2438 | return self.END_NOW 2439 | iterable = self.stack.pop() 2440 | jump_addr = addr.jump() 2441 | end_body = jump_addr 2442 | if end_body.opcode != POP_BLOCK: 2443 | end_body = end_body[-1] 2444 | d_body = SuiteDecompiler(addr[1], end_body) 2445 | for_stmt = ForStatement(iterable) 2446 | d_body.stack.push(for_stmt) 2447 | d_body.run() 2448 | for_stmt.body = d_body.suite 2449 | self.suite.add_statement(for_stmt) 2450 | return jump_addr 2451 | 2452 | # Function creation 2453 | 2454 | def MAKE_FUNCTION_OLD(self, addr, argc, is_closure=False): 2455 | testType = self.stack.pop().val 2456 | if isinstance(testType, str): 2457 | code = Code(self.stack.pop().val, self.code) 2458 | else: 2459 | code = Code(testType, self.code) 2460 | closure = self.stack.pop() if is_closure else None 2461 | # parameter annotation objects 2462 | paramobjs = {} 2463 | paramcount = (argc >> 16) & 0x7FFF 2464 | if paramcount: 2465 | paramobjs = dict(zip(self.stack.pop().val, self.stack.pop(paramcount - 1))) 2466 | # default argument objects in positional order 2467 | defaults = self.stack.pop(argc & 0xFF) 2468 | # pairs of name and default argument, with the name just below the object on the stack, for keyword-only parameters 2469 | kwdefaults = {} 2470 | for i in range((argc >> 8) & 0xFF): 2471 | k, v = self.stack.pop(2) 2472 | if hasattr(k, 'name'): 2473 | kwdefaults[k.name] = v 2474 | elif hasattr(k, 'val'): 2475 | kwdefaults[k.val] = v 2476 | else: 2477 | kwdefaults[str(k)] = v 2478 | func_maker = code_map.get(code.name, DefStatement) 2479 | self.stack.push(func_maker(code, defaults, kwdefaults, closure, paramobjs)) 2480 | 2481 | def MAKE_FUNCTION_NEW(self, addr, argc, is_closure=False): 2482 | testType = self.stack.pop().val 2483 | if isinstance(testType, str): 2484 | code = Code(self.stack.pop().val, self.code) 2485 | else: 2486 | code = Code(testType, self.code) 2487 | closure = self.stack.pop() if is_closure else None 2488 | annotations = {} 2489 | kwdefaults = {} 2490 | defaults = {} 2491 | if argc & 8: 2492 | annotations = list(self.stack.pop()) 2493 | if argc & 4: 2494 | annotations = self.stack.pop() 2495 | if isinstance(annotations, PyDict): 2496 | annotations = {str(k[0].val).replace('\'', ''): str(k[1]) for k in annotations.items} 2497 | if argc & 2: 2498 | kwdefaults = self.stack.pop() 2499 | if isinstance(kwdefaults, PyDict): 2500 | kwdefaults = {str(k[0].val): str(k[1] if isinstance(k[1], PyExpr) else PyConst(k[1])) for k in 2501 | kwdefaults.items} 2502 | if not kwdefaults: 2503 | kwdefaults = {} 2504 | if argc & 1: 2505 | defaults = list(map(lambda x: str(x if isinstance(x, PyExpr) else PyConst(x)), self.stack.pop())) 2506 | func_maker = code_map.get(code.name, DefStatement) 2507 | self.stack.push(func_maker(code, defaults, kwdefaults, closure, annotations, annotations)) 2508 | 2509 | def MAKE_FUNCTION(self, addr, argc, is_closure=False): 2510 | if sys.version_info < (3, 6): 2511 | self.MAKE_FUNCTION_OLD(addr, argc, is_closure) 2512 | else: 2513 | self.MAKE_FUNCTION_NEW(addr, argc, is_closure) 2514 | 2515 | def LOAD_CLOSURE(self, addr, i): 2516 | # Push the varname. It doesn't matter as it is not used for now. 2517 | self.stack.push(self.code.derefnames[i]) 2518 | 2519 | def MAKE_CLOSURE(self, addr, argc): 2520 | self.MAKE_FUNCTION(addr, argc, is_closure=True) 2521 | 2522 | # 2523 | # Raising exceptions 2524 | # 2525 | 2526 | def RAISE_VARARGS(self, addr, argc): 2527 | # TODO: find out when argc is 2 or 3 2528 | # Answer: In Python 3, only 0, 1, or 2 argument (see PEP 3109) 2529 | if argc == 0: 2530 | self.write("raise") 2531 | elif argc == 1: 2532 | exception = self.stack.pop() 2533 | self.write("raise {}", exception) 2534 | elif argc == 2: 2535 | from_exc, exc = self.stack.pop(), self.stack.pop() 2536 | self.write("raise {} from {}".format(exc, from_exc)) 2537 | else: 2538 | raise Unknown 2539 | 2540 | def EXTENDED_ARG(self, addr, ext): 2541 | # self.write("# ERROR: {} : {}".format(addr, ext) ) 2542 | pass 2543 | 2544 | def WITH_CLEANUP(self, addr, *args, **kwargs): 2545 | # self.write("# ERROR: {} : {}".format(addr, args)) 2546 | pass 2547 | 2548 | def WITH_CLEANUP_START(self, addr, *args, **kwargs): 2549 | pass 2550 | 2551 | def WITH_CLEANUP_FINISH(self, addr, *args, **kwargs): 2552 | jaddr = addr.jump() 2553 | return jaddr 2554 | 2555 | # Formatted string literals 2556 | def FORMAT_VALUE(self, addr, flags): 2557 | val = self.stack.pop() 2558 | self.stack.push(PyFormatValue(val)) 2559 | 2560 | def BUILD_STRING(self, addr, c): 2561 | params = self.stack.pop(c) 2562 | self.stack.push(PyFormatString(params)) 2563 | 2564 | # Coroutines 2565 | def GET_AWAITABLE(self, addr: Address): 2566 | func: AwaitableMixin = self.stack.pop() 2567 | func.is_awaited = True 2568 | self.stack.push(func) 2569 | yield_op = addr.seek_forward(YIELD_FROM) 2570 | return yield_op[1] 2571 | 2572 | def BEFORE_ASYNC_WITH(self, addr: Address): 2573 | with_addr = addr.seek_forward(SETUP_ASYNC_WITH) 2574 | end_with = with_addr.jump() 2575 | with_stmt = WithStatement(self.stack.pop()) 2576 | with_stmt.is_async = True 2577 | d_with = SuiteDecompiler(addr[1], end_with) 2578 | d_with.stack.push(with_stmt) 2579 | d_with.run() 2580 | with_stmt.suite = d_with.suite 2581 | self.suite.add_statement(with_stmt) 2582 | if sys.version_info <= (3, 4): 2583 | assert end_with.opcode == WITH_CLEANUP 2584 | assert end_with[1].opcode == END_FINALLY 2585 | return end_with[2] 2586 | else: 2587 | assert end_with.opcode == WITH_CLEANUP_START 2588 | assert end_with[1].opcode == GET_AWAITABLE 2589 | assert end_with[4].opcode == WITH_CLEANUP_FINISH 2590 | return end_with[5] 2591 | 2592 | def SETUP_ASYNC_WITH(self, addr: Address, arg): 2593 | pass 2594 | 2595 | def GET_AITER(self, addr: Address): 2596 | return addr[2] 2597 | 2598 | def GET_ANEXT(self, addr: Address): 2599 | iterable = self.stack.pop() 2600 | for_stmt = ForStatement(iterable) 2601 | for_stmt.is_async = True 2602 | jump_addr = addr[-1].jump() 2603 | d_body = SuiteDecompiler(addr[3], jump_addr[-1]) 2604 | d_body.stack.push(for_stmt) 2605 | d_body.run() 2606 | jump_addr = jump_addr[-1].jump() 2607 | new_start = jump_addr 2608 | new_end = jump_addr[-2].jump()[-1] 2609 | d_body.start_addr = new_start 2610 | 2611 | d_body.end_addr = new_end 2612 | 2613 | d_body.run() 2614 | 2615 | for_stmt.body = d_body.suite 2616 | self.suite.add_statement(for_stmt) 2617 | new_end = new_end.seek_forward(POP_BLOCK) 2618 | return new_end 2619 | 2620 | 2621 | def make_dynamic_instr(cls): 2622 | def method(self, addr): 2623 | cls.instr(self.stack) 2624 | 2625 | return method 2626 | 2627 | 2628 | # Create unary operators types and opcode handlers 2629 | for op, name, ptn, prec in unary_ops: 2630 | name = 'Py' + name 2631 | tp = type(name, (PyUnaryOp,), dict(pattern=ptn, precedence=prec)) 2632 | globals()[name] = tp 2633 | setattr(SuiteDecompiler, op, make_dynamic_instr(tp)) 2634 | 2635 | # Create binary operators types and opcode handlers 2636 | for op, name, ptn, prec, inplace_ptn in binary_ops: 2637 | # Create the binary operator 2638 | tp_name = 'Py' + name 2639 | tp = globals().get(tp_name, None) 2640 | if tp is None: 2641 | tp = type(tp_name, (PyBinaryOp,), dict(pattern=ptn, precedence=prec)) 2642 | globals()[tp_name] = tp 2643 | 2644 | setattr(SuiteDecompiler, 'BINARY_' + op, make_dynamic_instr(tp)) 2645 | # Create the in-place operation 2646 | if inplace_ptn is not None: 2647 | inplace_op = "INPLACE_" + op 2648 | tp_name = 'InPlace' + name 2649 | tp = type(tp_name, (InPlaceOp,), dict(pattern=inplace_ptn)) 2650 | globals()[tp_name] = tp 2651 | setattr(SuiteDecompiler, inplace_op, make_dynamic_instr(tp)) 2652 | 2653 | if __name__ == "__main__": 2654 | import sys 2655 | 2656 | if len(sys.argv) == 1: 2657 | print('USAGE: {} '.format(sys.argv[0])) 2658 | else: 2659 | print(decompile(sys.argv[1])) 2660 | --------------------------------------------------------------------------------