├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── docs └── ttyd_structures_pseudocode.txt ├── resources ├── enemy_names.txt ├── eu_symbols.csv ├── item_names.txt ├── jp_symbols.csv ├── old_stuff │ └── ttyd_u_symboldiffs_20200707.csv └── us_symbols.csv └── source ├── README.md ├── annotate_map_symbols.py ├── combine_event_dumps.py ├── combine_rels.py ├── dump_sections.py ├── export_classes.py ├── export_classes_parsers.py ├── export_events.py ├── jdalibpy ├── bindatastore.py ├── bindump.py ├── conv.py ├── flags.py └── rngutil.py ├── map_to_symbols.py ├── old_utils ├── ttyd_exporteventscripts.py ├── ttyd_extractclassdata.py ├── ttyd_generatesymbolmaps.py └── ttyd_maplib.py ├── setup.py ├── sort_events_by_prefix.py └── symbol_to_maps.py /.gitattributes: -------------------------------------------------------------------------------- 1 | ############################################################################### 2 | # Set default behavior to automatically normalize line endings. 3 | ############################################################################### 4 | * text=auto 5 | 6 | ############################################################################### 7 | # Set default behavior for command prompt diff. 8 | # 9 | # This is need for earlier builds of msysgit that does not have it on by 10 | # default for csharp files. 11 | # Note: This is only used by command line 12 | ############################################################################### 13 | #*.cs diff=csharp 14 | 15 | ############################################################################### 16 | # Set the merge driver for project and solution files 17 | # 18 | # Merging from the command prompt will add diff markers to the files if there 19 | # are conflicts (Merging from VS is not affected by the settings below, in VS 20 | # the diff markers are never inserted). Diff markers may cause the following 21 | # file extensions to fail to load in VS. An alternative would be to treat 22 | # these files as binary and thus will always conflict and require user 23 | # intervention with every merge. To do so, just uncomment the entries below 24 | ############################################################################### 25 | #*.sln merge=binary 26 | #*.csproj merge=binary 27 | #*.vbproj merge=binary 28 | #*.vcxproj merge=binary 29 | #*.vcproj merge=binary 30 | #*.dbproj merge=binary 31 | #*.fsproj merge=binary 32 | #*.lsproj merge=binary 33 | #*.wixproj merge=binary 34 | #*.modelproj merge=binary 35 | #*.sqlproj merge=binary 36 | #*.wwaproj merge=binary 37 | 38 | ############################################################################### 39 | # behavior for image files 40 | # 41 | # image files are treated as binary by default. 42 | ############################################################################### 43 | #*.jpg binary 44 | #*.png binary 45 | #*.gif binary 46 | 47 | ############################################################################### 48 | # diff behavior for common document formats 49 | # 50 | # Convert binary document formats to text before diffing them. This feature 51 | # is only available from the command line. Turn it on by uncommenting the 52 | # entries below. 53 | ############################################################################### 54 | #*.doc diff=astextplain 55 | #*.DOC diff=astextplain 56 | #*.docx diff=astextplain 57 | #*.DOCX diff=astextplain 58 | #*.dot diff=astextplain 59 | #*.DOT diff=astextplain 60 | #*.pdf diff=astextplain 61 | #*.PDF diff=astextplain 62 | #*.rtf diff=astextplain 63 | #*.RTF diff=astextplain 64 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # Jda TTYD Utils exceptions 5 | bazel-*/* # Bazel generated files. 6 | bin/* # Local outputs from Python binaries, etc. 7 | deps/* # Local dependencies (RAM dumps, MAP files, PistonMiner's ttydasm, etc.) 8 | 9 | # User-specific files 10 | *.suo 11 | *.user 12 | *.userosscache 13 | *.sln.docstates 14 | 15 | # User-specific files (MonoDevelop/Xamarin Studio) 16 | *.userprefs 17 | 18 | # Build results 19 | [Dd]ebug/ 20 | [Dd]ebugPublic/ 21 | [Rr]elease/ 22 | [Rr]eleases/ 23 | x64/ 24 | x86/ 25 | bld/ 26 | [Bb]in/ 27 | [Oo]bj/ 28 | [Ll]og/ 29 | 30 | # Visual Studio 2015 cache/options directory 31 | .vs/ 32 | # Uncomment if you have tasks that create the project's static files in wwwroot 33 | #wwwroot/ 34 | 35 | # MSTest test Results 36 | [Tt]est[Rr]esult*/ 37 | [Bb]uild[Ll]og.* 38 | 39 | # NUNIT 40 | *.VisualState.xml 41 | TestResult.xml 42 | 43 | # Build Results of an ATL Project 44 | [Dd]ebugPS/ 45 | [Rr]eleasePS/ 46 | dlldata.c 47 | 48 | # DNX 49 | project.lock.json 50 | project.fragment.lock.json 51 | artifacts/ 52 | 53 | *_i.c 54 | *_p.c 55 | *_i.h 56 | *.ilk 57 | *.meta 58 | *.obj 59 | *.pch 60 | *.pdb 61 | *.pgc 62 | *.pgd 63 | *.rsp 64 | *.sbr 65 | *.tlb 66 | *.tli 67 | *.tlh 68 | *.tmp 69 | *.tmp_proj 70 | *.log 71 | *.vspscc 72 | *.vssscc 73 | .builds 74 | *.pidb 75 | *.svclog 76 | *.scc 77 | 78 | # Chutzpah Test files 79 | _Chutzpah* 80 | 81 | # Visual C++ cache files 82 | ipch/ 83 | *.aps 84 | *.ncb 85 | *.opendb 86 | *.opensdf 87 | *.sdf 88 | *.cachefile 89 | *.VC.db 90 | *.VC.VC.opendb 91 | 92 | # Visual Studio profiler 93 | *.psess 94 | *.vsp 95 | *.vspx 96 | *.sap 97 | 98 | # TFS 2012 Local Workspace 99 | $tf/ 100 | 101 | # Guidance Automation Toolkit 102 | *.gpState 103 | 104 | # ReSharper is a .NET coding add-in 105 | _ReSharper*/ 106 | *.[Rr]e[Ss]harper 107 | *.DotSettings.user 108 | 109 | # JustCode is a .NET coding add-in 110 | .JustCode 111 | 112 | # TeamCity is a build add-in 113 | _TeamCity* 114 | 115 | # DotCover is a Code Coverage Tool 116 | *.dotCover 117 | 118 | # NCrunch 119 | _NCrunch_* 120 | .*crunch*.local.xml 121 | nCrunchTemp_* 122 | 123 | # MightyMoose 124 | *.mm.* 125 | AutoTest.Net/ 126 | 127 | # Web workbench (sass) 128 | .sass-cache/ 129 | 130 | # Installshield output folder 131 | [Ee]xpress/ 132 | 133 | # DocProject is a documentation generator add-in 134 | DocProject/buildhelp/ 135 | DocProject/Help/*.HxT 136 | DocProject/Help/*.HxC 137 | DocProject/Help/*.hhc 138 | DocProject/Help/*.hhk 139 | DocProject/Help/*.hhp 140 | DocProject/Help/Html2 141 | DocProject/Help/html 142 | 143 | # Click-Once directory 144 | publish/ 145 | 146 | # Publish Web Output 147 | *.[Pp]ublish.xml 148 | *.azurePubxml 149 | # TODO: Comment the next line if you want to checkin your web deploy settings 150 | # but database connection strings (with potential passwords) will be unencrypted 151 | #*.pubxml 152 | *.publishproj 153 | 154 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 155 | # checkin your Azure Web App publish settings, but sensitive information contained 156 | # in these scripts will be unencrypted 157 | PublishScripts/ 158 | 159 | # NuGet Packages 160 | *.nupkg 161 | # The packages folder can be ignored because of Package Restore 162 | **/packages/* 163 | # except build/, which is used as an MSBuild target. 164 | !**/packages/build/ 165 | # Uncomment if necessary however generally it will be regenerated when needed 166 | #!**/packages/repositories.config 167 | # NuGet v3's project.json files produces more ignoreable files 168 | *.nuget.props 169 | *.nuget.targets 170 | 171 | # Microsoft Azure Build Output 172 | csx/ 173 | *.build.csdef 174 | 175 | # Microsoft Azure Emulator 176 | ecf/ 177 | rcf/ 178 | 179 | # Windows Store app package directories and files 180 | AppPackages/ 181 | BundleArtifacts/ 182 | Package.StoreAssociation.xml 183 | _pkginfo.txt 184 | 185 | # Visual Studio cache files 186 | # files ending in .cache can be ignored 187 | *.[Cc]ache 188 | # but keep track of directories ending in .cache 189 | !*.[Cc]ache/ 190 | 191 | # Others 192 | ClientBin/ 193 | ~$* 194 | *~ 195 | *.dbmdl 196 | *.dbproj.schemaview 197 | *.jfm 198 | *.pfx 199 | *.publishsettings 200 | node_modules/ 201 | orleans.codegen.cs 202 | 203 | # Since there are multiple workflows, uncomment next line to ignore bower_components 204 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 205 | #bower_components/ 206 | 207 | # RIA/Silverlight projects 208 | Generated_Code/ 209 | 210 | # Backup & report files from converting an old project file 211 | # to a newer Visual Studio version. Backup files are not needed, 212 | # because we have git ;-) 213 | _UpgradeReport_Files/ 214 | Backup*/ 215 | UpgradeLog*.XML 216 | UpgradeLog*.htm 217 | 218 | # SQL Server files 219 | *.mdf 220 | *.ldf 221 | 222 | # Business Intelligence projects 223 | *.rdl.data 224 | *.bim.layout 225 | *.bim_*.settings 226 | 227 | # Microsoft Fakes 228 | FakesAssemblies/ 229 | 230 | # GhostDoc plugin setting file 231 | *.GhostDoc.xml 232 | 233 | # Node.js Tools for Visual Studio 234 | .ntvs_analysis.dat 235 | 236 | # Visual Studio 6 build log 237 | *.plg 238 | 239 | # Visual Studio 6 workspace options file 240 | *.opt 241 | 242 | # Visual Studio LightSwitch build output 243 | **/*.HTMLClient/GeneratedArtifacts 244 | **/*.DesktopClient/GeneratedArtifacts 245 | **/*.DesktopClient/ModelManifest.xml 246 | **/*.Server/GeneratedArtifacts 247 | **/*.Server/ModelManifest.xml 248 | _Pvt_Extensions 249 | 250 | # Paket dependency manager 251 | .paket/paket.exe 252 | paket-files/ 253 | 254 | # FAKE - F# Make 255 | .fake/ 256 | 257 | # JetBrains Rider 258 | .idea/ 259 | *.sln.iml 260 | 261 | # CodeRush 262 | .cr/ 263 | 264 | # Python Tools for Visual Studio (PTVS) 265 | __pycache__/ 266 | *.pyc -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### Jdaster64's PM:TTYD Utils & Docs (latest update: 2023-04-07) 2 | 3 | ### Credits 4 | * **PistonMiner** for the TTYD scripting disassembly tool, ttydasm. (GitHub repo [here](https://github.com/PistonMiner/ttyd-tools).) 5 | * PistonMiner and **Zephiles** for their contributions to the symbol maps, including nearly all of the main binary's .text symbols. 6 | * Both of the above, **Jasper**, **SolidifiedGaming**, and others who've helped with TTYD documentation elsewhere. 7 | * **Rain** for writing the setup.py script to make setting up the TTYD utilities easier, as well as sort_events_by_prefix. 8 | 9 | ### Contents 10 | * **docs**: 11 | * **ttyd_structures_pseudocode** - Notes on various TTYD structure layouts / enums, mostly for battle-related stuff. 12 | * **resources**: 13 | * **us_symbols.csv** - A near-complete symbol table for the US retail version of TTYD, including some type information (including marking ~all evts). 14 | * **eu_symbols.csv** - Same for the European / PAL version. 15 | * **jp_symbols.csv** - Same for the retail JP version. 16 | * Text files containing TTYD's battle units' and items' names in ID order, one per line. 17 | * A dump from an old curated table of symbol diffs between the JP demo and US retail versions of TTYD, used to locate individual functions / data / class instances (was used for older Python scripts) 18 | * **source**: 19 | * **jdalibpy**: 20 | * General command-line flag (**flags**) & binary memory view utilities (**bindatastore**, and its crustier brother **bindump** for the old utils). 21 | * A couple unrelated tools I use occasionally on the command-line: 22 | * **conv** - Converts between various numeric and datetime formats. 23 | * **rngutil** - Simulates TTYD's (and a few other games') random number generators. 24 | * **TTYD utility scripts**: (explained more fully in source/README.md) 25 | * **setup** - Runs all the important scripts end-to-end. 26 | * **dump_sections** - Provided .dol/.rel files from TTYD, dumps their individual sections. 27 | * **symbol_to_maps** - Converts a symbol table to .MAP files and symbol files for PistonMiner's *ttydasm* tool. 28 | * **extract_events** - Exports all labeled evts in a symbol table to text files using *ttydasm*. 29 | * **extract_classes** - Exports all labeled instances of various structures to .csv files, one for their fields and one for byte representations. 30 | * **map_to_symbols**, **annotate_map_symbols** - For converting existing .MAP files for other versions of TTYD (e.g. the JP demo) to symbol tables, for use with the other scripts. 31 | * Optional utilities: 32 | * **combine_event_dumps** - Combines all unique event dumps from extract_events into a single text file. 33 | * **combine_rels** - Builds custom REL files composed of symbols from one or more existing RELs. 34 | * **sort_events_by_prefix** - Separates event dumps into subfolders based on their prefix. 35 | * **old_utils**: 36 | * Were used for some of the same purposes as the newer utilities, but much messier and generally less fully-featured. -------------------------------------------------------------------------------- /resources/enemy_names.txt: -------------------------------------------------------------------------------- 1 | Nothing 2 | Goomba 3 | Paragoomba 4 | Spiky Goomba 5 | Spinia 6 | Spania 7 | Lord Crump (Prologue) 8 | Gus 9 | Blooper 10 | Left Tentacle 11 | Right Tentacle 12 | Koopatrol 13 | Magikoopa 14 | Magikoopa (fake) 15 | Koopa 16 | Paratroopa 17 | Fuzzy 18 | Dull Bones 19 | Bald Cleft 20 | Bristle 21 | Gold Fuzzy 22 | Fuzzy Horde 23 | Red Bones 24 | Hooktail 25 | Dark Puff 26 | Pale Piranha 27 | Cleft 28 | Pider 29 | X-Naut 30 | Yux 31 | Mini-Yux 32 | Beldam (Ch. 2) 33 | Marilyn (Ch. 2) 34 | Vivian 35 | Magnus 36 | X-Fist 37 | Goomba (Glitzville) 38 | KP Koopa 39 | KP Paratroopa 40 | Pokey 41 | Lakitu 42 | Spiny 43 | H. Bald Cleft 44 | Bob-omb 45 | Bandit 46 | Big Bandit 47 | R. S. Buzzy 48 | Shady Koopa 49 | S. Paratroopa 50 | R. Magikoopa 51 | R. Magikoopa (fake) 52 | W. Magikoopa 53 | W. Magikoopa (fake) 54 | G. Magikoopa 55 | G. Magikoopa (fake) 56 | Dark Craw 57 | Hammer Bro 58 | Boomerang Bro 59 | Fire Bro 60 | Red Chomp 61 | D. Koopatrol 62 | Iron Cleft 1 63 | Iron Cleft 2 64 | Bowser (Ch. 3) 65 | Rawk Hawk 66 | Macho Grubba 67 | Hyper Goomba 68 | H. Paragoomba 69 | H. S. Goomba 70 | Crazee Dayzee 71 | Amazy Dayzee 72 | Hyper Cleft 73 | Buzzy Beetle 74 | Spike Top 75 | Swooper 76 | Boo 77 | Atomic Boo 78 | Doopliss (Ch. 4, first) 79 | Doopliss (Ch. 4, invincible) 80 | Doopliss (Ch. 4, second) 81 | Goombella (Ch. 4) 82 | Koops (Ch. 4) 83 | Yoshi (Ch. 4) 84 | Flurrie (Ch. 4) 85 | Ember 86 | Lava Bubble 87 | Green Fuzzy 88 | Flower Fuzzy 89 | Putrid Piranha 90 | Parabuzzy 91 | Bill Blaster 92 | Bullet Bill 93 | Bulky Bob-omb 94 | Cortez 95 | Cortez Bone Pile 96 | Cortez Sword 97 | Cortez Claw 98 | Cortez Rapier 99 | Cortez Saber 100 | Lord Crump (Ch. 5) 101 | X-Nauts [1] 102 | X-Nauts [2] 103 | X-Nauts [3] 104 | Ruff Puff 105 | Poison Pokey 106 | S. Parabuzzy 107 | Dark Boo 108 | Smorg 109 | Smorg Tentacle A 110 | Smorg Tentacle B 111 | Smorg Tentacle C 112 | Smorg Claw 113 | Ice Puff 114 | Frost Piranha 115 | Moon Cleft 116 | Z-Yux 117 | Mini-Z-Yux 118 | X-Yux 119 | Mini-X-Yux 120 | X-Naut PhD 121 | Elite X-Naut 122 | Magnus 2.0 123 | X-Punch 124 | Swoopula 125 | Phantom Ember 126 | B. Bill Blaster 127 | Bombshell Bill 128 | Chain Chomp 129 | Dark Wizzerd 130 | Dark Wizzerd (fake) 131 | Dry Bones 132 | Dark Bones 133 | Gloomtail 134 | Beldam (Ch. 8) 135 | Marilyn (Ch. 8) 136 | Doopliss (Ch. 8) 137 | Doopliss (Ch. 8, Fake Mario) 138 | Doopliss (Ch. 8, Goombella) 139 | Doopliss (Ch. 8, Koops) 140 | Doopliss (Ch. 8, Yoshi) 141 | Doopliss (Ch. 8, Flurrie) 142 | Doopliss (Ch. 8, Vivian) 143 | Doopliss (Ch. 8, Bobbery) 144 | Doopliss (Ch. 8, Ms. Mowz) 145 | Bowser (Ch. 8) 146 | Kammy Koopa 147 | Grodus 148 | Grodus X 149 | Shadow Queen (First phase) 150 | Shadow Queen (Invincible phase) 151 | Shadow Queen (Second phase) 152 | L/R Hand 153 | Dead Hands 154 | Gloomba 155 | Paragloomba 156 | Spiky Gloomba 157 | Dark Koopa 158 | D. Paratroopa 159 | Badge Bandit 160 | Dark Lakitu 161 | S. Blue Spiny 162 | Wizzerd 163 | Piranha Plant 164 | Spunia 165 | Arantula 166 | Dark Bristle 167 | Poison Puff 168 | Swampire 169 | Bob-ulk 170 | Elite Wizzerd 171 | Elite Wizzerd (fake) 172 | Bonetail 173 | [Red Buzzy] 174 | [R. Parabuzzy] 175 | [R. S. Parabuzzy] 176 | [Hyper Bob-omb] 177 | [Ultra Bob-omb] 178 | [Tutorial Goombella] 179 | [Tutorial Frankly B2] 180 | [Tutorial Frankly B3] 181 | [Tutorial Frankly B4] 182 | [Epilogue Doopliss] 183 | [Epilogue Flurrie] 184 | [Epilogue Boo] 185 | [Epilogue Atomic Boo] 186 | [Epilogue Toad] 187 | [Epilogue Female Toad] 188 | [Unused Test] 189 | [Unused Kanbu 2] 190 | [Unused Beldam 2] 191 | [Unused Marilyn 2] 192 | [Unused Vivian 2] 193 | [Unused Beldam 3] 194 | [Unused Marilyn 3] 195 | [Unused Mecha-Goomba] 196 | [Unused Mecha-Turtle] 197 | [Unused Okorl] 198 | [Unused Yowarl] 199 | [Unused Tuyonarl] 200 | [Unused Wanawana] 201 | [Unused Apprentice Magikoopa] 202 | [Unused Shy Guy] 203 | [Unused Groove Guy] 204 | [Unused Pyro Guy] 205 | [Unused Spy Guy] 206 | [Unused Anti Guy] 207 | [Unused Bzzap!] 208 | [Unused Mini-Bzzap!] 209 | [Unused UFO] 210 | [Unused Pennington] 211 | [Unused Fighter] 212 | [Unused Zess T.] 213 | [Unused Master] 214 | [Unused Reporter] 215 | [Unused Hot-Dog Master] 216 | [Unused Flavio] 217 | [Unused Tree] 218 | [Unused Switch] 219 | [Unused Test NPC] 220 | Bomb-Squad Bomb 221 | System 222 | [Prologue NPC Goombella] 223 | Mario 224 | Shell Shield 225 | Goombella 226 | Koops 227 | Yoshi 228 | Flurrie 229 | Vivian 230 | Bobbery 231 | Ms. Mowz 232 | -------------------------------------------------------------------------------- /resources/item_names.txt: -------------------------------------------------------------------------------- 1 | 000 - Nothing 2 | 001 - Strange Sack 3 | 002 - Paper Curse 4 | 003 - Tube Curse 5 | 004 - Plane Curse 6 | 005 - Boat Curse 7 | 006 - Boots 8 | 007 - Super Boots 9 | 008 - Ultra Boots 10 | 009 - Hammer 11 | 00a - Super Hammer 12 | 00b - Ultra Hammer 13 | 00c - Castle Key 14 | 00d - Castle Key 15 | 00e - Castle Key 16 | 00f - Castle Key 17 | 010 - Red Key 18 | 011 - Blue Key 19 | 012 - Storage Key 20 | 013 - Storage Key 21 | 014 - Grotto Key 22 | 015 - Shop Key 23 | 016 - Steeple Key 24 | 017 - Steeple Key 25 | 018 - Station Key 26 | 019 - Station Key 27 | 01a - Elevator Key 28 | 01b - Elevator Key 29 | 01c - Elevator Key 30 | 01d - Card Key 31 | 01e - Card Key 32 | 01f - Card Key 33 | 020 - Card Key 34 | 021 - Black Key 35 | 022 - Black Key 36 | 023 - Black Key 37 | 024 - Black Key 38 | 025 - Star Key 39 | 026 - Palace Key 40 | 027 - Palace Key 41 | 028 - Palace Key 42 | 029 - Palace Key 43 | 02a - Palace Key 44 | 02b - Palace Key 45 | 02c - Palace Key 46 | 02d - Palace Key 47 | 02e - Palace Key 48 | 02f - Palace Key 49 | 030 - Palace Key 50 | 031 - House Key 51 | 032 - Magical Map 52 | 033 - Contact Lens 53 | 034 - Blimp Ticket 54 | 035 - Train Ticket 55 | 036 - Mailbox SP 56 | 037 - Super Luigi 57 | 038 - Super Luigi 2 58 | 039 - Super Luigi 3 59 | 03a - Super Luigi 4 60 | 03b - Super Luigi 5 61 | 03c - Cookbook 62 | 03d - Moon Stone 63 | 03e - Sun Stone 64 | 03f - Necklace 65 | 040 - Puni Orb 66 | 041 - Champ's Belt 67 | 042 - Poisoned Cake 68 | 043 - Superbombomb 69 | 044 - The Letter "p" 70 | 045 - Old Letter 71 | 046 - Chuckola Cola 72 | 047 - Skull Gem 73 | 048 - Gate Handle 74 | 049 - Wedding Ring 75 | 04a - Galley Pot 76 | 04b - Gold Ring 77 | 04c - Shell Earrings 78 | 04d - Autograph 79 | 04e - Ragged Diary 80 | 04f - Blanket 81 | 050 - Vital Paper 82 | 051 - Briefcase 83 | 052 - Goldbob Guide 84 | 053 - Invalid Item 0053 (paper w/ award) 85 | 054 - Invalid Item 0054 (paper w/ award) 86 | 055 - Cog 87 | 056 - Data Disk 88 | 057 - Shine Sprite 89 | 058 - Ultra Stone 90 | 059 - Invalid Item 0059 (Bowser's upgrade meat) 91 | 05a - Invalid Item 005A (Poster w/ Mario) 92 | 05b - Special Card 93 | 05c - Platinum Card 94 | 05d - Gold Card 95 | 05e - Silver Card 96 | 05f - Box 97 | 060 - Magical Map (Bigger?) 98 | 061 - Dubious Paper 99 | 062 - Routing Slip 100 | 063 - Wrestling Mag 101 | 064 - Present 102 | 065 - Blue Potion 103 | 066 - Red Potion 104 | 067 - Orange Potion 105 | 068 - Green Potion 106 | 069 - Invalid Item 0069 / fn0ow (star?) 107 | 06a - Lottery Pick 108 | 06b - Battle Trunks 109 | 06c - Up Arrow 110 | 06d - Package 111 | 06e - Attack FX B 112 | 06f - Invalid Item 006F 113 | 070 - Invalid Item 0070 114 | 071 - Invalid Item 0071 115 | 072 - Diamond Star 116 | 073 - Emerald Star 117 | 074 - Gold Star 118 | 075 - Ruby Star 119 | 076 - Sapphire Star 120 | 077 - Garnet Star 121 | 078 - Crystal Star 122 | 079 - Coin 123 | 07a - Pianta 124 | 07b - Heart pickup 125 | 07c - Flower pickup 126 | 07d - Star Piece 127 | 07e - Gold Bar 128 | 07f - Gold Bar x3 129 | 080 - Thunder Bolt 130 | 081 - Thunder Rage 131 | 082 - Shooting Star 132 | 083 - Ice Storm 133 | 084 - Fire Flower 134 | 085 - Earth Quake 135 | 086 - Boo's Sheet 136 | 087 - Volt Shroom 137 | 088 - Repel Cape 138 | 089 - Ruin Powder 139 | 08a - Sleepy Sheep 140 | 08b - POW Block 141 | 08c - Stopwatch 142 | 08d - Dizzy Dial 143 | 08e - Power Punch 144 | 08f - Courage Shell 145 | 090 - HP Drain (Item) 146 | 091 - [Trade Off] 147 | 092 - Mini Mr. Mini 148 | 093 - Mr. Softener 149 | 094 - Mushroom 150 | 095 - Super Shroom 151 | 096 - Ultra Shroom 152 | 097 - Life Shroom 153 | 098 - Dried Shroom 154 | 099 - Tasty Tonic 155 | 09a - Honey Syrup 156 | 09b - Maple Syrup 157 | 09c - Jammin' Jelly 158 | 09d - Slow Shroom 159 | 09e - Gradual Syrup 160 | 09f - Hot Dog 161 | 0a0 - [Cake] 162 | 0a1 - Point Swap 163 | 0a2 - Fright Mask 164 | 0a3 - Mystery 165 | 0a4 - Inn Coupon 166 | 0a5 - Whacka Bump 167 | 0a6 - Coconut 168 | 0a7 - Dried Bouquet 169 | 0a8 - Mystic Egg 170 | 0a9 - Golden Leaf 171 | 0aa - Keel Mango 172 | 0ab - Fresh Pasta 173 | 0ac - Cake Mix 174 | 0ad - Hot Sauce 175 | 0ae - Turtley Leaf 176 | 0af - Horsetail 177 | 0b0 - Peachy Peach 178 | 0b1 - Spite Pouch 179 | 0b2 - [Koopa Curse] 180 | 0b3 - Shroom Fry 181 | 0b4 - Shroom Roast 182 | 0b5 - Shroom Steak 183 | 0b6 - Mistake 184 | 0b7 - Honey Shroom 185 | 0b8 - Maple Shroom 186 | 0b9 - Jelly Shroom 187 | 0ba - Honey Super 188 | 0bb - Maple Super 189 | 0bc - Jelly Super 190 | 0bd - Honey Ultra 191 | 0be - Maple Ultra 192 | 0bf - Jelly Ultra 193 | 0c0 - Spicy Soup 194 | 0c1 - Zess Dinner 195 | 0c2 - Zess Special 196 | 0c3 - Zess Deluxe 197 | 0c4 - Zess Dynamite 198 | 0c5 - Zess Tea 199 | 0c6 - Space Food 200 | 0c7 - Icicle Pop 201 | 0c8 - Zess Frappe 202 | 0c9 - Snow Bunny 203 | 0ca - Coconut Bomb 204 | 0cb - Courage Meal 205 | 0cc - Shroom Cake 206 | 0cd - Shroom Crepe 207 | 0ce - Mousse Cake 208 | 0cf - Fried Egg 209 | 0d0 - Fruit Parfait 210 | 0d1 - Egg Bomb 211 | 0d2 - Ink Pasta 212 | 0d3 - Spaghetti 213 | 0d4 - Shroom Broth 214 | 0d5 - Poison Shroom 215 | 0d6 - Choco Cake 216 | 0d7 - Mango Delight 217 | 0d8 - Love Pudding 218 | 0d9 - Meteor Meal 219 | 0da - Trial Stew 220 | 0db - Couple's Cake 221 | 0dc - Inky Sauce 222 | 0dd - Omelette Meal 223 | 0de - Koopa Tea 224 | 0df - Koopasta 225 | 0e0 - Spicy Pasta 226 | 0e1 - Heartful Cake 227 | 0e2 - Peach Tart 228 | 0e3 - Electro Pop 229 | 0e4 - Fire Pop 230 | 0e5 - Honey Candy 231 | 0e6 - Coco Candy 232 | 0e7 - Jelly Candy 233 | 0e8 - Zess Cookie 234 | 0e9 - Healthy Salad 235 | 0ea - Koopa Bun 236 | 0eb - Fresh Juice 237 | 0ec - Audience Can 238 | 0ed - Audience Rock 239 | 0ee - Audience Bone 240 | 0ef - Audience Hammer 241 | 0f0 - Power Jump 242 | 0f1 - Multibounce 243 | 0f2 - Power Bounce 244 | 0f3 - Tornado Jump 245 | 0f4 - Shrink Stomp 246 | 0f5 - Sleepy Stomp 247 | 0f6 - Soft Stomp 248 | 0f7 - Power Smash 249 | 0f8 - Quake Hammer 250 | 0f9 - Hammer Throw 251 | 0fa - Piercing Blow 252 | 0fb - Head Rattle 253 | 0fc - Fire Drive 254 | 0fd - Ice Smash 255 | 0fe - Double Dip 256 | 0ff - Double Dip P 257 | 100 - Charge 258 | 101 - Charge P 259 | 102 - Super Appeal 260 | 103 - Super Appeal P 261 | 104 - Power Plus 262 | 105 - Power Plus P 263 | 106 - P-Up D-Down 264 | 107 - P-Up D-Down P 265 | 108 - All or Nothing 266 | 109 - [All or Nothing P] 267 | 10a - Mega Rush 268 | 10b - Mega Rush P 269 | 10c - Power Rush 270 | 10d - Power Rush P 271 | 10e - P-Down D-Up 272 | 10f - P-Down D-Up P 273 | 110 - Last Stand 274 | 111 - Last Stand P 275 | 112 - Defend Plus 276 | 113 - Defend Plus P 277 | 114 - Damage Dodge 278 | 115 - Damage Dodge P 279 | 116 - HP Plus 280 | 117 - HP Plus P 281 | 118 - FP Plus 282 | 119 - Flower Saver 283 | 11a - Flower Saver P 284 | 11b - Ice Power 285 | 11c - Spike Shield 286 | 11d - Feeling Fine 287 | 11e - Feeling Fine P 288 | 11f - Zap Tap 289 | 120 - Double Pain 290 | 121 - Jumpman 291 | 122 - Hammerman 292 | 123 - Return Postage 293 | 124 - Happy Heart 294 | 125 - Happy Heart P 295 | 126 - Happy Flower 296 | 127 - HP Drain (Badge) 297 | 128 - HP Drain P 298 | 129 - FP Drain 299 | 12a - [FP Drain P] 300 | 12b - Close Call 301 | 12c - Close Call P 302 | 12d - Pretty Lucky 303 | 12e - Pretty Lucky P 304 | 12f - Lucky Day 305 | 130 - [Lucky Day P] 306 | 131 - Refund 307 | 132 - Pity Flower 308 | 133 - [Pity Flower P] 309 | 134 - Quick Change 310 | 135 - Peekaboo 311 | 136 - Timing Tutor 312 | 137 - Heart Finder 313 | 138 - Flower Finder 314 | 139 - Money Money 315 | 13a - Item Hog 316 | 13b - Attack FX R 317 | 13c - Attack FX B 318 | 13d - Attack FX G 319 | 13e - Attack FX Y 320 | 13f - Attack FX P 321 | 140 - Chill Out 322 | 141 - First Attack 323 | 142 - Bump Attack 324 | 143 - Slow Go 325 | 144 - Simplifier 326 | 145 - Unsimplifier 327 | 146 - Lucky Start 328 | 147 - L Emblem 329 | 148 - W Emblem 330 | 149 - [Triple Dip] 331 | 14a - [Lucky Start P] 332 | 14b - [Auto-Command Badge] 333 | 14c - [Mega Jump] 334 | 14d - [Mega Smash] 335 | 14e - [Mega Quake] 336 | 14f - [Defend Command Badge] 337 | 150 - [Defend Command Badge P] 338 | 151 - [Super Charge] 339 | 152 - [Super Charge P] -------------------------------------------------------------------------------- /source/README.md: -------------------------------------------------------------------------------- 1 | ### Guide to using the TTYD-Utils suite 2 | 3 | The Python utilities in this directory cannot be used directly to create Paper 4 | Mario: The Thousand-Year Door mods, but can be considered a means of creating 5 | a directory in which one can look up the location of code one wishes to change. 6 | 7 | To make full use of these utilities, you need: 8 | 9 | * A reasonably up-to-date Python 3 installation with NumPy and Pandas available. 10 | * A binary of PistonMiner's *ttydasm* tool 11 | (can be obtained [here](https://github.com/PistonMiner/ttyd-tools/releases)). 12 | * The extracted boot.dol and \*.rel files from any version of TTYD. 13 | * Either: 14 | * A .csv containing symbol data for the same version 15 | (**resources/us_symbols.csv**, or its PAL/JP equivalents can be used), or 16 | * .MAP files containing symbol information for the .dol and .rel files for 17 | the same version, which can be used to produce a symbol csv. 18 | 19 | ### Quick Setup 20 | 21 | For the quickest setup, you can run the script **setup.py** on the 22 | command-line. This will check that your DOL and REL dumps are valid, 23 | will attempt to install any needed Python dependencies, and will run all of 24 | the downstream binaries with the correct parameters automatically. 25 | 26 | **Sample invocation** (from within the ttyd_utils/source directory): 27 | ``` 28 | setup.py \ 29 | --ver=VERSION (one of us, jp, or pal) 30 | --out=YOUR_OUTPUT_PATH \ 31 | --dol_filepath=PATH_TO_YOUR_DOL.dol \ 32 | --rels_directory=PATH_TO_DIRECTORY_CONTAINING_RELS 33 | --ttydasm=PATH_TO_TTYDASM_EXE 34 | ``` 35 | 36 | Alternatively, you can run the utilities in this directory individually in the 37 | following order (making sure to use the same *--out_path* for each invocation): 38 | 39 | ### Step 1. Dump the raw data from the sections of the .dol/.rel files. 40 | 41 | **dump_sections.py** takes the .dol/.rel files provided, and outputs the following: 42 | * A .csv file containing information regarding the type, size, and file-/RAM-relative 43 | locations of the sections of all the provided binary files. 44 | * .raw dumps of every complete binary file and individual section. 45 | 46 | Providing a link address (with optional area-specific overrides) 47 | will also generate versions of the .REL file / section dumps that have 48 | gone through the linking process, meaning e.g. pointers in evts or data 49 | structures will have their values filled in rather than being placeholders. 50 | Providing *some* value is required for many of the later utilities to run 51 | correctly; by default, --link_address is set to an arbitrary value far enough 52 | into the RAM space to not overlap the .DOL's sections, but it isn't the real 53 | link address used for any particular version of TTYD. 54 | 55 | Providing a rel_bss address will ensure that the .bss sections in .rel files 56 | are correctly linked to a separate memory location; supplying an accurate 57 | address (the address of the symbol *rel_bss seq_mapchange.o*) is not required, 58 | but doing so will make sure the correct addresses are used for .rel BSS symbols 59 | when exporting .MAP files in Step 3. 60 | 61 | **Sample invocation:** 62 | ``` 63 | dump_sections.py \ 64 | --out_path=YOUR_OUTPUT_PATH \ 65 | --dol=PATH_TO_YOUR_DOL.dol \ 66 | --rel=PATH_TO_YOUR_REL_FOR_AREA_*.rel \ 67 | --link_address=0x80600000 \ 68 | --link_address_overrides=jon:0x80c00000,tst:0x80c00000 \ 69 | --rel_bss_address=0x80a00000 70 | ``` 71 | 72 | **As a side note**, if you're most interested in using these scripts as reference to 73 | hex-edit the DOL or RELs, the symbol maps generated in Step 3 will tell you the 74 | section-relative address of each symbol in the game, and you can add those 75 | addresses to the file-level start addresses for their respective section listed 76 | in the *section_info.csv* file generated by this step. 77 | 78 | For instance, Lakitu's item drop table (*battle_item_jyugem battle_database_common.o*) 79 | is located in the DOL's .data section with a section offset of **0x6b550**. 80 | The *section_info.csv* file lists the DOL's .data section as starting at 81 | file-level address **0x3011e0**. Adding these together gives you the address 82 | 0x3011e0 + 0x6b550 = **0x36c730**, which is the offset in the boot.dol file 83 | that item drop table can be found at. 84 | 85 | ### Step 2. If necessary, convert your externally-sourced .MAP files to a symbol table. 86 | 87 | * **NOTE:** *You can skip this step if using one of the already-provided **resources/\*_symbols.csv** files.* 88 | 89 | Running the **map_to_symbols** and **annotate_map_symbols** utilities in sequence 90 | will produce a .csv of symbols with the same columns as **resources/us_symbols.csv**, 91 | and some heuristically-predicted common types (notably, strings and evts). 92 | The resulting file will be in `YOUR_OUTPUT_PATH/annotated_symbols.csv`. 93 | 94 | **Sample invocations:** 95 | ``` 96 | map_to_symbols.py \ 97 | --out_path=YOUR_OUTPUT_PATH \ 98 | --dol_map=PATH_TO_YOUR_DOL_MAP.map \ 99 | --rel_map=PATH_TO_YOUR_REL_MAP_FOR_AREA_*.map 100 | 101 | annotate_map_symbols.py --out_path=YOUR_OUTPUT_PATH 102 | ``` 103 | 104 | * You can also specify --encoding=YOUR_ALTERNATE_ENCODING if the .MAP files are 105 | in a *codecs*-supported encoding other than the default exported by 106 | **symbol_to_maps** (UTF-8). 107 | 108 | ### Step 3. Export .MAP files and ttydasm symbol files using the symbol table. 109 | 110 | The **symbol_to_maps** utility produces .MAP files from your symbol table, 111 | which can be useful if you don't already have them as a more human-readable 112 | format (as well as for labeling symbols in the Dolphin emulator), as well as 113 | symbol files for ttydasm, which are immensely useful for producing more 114 | readable script dumps. 115 | 116 | The exact files produced are: 117 | * One .MAP file and one ttydasm symbol file containing only the .dol symbols. 118 | * Per .rel file: 119 | * One .MAP file and one ttydasm symbol file containing the .dol's symbols and 120 | that .rel's symbols (w/RAM-relative addresses, as specified in Step 1). 121 | * One .MAP file containing only the .rel's symbols (w/section-relative addresses). 122 | 123 | **Sample invocation:** 124 | ``` 125 | symbol_to_maps.py \ 126 | --out_path=YOUR_OUTPUT_PATH \ 127 | --symbols_path=PATH_TO_YOUR_SYMBOLS_FILE.csv \ 128 | --rel_bss_address=0x80a00000 129 | ``` 130 | 131 | ### Step 4. Export evt scripts to text files using ttydasm. 132 | 133 | The **export_events** utility takes the ttydasm symbol files and dumped 134 | sections, and uses PistonMiner's *ttydasm* tool to produce text dumps of 135 | all symbols labeled with the "evt" type in the provided symbols csv. 136 | 137 | The provided **resources/us_symbols.csv** file should have ~all such evts 138 | in the US version of TTYD marked accurately; barring that, the binary in Step 2 139 | should be fairly good at identifying them if you have the start addresses 140 | correctly labeled. 141 | 142 | **Example output script:** 143 | ``` 144 | ... 145 | 805C6FE4: switchi GSW(0) 146 | 805C6FEC: case_int_lt 336 147 | 805C6FF4: callc [evt_npc_set_position evt_npc.o] ["me"] 30 10 -950 148 | 805C700C: callc [evt_npc_set_ry evt_npc.o] ["me"] 270 149 | 805C701C: end_switch 150 | ... 151 | ``` 152 | 153 | **Sample invocation:** 154 | ``` 155 | export_events.py \ 156 | --out_path=YOUR_OUTPUT_PATH \ 157 | --symbols_path=PATH_TO_YOUR_SYMBOLS_FILE.csv \ 158 | --ttydasm_exe=PATH_TO_TTYDASM.exe 159 | ``` 160 | 161 | ### Step 5. Export instances of certain C data types to .csv files. 162 | 163 | The **export_classes** utility takes the dumped section info from Step 1, 164 | and produces .csv dumps containing the field values and bytewise representation 165 | of all instances of the class types supported in **export_classes_parsers**. 166 | 167 | Note that by default nothing will be exported if you're using the 168 | **annotated_symbols.csv** file provided in Step 2; currently none of the 169 | supported types are automatically detectable, and have to be manually annotated. 170 | The **resources/us_symbols.csv** file should have most instances of the 171 | supported types already annotated, however. 172 | 173 | Currently, the list of supported types and their fields are hardcoded, but 174 | support may be added to specify one's own struct definitions in the future. 175 | These are the currently supported types (for the US version, if the struct 176 | varies across versions; see **docs/ttyd_structures_pseudocode.txt** for 177 | further reference): 178 | ``` 179 | AudienceItemWeight 180 | BattleGroupSetup 181 | BattleSetupData 182 | BattleSetupNoTable 183 | BattleSetupWeightedLoadout 184 | BattleStageData 185 | BattleStageFallObjectData 186 | BattleStageNozzleData 187 | BattleUnitPoseTable 188 | BattleStageObjectData 189 | BattleUnitDataTable 190 | BattleUnitDefense 191 | BattleUnitDefenseAttr 192 | BattleUnitKind 193 | BattleUnitKindPart 194 | BattleUnitPoseTable 195 | BattleUnitSetup 196 | BattleWeapon 197 | BeroInfo 198 | CookingRecipe 199 | ItemData 200 | ItemDropData 201 | NpcAiTypeTable 202 | NpcSetupInfo 203 | NpcTribeDescription 204 | PointDropData 205 | ShopItemTable 206 | ShopSellPriceList 207 | StatusVulnerability 208 | ``` 209 | 210 | **Sample invocation:** 211 | ``` 212 | export_classes.py \ 213 | --out_path=YOUR_OUTPUT_PATH \ 214 | --symbols_path=PATH_TO_YOUR_SYMBOLS_FILE.csv 215 | ``` 216 | 217 | ### Additional Tools 218 | 219 | #### combine_event_dumps 220 | 221 | The **combine_event_dumps** utility simply outputs all the event dumps from 222 | **export_events** in a single text file, assuming it has already been run. 223 | 224 | **Sample invocation:** 225 | ``` 226 | combine_event_dumps.py --out_path=YOUR_OUTPUT_PATH 227 | ``` 228 | 229 | #### sort_events_by_prefix 230 | 231 | The **sort_events_by_prefix** utility moves all the event dumps from 232 | **export_events** into subfolders based on their prefixes 233 | (scripts for battle units, specific rels, etc.) 234 | 235 | **Sample invocation:** 236 | ``` 237 | sort_events_by_prefix.py YOUR_OUTPUT_PATH/events 238 | ``` 239 | 240 | #### combine_rels 241 | 242 | The **combine_rels** utility can be used to construct a new relocatable file 243 | composed of combinations of symbols from TTYD's original REL files. This could 244 | then be used in TTYD mods by adding it into a rebuilt ISO, or loaded from the 245 | memory card using PistonMiner's REL framework. 246 | 247 | The utility requires a dump of all the original .REL files that you wish to use, 248 | as well as a text file with newline-delimited data ranges or symbol names to 249 | include. (If using symbol names, you must also provide a symbol info .csv, 250 | such as the **resources/us_symbols.csv** file in this repo for the US version). 251 | 252 | If successful, the utility will output the combined REL, as well as a CSV file 253 | that provides info on where the symbols / ranges are located in the new REL. 254 | 255 | The utility will throw an error if a requested symbol matches multiple entries 256 | in the provided symbol info file, or if a requested symbol / range has linking 257 | dependencies that were not also included in the requested ranges. (Currently, 258 | the utility does not have any functionality for looking up dependencies 259 | proactively, but it should be able to detect if any are missing.) 260 | 261 | Symbols of alignment greater than 4 for the .text section or 8 for the .rodata, 262 | and .data, .bss sections are not supported, nor are sections other than 263 | .text, .rodata, .data, and .bss with indices 1, 4, 5 and 6. 264 | 265 | **Sample invocation for symbol names:** 266 | 267 | ``` 268 | combine_rels.py \ 269 | --out_path=YOUR_OUTPUT_PATH \ 270 | --rel=PATH_TO_YOUR_REL_FOR_AREA_*.rel \ 271 | --symbol_info=PATH_TO_YOUR_SYMBOLS_FILE.csv \ 272 | --symbol_names=SYMBOL_NAMES_PATH.txt 273 | 274 | SYMBOL_NAMES_PATH.txt contents: 275 | 276 | # All symbols related to Piranha Plant unit in MRI rel. 277 | mri:unit_pakkun_flower.o:* 278 | # All symbols related to Ember unit in MUJ rel. 279 | muj:unit_hermos.o:* 280 | # BattleGroupSetup information for an Ember fight in MUJ. 281 | muj:battle_database_muj.o:btlparty_muj_muj_05_01 282 | 283 | ``` 284 | 285 | **Sample invocation for symbol ranges:** 286 | 287 | ``` 288 | combine_rels.py \ 289 | --out_path=YOUR_OUTPUT_PATH \ 290 | --rel=PATH_TO_YOUR_REL_FOR_AREA_*.rel \ 291 | --symbol_ranges=SYMBOL_RANGES_PATH.txt 292 | 293 | SYMBOL_RANGES_PATH.txt contents: 294 | 295 | # All symbols related to Piranha Plant unit in MRI rel. 296 | mri:1:0001ee0c-0001f09c 297 | mri:4:00006ce8-00006ea8 298 | mri:5:00032598-00033758 299 | # All symbols related to Ember unit in MUJ rel. 300 | muj:1:0000bcbc-0000c18c 301 | muj:4:000064a0-00006684 302 | muj:5:00039af0-0003b694 303 | muj:6:00000070-000000c4 304 | # BattleGroupSetup information for an Ember fight in MUJ. 305 | muj:5:00020a78-00020b08 306 | 307 | ``` -------------------------------------------------------------------------------- /source/annotate_map_symbols.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.6 2 | 3 | """Annotates the result of map_to_symbols with heuristic type information. 4 | 5 | This file must be run after the following binaries, using the same --out_path: 6 | - dump_sections.py (to generate section_info and linked section .raw files) 7 | - map_to_symbols.py (to generate map_symbols.csv) 8 | 9 | This program augments the output of map_to_symbols.py, adding file-level and 10 | RAM addresses of the identified symbols, as well as heuristically predicted 11 | common types (e.g. floats, strings, pointers, TTYD evts) and values.""" 12 | # Jonathan Aldrich 2021-01-26 ~ 2021-03-04 13 | 14 | import codecs 15 | import os 16 | import sys 17 | import numpy as np 18 | import pandas as pd 19 | from pathlib import Path 20 | 21 | import jdalibpy.bindatastore as bd 22 | import jdalibpy.flags as flags 23 | 24 | FLAGS = flags.Flags() 25 | 26 | # Output directory; should contain these outputs from previous binaries: 27 | # - Section_info and linked sections from dump_sections.py 28 | # - map_symbols.csv file from map_to_symbols.py 29 | # Annotated symbols will be saved to /annotated_symbols.csv. 30 | FLAGS.DefineString("out_path", "") 31 | 32 | # Whether to display debug strings. 33 | FLAGS.DefineInt("debug_level", 1) 34 | 35 | class AnnotateMapSymbolsError(Exception): 36 | def __init__(self, message=""): 37 | self.message = message 38 | 39 | def _InferType(view, size, exact): 40 | """Uses simple heuristics to try to determine the type/value of a symbol.""" 41 | def _IsFloatCompatible(view, offset=0): 42 | u32 = view.ru32(offset) 43 | # Either 0.0 or in range +/- 1e-7 to 1e7. 44 | return not u32 or (0x33d6bf95 <= (u32 & 2**31-1) <= 0x4b189680) 45 | 46 | def _IsDoubleCompatible(view, offset=0): 47 | u64 = view.ru64(offset) 48 | # Either 0.0 or in range +/- 1e-7 to 1e7. 49 | return not u64 or ( 50 | 0x3e7ad7f29abcaf48 <= (u64 & 2**63-1) <= 0x416312d000000000) 51 | 52 | def _IsPointerCompatible(view, offset=0): 53 | u32 = view.ru32(offset) 54 | # Either 0.0 or in slightly reduced range of valid pointers 55 | # (the range is reduced so as to not be ambiguous w/valid Shift-JIS). 56 | return not u32 or (0x80000000 <= u32 < 0x81400000) 57 | 58 | def _IsEvtCompatible(view, size, exact): 59 | offset = 0 60 | last_command = -1 61 | while offset < size: 62 | command = view.ru32(offset) 63 | # Each command must be between 0x1 and 0x77. 64 | if not (1 <= command & 0xffff <= 0x77): 65 | return False 66 | # Must end in 00000002, 00000001 (RETURN, END). 67 | if last_command == 2 and command == 1: 68 | if not exact: 69 | return True 70 | # Verify that this is the exact end of the evt command array. 71 | return offset + 4 == size 72 | # Advance by 4 bytes, plus 4 per argument to the evt command. 73 | offset += (command >> 16) * 4 + 4 74 | last_command = command 75 | # Reached maximum length of symbol without finding the end of an event. 76 | return False 77 | 78 | def _IsShiftJisCompatible(view, size, exact): 79 | offset = 0 80 | while offset < size: 81 | b = view.ru8(offset) 82 | if b == 0: 83 | # If not exactly at the end of the string, return False. 84 | if exact and offset + 1 != size: 85 | return False 86 | # End of string; double-check for false multi-byte sequences. 87 | try: 88 | s = codecs.decode(view.rcstring(0), "shift-jis") 89 | except: 90 | return False 91 | # String should be technically valid, but make sure 92 | # that string isn't empty or a likely false positive. 93 | return not (view.rcstring(0) in (b"", b"\x40", b"C0")) 94 | elif 0x20 <= b < 0x7f or b in (9, 10, 13): 95 | # Printable one-byte sequence. 96 | offset += 1 97 | elif 0x81 <= b < 0xa0 or 0xe0 <= b <= 0xea or 0xed <= b <= 0xef: 98 | if offset + 1 == size: 99 | return False 100 | # Valid multi-byte sequence. 101 | b2 = view.ru8(offset + 1) 102 | if not 0x40 <= b2 <= 0xfc: 103 | return False 104 | offset += 2 105 | else: 106 | return False 107 | return False 108 | 109 | def _SanitizeString(s): 110 | s = s.replace("\\", "\\\\") 111 | s = s.replace("\t", "\\t") 112 | s = s.replace("\n", "\\n") 113 | s = s.replace("\r", "\\r") 114 | return s 115 | 116 | bs = view.rbytes(size) 117 | # Check most restrictive types first: valid evts, common float constants, 118 | # Shift-JIS compatible strings of the exact length of the symbol. 119 | if exact and size & 3 == 0 and _IsEvtCompatible(view, size, exact=True): 120 | return ("evt", "") 121 | if size == 8 and view.ru64() == 0x4330000080000000: 122 | return ("double", "to-int") 123 | if size == 8 and view.ru64() == 0x4330000000000000: 124 | return ("double", "to-int-mask") 125 | if exact and _IsShiftJisCompatible(view, size, exact=True): 126 | s = codecs.decode(view.rcstring(), "shift-jis") 127 | return ("string", _SanitizeString(s)) 128 | # If all zero bytes, return "zero". 129 | if sum(bs) == 0: 130 | return ("zero", 0.0) 131 | # Check for reasonable-looking floating-point, pointer, or vec3 values. 132 | if size == 4 and _IsFloatCompatible(view): 133 | return ("float", view.rf32()) 134 | if size == 8 and _IsDoubleCompatible(view): 135 | return ("double", view.rf64()) 136 | if size == 4 and _IsPointerCompatible(view): 137 | return ("pointer", "%08x" % view.ru32()) 138 | if size == 12: 139 | if (_IsFloatCompatible(view, 0) and _IsFloatCompatible(view, 4) and 140 | _IsFloatCompatible(view, 8)): 141 | return ("vec3", "%f, %f, %f" % ( 142 | view.rf32(0), view.rf32(4), view.rf32(8))) 143 | # Look for arbitrary floating-point arrays or non-exact-length evts/strings; 144 | # these are more likely to be false positives. 145 | if size & 3 == 0: 146 | if _IsEvtCompatible(view, size, exact=False): 147 | return ("evt", "") 148 | # TODO: Improve heuristics for detecting float arrays vs. strings? 149 | is_valid = True 150 | for offset in range(0, size, 4): 151 | if not _IsFloatCompatible(view, offset): 152 | is_valid = False 153 | break 154 | if is_valid: 155 | return ("floatarr", "") 156 | is_valid = True 157 | for offset in range(0, size, 4): 158 | if not _IsPointerCompatible(view, offset): 159 | is_valid = False 160 | break 161 | if is_valid: 162 | return ("pointerarr", "") 163 | if _IsShiftJisCompatible(view, size, exact=False): 164 | s = codecs.decode(view.rcstring(), "shift-jis") 165 | return ("string", _SanitizeString(s)) 166 | # Not obviously compatible with any common types. 167 | return (None, None) 168 | 169 | def _AnnotateSymbols(symbols, section_info, out_path): 170 | def _AddSectionInfoFields(s, section_info): 171 | section = section_info.loc[(s["area"], s["sec_id"])] 172 | s["sec_name"] = section["name"] 173 | s["sec_type"] = section["type"] 174 | ram_addr = section["ram_start"] 175 | s["ram_addr"] = ( 176 | "%08x" % (int(ram_addr, 16) + int(s["sec_offset"], 16)) 177 | if isinstance(ram_addr, str) and ram_addr else np.nan 178 | ) 179 | file_addr = section["file_start"] 180 | s["file_addr"] = ( 181 | "%08x" % (int(file_addr, 16) + int(s["sec_offset"], 16)) 182 | if isinstance(file_addr, str) and file_addr else np.nan 183 | ) 184 | return s 185 | 186 | def _InferSymbolType(s, stores): 187 | # Not a data symbol. 188 | if s["sec_type"] != "data": 189 | return s 190 | # Symbol's section was not dumped, or out of range. 191 | section_lookup = "%s-%02d" % (s["area"], s["sec_id"]) 192 | if section_lookup not in stores: 193 | return s 194 | offset = int(s["sec_offset"], 16) 195 | if offset < 0: 196 | return s 197 | # Otherwise, infer the type and value of the symbol, if possible. 198 | view = stores[section_lookup].view(offset) 199 | (t, v) = _InferType(view, int(s["size"], 16), exact=True) 200 | if t: 201 | s["type"] = t 202 | s["value"] = v 203 | return s 204 | 205 | # Create a copy of the symbols DataFrame with the desired output columns. 206 | df = pd.DataFrame(symbols, columns=[ 207 | "area", "sec_id", "sec_offset", "sec_name", "sec_type", "ram_addr", 208 | "file_addr", "name", "namespace", "size", "align", "type", "value"]) 209 | 210 | # Load previously dumped .DOL / .REL file sections into BDStores. 211 | stores = {} 212 | for sec_id in (0, 1, 7, 8, 9, 10, 11, 12): 213 | section_path = "sections/_main/%02d.raw" % sec_id 214 | store = bd.BDStore(big_endian=True) 215 | store.RegisterFile(out_path / section_path, offset=0) 216 | stores["_main-%02d" % sec_id] = store 217 | 218 | rels_dir = out_path / "sections/rel_linked" 219 | areas = [f.name for f in os.scandir(rels_dir) if f.is_dir()] 220 | for area in areas: 221 | for sec_id in range(1,6): 222 | store = bd.BDStore(big_endian=True) 223 | store.RegisterFile(rels_dir / area / ("%02d.raw" % sec_id), offset=0) 224 | stores["%s-%02d" % (area, sec_id)] = store 225 | 226 | # Fill in remaining columns based on section_info and dumped sections. 227 | if FLAGS.GetFlag("debug_level"): 228 | print("Converting section offsets to ram/file addresses...") 229 | df = df.apply( 230 | lambda s: _AddSectionInfoFields(s, section_info), axis=1) 231 | 232 | if FLAGS.GetFlag("debug_level"): 233 | print("Inferring symbol types...") 234 | df = df.apply(lambda s: _InferSymbolType(s, stores), axis=1) 235 | 236 | # Output the final table of joined symbols. 237 | df.to_csv(out_path / "annotated_symbols.csv", index=False) 238 | 239 | def main(argc, argv): 240 | out_path = FLAGS.GetFlag("out_path") 241 | if not out_path or not os.path.exists(Path(out_path)): 242 | raise AnnotateMapSymbolsError( 243 | "--out_path must point to a valid directory.") 244 | out_path = Path(out_path) 245 | 246 | if not os.path.exists(out_path / "section_info.csv"): 247 | raise AnnotateMapSymbolsError( 248 | "You must first run dump_sections.py using the same --out_path.") 249 | section_info = pd.read_csv(out_path / "section_info.csv") 250 | section_info = section_info.set_index(["area", "id"]) 251 | 252 | if not os.path.exists(out_path / "map_symbols.csv"): 253 | raise AnnotateMapSymbolsError( 254 | "You must first run map_to_symbols.py using the same --out_path.") 255 | symbols = pd.read_csv(out_path / "map_symbols.csv") 256 | 257 | _AnnotateSymbols(symbols, section_info, out_path) 258 | 259 | if __name__ == "__main__": 260 | (argc, argv) = FLAGS.ParseFlags(sys.argv[1:]) 261 | main(argc, argv) 262 | -------------------------------------------------------------------------------- /source/combine_event_dumps.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.6 2 | 3 | """Combines all events generated by export_events.py. 4 | 5 | This file must be run after the following binaries, using the same --out_path: 6 | - export_events.py (and its dependencies) 7 | 8 | Concatenates the text files generated by export_events.py into a single file 9 | containing every unique event (as determined by name x area) once.""" 10 | # Jonathan Aldrich 2022-03-17 11 | 12 | import codecs 13 | import os 14 | import sys 15 | import subprocess 16 | from pathlib import Path 17 | 18 | import jdalibpy.flags as flags 19 | 20 | FLAGS = flags.Flags() 21 | 22 | # Output directory; should contain these outputs from previous binaries: 23 | # - event scripts from export_events (in "/events" dir underneath). 24 | # The combined events will be exported to "(out path)/events/combined/all.txt". 25 | FLAGS.DefineString("out_path", "") 26 | 27 | # Whether to display debug strings. 28 | FLAGS.DefineInt("debug_level", 1) 29 | 30 | class CombineEventsError(Exception): 31 | def __init__(self, message=""): 32 | self.message = message 33 | 34 | def _GetEventFilenames(events_path): 35 | if FLAGS.GetFlag("debug_level"): 36 | print("Getting filenames of event script dumps...") 37 | return [f for f in os.listdir(events_path) if f.endswith('.txt')] 38 | 39 | def _GetEventLookupDict(events_path, event_files): 40 | if FLAGS.GetFlag("debug_level"): 41 | print("Looking up event areas / addresses...") 42 | 43 | lookup = {} 44 | for fn in event_files: 45 | area = fn[:fn.find("_", 1)] 46 | address = None 47 | 48 | # Open file and inspect lines until finding one that ends in an address. 49 | f = codecs.open(events_path / fn, "r", encoding="utf-8") 50 | for line in f: 51 | pos = line.find("] AT ") 52 | if pos >= 0: 53 | address = int(line[pos + 5 :][:8], 16) 54 | break 55 | f.close() 56 | 57 | if address: 58 | lookup["%s_%08x" % (area, address)] = events_path / fn 59 | 60 | return lookup 61 | 62 | def _CombineEventFiles(event_lookup, out_path): 63 | if FLAGS.GetFlag("debug_level"): 64 | print("Combining event dumps into single text file...") 65 | 66 | outfile = codecs.open(out_path, "w", encoding="utf-8") 67 | outfile.write("Original disassembly by PistonMiner's ttydasm tool.\n\n") 68 | 69 | for k,fn in sorted(event_lookup.items()): 70 | outfile.write("Script %s%s:\n" % ("@", k)) 71 | 72 | evt = codecs.open(fn, "r", encoding="utf-8") 73 | found_start = False 74 | for line in evt: 75 | if found_start and not line.strip(): 76 | break 77 | if line.find("START OF DISASSEMBLY") >= 0: 78 | found_start = True 79 | if found_start: 80 | outfile.write(line) 81 | evt.close() 82 | outfile.write("\n") 83 | 84 | outfile.close() 85 | 86 | if FLAGS.GetFlag("debug_level"): 87 | print("Done! Output in %s." % out_path) 88 | 89 | def main(argc, argv): 90 | out_path = FLAGS.GetFlag("out_path") 91 | if not out_path or not os.path.exists(Path(out_path)): 92 | raise CombineEventsError("--out_path must point to a valid directory.") 93 | out_path = Path(out_path) 94 | 95 | if not os.path.exists(out_path / "events"): 96 | raise CombineEventsError( 97 | "You must first run export_events.py using the same --out_path.") 98 | 99 | events_path = out_path / "events" 100 | event_files = _GetEventFilenames(events_path) 101 | event_lookup = _GetEventLookupDict(events_path, event_files) 102 | 103 | if FLAGS.GetFlag("debug_level"): 104 | print("\nFound scripts:") 105 | for k,v in sorted(event_lookup.items())[:5]: 106 | print((k,v)) 107 | print("...") 108 | for k,v in sorted(event_lookup.items())[-5:]: 109 | print((k,v)) 110 | print("") 111 | 112 | if not os.path.exists(events_path / "combined"): 113 | os.makedirs(events_path / "combined") 114 | _CombineEventFiles(event_lookup, events_path / "combined/all.txt") 115 | 116 | if __name__ == "__main__": 117 | (argc, argv) = FLAGS.ParseFlags(sys.argv[1:]) 118 | main(argc, argv) 119 | -------------------------------------------------------------------------------- /source/combine_rels.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.6 2 | 3 | """Constructs a new REL file from a list of symbols or regions in other RELs. 4 | 5 | Takes the following inputs: 6 | - A filepattern matching .REL files with a single wildcard replacing the name 7 | - Either: 8 | - A list of data ranges to include in the custom REL, in the format: 9 | AREA:SEC_ID:START_OFFSET-END_OFFSET (e.g. aji:5:00001ab0-00001ae0) 10 | - Or: 11 | - A list of newline-delimited symbol names or patterns to include, in format: 12 | AREA:NAMESPACE:SYMBOL/* (e.g. aji:unit_gundan_zako.o:*) 13 | - A symbol info .csv (must include data on all names you wish to include) 14 | , then outputs the following: 15 | - A .REL file containing all of the symbols requested 16 | - A .csv file listing all the symbols' locations in the generated .REL file. 17 | 18 | Will throw an error if any requested data has a relocation table entry that 19 | requires a REL dependency that is not also included in the requested data. 20 | 21 | TODO: 22 | - Support alignment restrictions above 8 bytes. 23 | - Add support for automatically looking up dependencies if using symbol names? 24 | - Add support for subranges of particular symbols?""" 25 | # Jonathan Aldrich 2021-05-16 ~ 2021-05-17 26 | 27 | import math 28 | import os 29 | import sys 30 | import numpy as np 31 | import pandas as pd 32 | from pathlib import Path 33 | 34 | import jdalibpy.bindatastore as bd 35 | import jdalibpy.flags as flags 36 | 37 | FLAGS = flags.Flags() 38 | 39 | # Output directory. 40 | FLAGS.DefineString("out_path", "") 41 | 42 | # Input patterns for the source .REL files. 43 | FLAGS.DefineString("rel", "") 44 | # Filepath to a text file containing data ranges to include in the output REL. 45 | FLAGS.DefineString("symbol_ranges", "") 46 | # Filepath to a text file containing desired symbols in the output REL. 47 | FLAGS.DefineString("symbol_names", "") 48 | # Filepath to a .csv file containing info on symbols in the source .REL files; 49 | # must at least include info for all of the desired symbols. 50 | FLAGS.DefineString("symbol_info", "") 51 | 52 | # Whether to display debug strings. 53 | FLAGS.DefineInt("debug_level", 1) 54 | 55 | class CombineRelsError(Exception): 56 | def __init__(self, message=""): 57 | self.message = message 58 | 59 | def _GetOutputPath(filepath, create_parent=True): 60 | path = Path(FLAGS.GetFlag("out_path")) / filepath 61 | if create_parent and not os.path.exists(os.path.dirname(path)): 62 | os.makedirs(os.path.dirname(path)) 63 | return path 64 | 65 | def _CombineRels(rel_pattern_str, symbol_table): 66 | def _CreateExportedTableFormat(row): 67 | """Converts a symbol table row to exported format.""" 68 | return pd.DataFrame( 69 | [["custom", row["sec_id"], "%08x" % row["out_offset"], row["name"], 70 | row["namespace"], "%08x" % row["size"], row["align"]]], 71 | columns=[ 72 | "area", "sec_id", "sec_offset", 73 | "name", "namespace", "size", "align"]) 74 | 75 | def _CreateLookupRowWithOutput(row, out_offset): 76 | """Takes the input symbol_info row and adds the out_offset.""" 77 | return pd.DataFrame( 78 | [[row["area"], row["sec_id"], row["sec_offset"], 79 | row["sec_offset_end"], row["name"], row["namespace"], 80 | row["size"], row["align"], out_offset]], 81 | columns=[ 82 | "area", "sec_id", "sec_offset", "sec_offset_end", 83 | "name", "namespace", "size", "align", "out_offset"]) 84 | 85 | def _LookupNewOffset(symbol_table, area, sec_id, sec_offset): 86 | """Returns the new offset corresponding to the old one if one exists. 87 | 88 | If the old offset corresponds to a symbol not in symbol_table, returns 89 | None, assuming that symbol won't be included in the combined REL. 90 | Raises an error if there are multiple matches in symbol_table.""" 91 | matches = symbol_table[ 92 | (symbol_table.area == area) & 93 | (symbol_table.sec_id == sec_id) & 94 | (symbol_table.sec_offset <= sec_offset) & 95 | (symbol_table.sec_offset_end > sec_offset)] 96 | if matches.shape[0] < 1: 97 | return None 98 | if matches.shape[0] > 1: 99 | raise CombineRelsError( 100 | "Ambiguous symbol match at %s:%d:%08x." % 101 | (area, sec_id, sec_offset)) 102 | # Return the new offset + how far into the symbol the old offset is. 103 | return (matches["out_offset"].iloc[0] + 104 | sec_offset - matches["sec_offset"].iloc[0]) 105 | 106 | def _WriteToBuffer(buffer, offset, value, size): 107 | """Writes an integer value of the given size to a buffer.""" 108 | for x in range(size): 109 | buffer[offset + x] = ((value >> ((size-1-x)*8)) & 0xff) 110 | 111 | def _AppendToBuffer(buffer, value, size): 112 | """Writes an integer value of the given size to the end of a buffer.""" 113 | for x in range(size): 114 | buffer.append((value >> ((size-1-x)*8)) & 0xff) 115 | 116 | # Buffers for each section's raw, unlinked data (for .bss, track the length) 117 | section_data = [[] for x in range(6)] 118 | bss_length = 0 119 | 120 | # Copy all symbols into their respective sections' buffers, and create 121 | # a new lookup DataFrame including the output locations. 122 | # (For .bss data, keep track of the total length of the combined sections.) 123 | dfs = [] 124 | for area in sorted(symbol_table.area.unique()): 125 | if FLAGS.GetFlag("debug_level"): 126 | print("Processing %s symbols..." % area) 127 | 128 | store = bd.BDStore(big_endian=True) 129 | store.RegisterFile(rel_pattern_str.replace("*", area), offset=0) 130 | section_tbl = store.view(0)[0x10] 131 | 132 | for (index, row) in symbol_table.iterrows(): 133 | if row["area"] == area: 134 | # Add symbol's data to its respective section. 135 | if row["sec_id"] == 6: 136 | # Pad to alignment to find start point of next symbol. 137 | while bss_length % 8 != row["sec_offset"] % 8: 138 | bss_length += 1 139 | out_offset = bss_length 140 | # Add size of symbol to bss_length (bss data is not stored). 141 | bss_length += row["size"] 142 | else: 143 | id = row["sec_id"] 144 | if id > 1: 145 | # If not .text section (fixed-alignment of 4), pad to 146 | # alignment to find the start point of next symbol. 147 | while len(section_data[id]) % 8 != row["sec_offset"] % 8: 148 | section_data[id].append(0) 149 | out_offset = len(section_data[id]) 150 | # Copy bytes from original symbol into section_data. 151 | file_offset = section_tbl.ru32(8 * id) & ~3 152 | for b in store.view(0).rbytes( 153 | row["size"], file_offset + row["sec_offset"]): 154 | section_data[id].append(b) 155 | # Add the new location of the symbol to a new table. 156 | dfs.append(_CreateLookupRowWithOutput(row, out_offset)) 157 | 158 | # Join all lookup rows with output offsets, and order lexicographically. 159 | df = pd.concat(dfs, ignore_index=True) 160 | symbol_table = df.sort_values(by=["area", "sec_id", "sec_offset"]) 161 | 162 | if FLAGS.GetFlag("debug_level"): 163 | print("REL symbols extracted; %d symbols processed." % 164 | symbol_table.shape[0]) 165 | 166 | # Buffers for each section's relocatable data (against the same REL). 167 | rel_data = [[] for x in range(6)] 168 | rel_offsets = [0 for x in range(6)] 169 | # Buffers for each section's relocatable data (against the main DOL). 170 | rel_data_main = [[] for x in range(6)] 171 | rel_offsets_main = [0 for x in range(6)] 172 | 173 | # Port all needed relocation information from the original .REL files. 174 | for area in sorted(symbol_table.area.unique()): 175 | if FLAGS.GetFlag("debug_level"): 176 | print("Processing %s relocation tables..." % area) 177 | 178 | store = bd.BDStore(big_endian=True) 179 | store.RegisterFile(rel_pattern_str.replace("*", area), offset=0) 180 | 181 | header = store.view(0) 182 | imp_table = header[0x28] 183 | imp_size = header.ru32(0x2c) 184 | imp_offset = 0 185 | while imp_offset < imp_size: 186 | module_id = imp_table.ru32(imp_offset) 187 | rel_table = imp_table[imp_offset + 4] 188 | imp_offset += 8 189 | 190 | current_section = 0 191 | section_offset = 0 192 | while True: 193 | offset = rel_table.ru16(0) 194 | type = rel_table.ru8(2) 195 | section = rel_table.ru8(3) 196 | addend = rel_table.ru32(4) 197 | # Advance to next relocation entry. 198 | rel_table = rel_table.at(8) 199 | 200 | if type == 203: 201 | # End of rel table. 202 | break 203 | elif type == 202: 204 | # Section change rel entry. 205 | current_section = section 206 | section_offset = 0 207 | else: 208 | # Assuming no symbols in .bss have linkage; need to add 209 | # specific support for that if I'm mistaken. 210 | if current_section > 5: 211 | raise CombineRelsError("Linking to section 6!") 212 | 213 | section_offset += offset 214 | new_offset = _LookupNewOffset( 215 | symbol_table, area, current_section, section_offset) 216 | # Not used in combined REL; move to next entry. 217 | if new_offset == None: 218 | continue 219 | # Look up the linked-to symbol address in the combined REL. 220 | # (if module_id == 0, i.e. linking to DOL, just use addend.) 221 | if module_id != 0: 222 | addend = _LookupNewOffset( 223 | symbol_table, area, section, addend) 224 | # If an address could not be found, a dependency must have 225 | # been missing from the input symbol_table; for shame. 226 | if addend == None: 227 | raise CombineRelsError( 228 | "Symbol missing dependency at %s:%d:%08x." % ( 229 | area, current_section, section_offset)) 230 | # Update the current offset for the given section/imp. 231 | if module_id == 0: 232 | offset = new_offset - rel_offsets_main[current_section] 233 | rel_offsets_main[current_section] = new_offset 234 | else: 235 | offset = new_offset - rel_offsets[current_section] 236 | rel_offsets[current_section] = new_offset 237 | # Add this relocation to the respective table. 238 | rel_table_buffer = (rel_data_main[current_section] if 239 | module_id == 0 else rel_data[current_section]) 240 | _AppendToBuffer(rel_table_buffer, offset, 2) 241 | _AppendToBuffer(rel_table_buffer, type, 1) 242 | _AppendToBuffer(rel_table_buffer, section, 1) 243 | _AppendToBuffer(rel_table_buffer, addend, 4) 244 | 245 | if FLAGS.GetFlag("debug_level"): 246 | print("Relocation tables processed.") 247 | 248 | # Construct the final REL from the buffers put together beforehand. 249 | rel = [] 250 | 251 | # REL header. 252 | _AppendToBuffer(rel, 40, 4) # id (arbitrary custom value) 253 | _AppendToBuffer(rel, 0, 4) # next 254 | _AppendToBuffer(rel, 0, 4) # prev 255 | _AppendToBuffer(rel, 15, 4) # numSections 256 | _AppendToBuffer(rel, 0x4c, 4) # sectionInfoOffset 257 | _AppendToBuffer(rel, 0, 4) # nameOffset 258 | _AppendToBuffer(rel, 0, 4) # nameSize 259 | _AppendToBuffer(rel, 3, 4) # version 260 | _AppendToBuffer(rel, bss_length, 4) # bssSize 261 | _AppendToBuffer(rel, 0, 4) # relOffset - will be filled in later. 262 | _AppendToBuffer(rel, 0, 4) # impOffset - will be filled in later. 263 | _AppendToBuffer(rel, 0x10, 4) # impSize 264 | _AppendToBuffer(rel, 0, 4) # prolog/epilog/unresolved/bssSection 265 | _AppendToBuffer(rel, 0, 4) # prolog offset 266 | _AppendToBuffer(rel, 0, 4) # epilog offset 267 | _AppendToBuffer(rel, 0, 4) # unresolved offset 268 | _AppendToBuffer(rel, 8, 4) # align 269 | _AppendToBuffer(rel, 8, 4) # bssAlign 270 | _AppendToBuffer(rel, 0, 4) # fixSize - will be filled in later. 271 | 272 | # Section table (initialize to 15 section table entries' worth of zeroes). 273 | for _ in range(15 * 8): 274 | rel.append(0) 275 | # Section 1 (text) 276 | section_start = len(rel) 277 | _WriteToBuffer(rel, 0x54, section_start | 1, 4) 278 | rel += section_data[1] 279 | _WriteToBuffer(rel, 0x58, len(rel) - section_start, 4) 280 | # Sections 2 and 3 (unused) 281 | _WriteToBuffer(rel, 0x5c, len(rel), 4) 282 | _WriteToBuffer(rel, 0x60, 4, 4) 283 | _AppendToBuffer(rel, 0, 4) 284 | _WriteToBuffer(rel, 0x64, len(rel), 4) 285 | _WriteToBuffer(rel, 0x68, 4, 4) 286 | _AppendToBuffer(rel, 0, 4) 287 | # Section 4 (rodata) 288 | while len(rel) % 8 != 0: 289 | rel.append(0) 290 | section_start = len(rel) 291 | _WriteToBuffer(rel, 0x6c, section_start, 4) 292 | rel += section_data[4] 293 | _WriteToBuffer(rel, 0x70, len(rel) - section_start, 4) 294 | # Section 5 (data) 295 | while len(rel) % 8 != 0: 296 | rel.append(0) 297 | section_start = len(rel) 298 | _WriteToBuffer(rel, 0x74, section_start, 4) 299 | rel += section_data[5] 300 | _WriteToBuffer(rel, 0x78, len(rel) - section_start, 4) 301 | # Section 6 (bss) 302 | _WriteToBuffer(rel, 0x7c, 0, 4) 303 | _WriteToBuffer(rel, 0x80, bss_length, 4) 304 | # Pad before imp table (not necessary, but easier to read in a hex editor.) 305 | while len(rel) % 8 != 0: 306 | rel.append(0) 307 | 308 | imp_table = len(rel) 309 | rel_table = len(rel) + 0x10 310 | _WriteToBuffer(rel, 0x24, rel_table, 4) # relOffset 311 | _WriteToBuffer(rel, 0x28, imp_table, 4) # impOffset 312 | _WriteToBuffer(rel, 0x48, rel_table, 4) # fixSize 313 | # Reserve space for imp table. 314 | for _ in range(16): 315 | rel.append(0) 316 | # Copy REL -> REL relocation data. 317 | _WriteToBuffer(rel, imp_table, 40, 4) 318 | _WriteToBuffer(rel, imp_table + 4, len(rel), 4) 319 | for x in range(6): 320 | if len(rel_data[x]): 321 | _AppendToBuffer(rel, 0, 2) # offset 322 | _AppendToBuffer(rel, 202, 1) # type (change section) 323 | _AppendToBuffer(rel, x, 1) # section 324 | _AppendToBuffer(rel, 0, 4) # addend 325 | rel += rel_data[x] 326 | _AppendToBuffer(rel, 0, 2) # offset 327 | _AppendToBuffer(rel, 203, 1) # type (end of table) 328 | _AppendToBuffer(rel, 0, 1) # section 329 | _AppendToBuffer(rel, 0, 4) # addend 330 | # Copy REL -> DOL relocation data. 331 | _WriteToBuffer(rel, imp_table + 8, 0, 4) 332 | _WriteToBuffer(rel, imp_table + 12, len(rel), 4) 333 | for x in range(6): 334 | if len(rel_data_main[x]): 335 | _AppendToBuffer(rel, 0, 2) # offset 336 | _AppendToBuffer(rel, 202, 1) # type (change section) 337 | _AppendToBuffer(rel, x, 1) # section 338 | _AppendToBuffer(rel, 0, 4) # addend 339 | rel += rel_data_main[x] 340 | _AppendToBuffer(rel, 0, 2) # offset 341 | _AppendToBuffer(rel, 203, 1) # type (end of table) 342 | _AppendToBuffer(rel, 0, 1) # section 343 | _AppendToBuffer(rel, 0, 4) # addend 344 | 345 | # Export the final REL. 346 | out_rel = open(_GetOutputPath("custom.rel"), "wb") 347 | out_rel.write(bytes(rel)) 348 | 349 | # Export the table of symbol info. 350 | # TODO: Add column with file-relative offsets? 351 | dfs = [] 352 | for (index, row) in symbol_table.iterrows(): 353 | dfs.append(_CreateExportedTableFormat(row)) 354 | df = pd.concat(dfs, ignore_index=True) 355 | df = df.sort_values(by=["area", "sec_id", "sec_offset"]) 356 | df.to_csv(_GetOutputPath("custom_symbols.csv"), index=False) 357 | 358 | def _CreateCombinedRelSymbolTable(symbol_info, symbol_names): 359 | """Creates a lookup table of `symbol_info` data matching `symbol_names`.""" 360 | def _CreateLookupRow(row): 361 | """Converts a symbol_info row into a lookup-friendly format.""" 362 | sec_offset = int(row["sec_offset"], 16) 363 | size = int(row["size"], 16) 364 | return pd.DataFrame( 365 | [[row["area"], row["sec_id"], sec_offset, sec_offset + size, 366 | row["name"], row["namespace"], size, row["align"]]], 367 | columns=[ 368 | "area", "sec_id", "sec_offset", "sec_offset_end", 369 | "name", "namespace", "size", "align"]) 370 | 371 | dfs = [] 372 | # TODO: Speed up / validate by looking up rows via symbol_names instead? 373 | for (index, row) in symbol_info.iterrows(): 374 | full_symbol = "%s:%s:%s" % (row["area"], row["namespace"], row["name"]) 375 | wildcard_symbol = "%s:%s:*" % (row["area"], row["namespace"]) 376 | if (full_symbol in symbol_names) or (wildcard_symbol in symbol_names): 377 | dfs.append(_CreateLookupRow(row)) 378 | if not len(dfs): 379 | raise CombineRelsError("No symbols found matching --symbol_names.") 380 | 381 | df = pd.concat(dfs, ignore_index=True) 382 | df = df.sort_values(by=["area", "sec_id", "sec_offset"]) 383 | 384 | if FLAGS.GetFlag("debug_level"): 385 | print("Symbol lookup table finished; %d symbols found." % df.shape[0]) 386 | return df 387 | 388 | def _CreateCombinedRelRangeLookupTable(symbol_ranges): 389 | """Creates a lookup table in symbol-info format from a series of range 390 | strings, assuming a standard alignment and using dummy names/namespaces. 391 | 392 | Does not validate that section names, ids, or ranges are valid; 393 | use the symbol name variant if you care about pre-checking that.""" 394 | def _CreateLookupRow(area, sec_id, start_offset, end_offset): 395 | """Converts a symbol_info row into a lookup-friendly format.""" 396 | sec_id = int(sec_id) 397 | sec_offset = int(start_offset, 16) 398 | sec_offset_end = int(end_offset, 16) 399 | size = sec_offset_end - sec_offset 400 | align = 8 if sec_id > 3 else 4 401 | return pd.DataFrame( 402 | [[area, sec_id, sec_offset, sec_offset_end, 403 | "%s-%08x-%08x" % (area, sec_offset, sec_offset_end), 404 | "", size, align]], 405 | columns=[ 406 | "area", "sec_id", "sec_offset", "sec_offset_end", 407 | "name", "namespace", "size", "align"]) 408 | 409 | dfs = [] 410 | for line in symbol_ranges: 411 | tokens = line.split(":") 412 | if len(tokens) != 3: 413 | raise CombineRelsError( 414 | "Ranges must be specified in format 'AREA:SEC_ID:START-END'.") 415 | range_tokens = tokens[2].split("-") 416 | if len(range_tokens) != 2: 417 | raise CombineRelsError( 418 | "Ranges must be specified in format 'AREA:SEC_ID:START-END'.") 419 | dfs.append(_CreateLookupRow( 420 | tokens[0], tokens[1], range_tokens[0], range_tokens[1])) 421 | 422 | df = pd.concat(dfs, ignore_index=True) 423 | df = df.sort_values(by=["area", "sec_id", "sec_offset"]) 424 | 425 | if FLAGS.GetFlag("debug_level"): 426 | print("Lookup table finished; %d ranges found." % df.shape[0]) 427 | return df 428 | 429 | def _RemoveComments(raw_input): 430 | """Removes '#' comments, blank lines and trailing whitespace from input.""" 431 | filtered_inputs = [] 432 | for line in raw_input: 433 | if line.find('#') != -1: 434 | line = line[:line.find('#')] 435 | line = line.rstrip() 436 | if line: 437 | filtered_inputs.append(line) 438 | return filtered_inputs 439 | 440 | def main(argc, argv): 441 | if not FLAGS.GetFlag("out_path"): 442 | raise CombineRelsError("Must provide a directory for --out_path.") 443 | elif not os.path.exists(Path(FLAGS.GetFlag("out_path"))): 444 | os.makedirs(Path(FLAGS.GetFlag("out_path"))) 445 | 446 | # Validate rel filepattern, verifying exactly one asterisk wildcard exists. 447 | rel_pattern = FLAGS.GetFlag("rel") 448 | normalized_pattern = str(Path(rel_pattern)) 449 | lpos = normalized_pattern.find("*") 450 | rpos = normalized_pattern.rfind("*") 451 | if lpos != rpos or lpos == -1: 452 | raise CombineRelsError( 453 | "--rel pattern must contain exactly one wildcard asterisk.") 454 | 455 | # Attempt to read range data, and symbol name data / info. 456 | symbol_ranges_path = FLAGS.GetFlag("symbol_ranges") 457 | symbol_names_path = FLAGS.GetFlag("symbol_names") 458 | symbol_info_path = FLAGS.GetFlag("symbol_info") 459 | 460 | can_use_ranges = os.path.exists(symbol_ranges_path) 461 | can_use_names = ( 462 | os.path.exists(symbol_names_path) and os.path.exists(symbol_info_path)) 463 | 464 | # Determine which symbols or data ranges to include in the combined REL. 465 | # Prioritize symbol names over ranges, if both are specified. 466 | symbol_table = None 467 | if can_use_names: 468 | symbol_info = pd.read_csv(Path(symbol_info_path)) 469 | symbol_names = [line.rstrip() for line in open(Path(symbol_names_path))] 470 | symbol_names = _RemoveComments(symbol_names) 471 | if len(symbol_names) < 1: 472 | raise CombineRelsError("--symbol_names has no requested symbols.") 473 | 474 | symbol_table = _CreateCombinedRelSymbolTable(symbol_info, symbol_names) 475 | elif can_use_ranges: 476 | symbol_ranges = [line.rstrip() for line in open(Path(symbol_ranges_path))] 477 | symbol_ranges = _RemoveComments(symbol_ranges) 478 | if len(symbol_ranges) < 1: 479 | raise CombineRelsError("--symbol_ranges has no requested ranges.") 480 | 481 | symbol_table = _CreateCombinedRelRangeLookupTable(symbol_ranges) 482 | else: 483 | raise CombineRelsError( 484 | "Must specify either --symbol_ranges, or " 485 | "--symbol_names and --symbol_info.") 486 | 487 | # Attempt to construct the combined REL from the requested data. 488 | _CombineRels(normalized_pattern, symbol_table) 489 | 490 | if __name__ == "__main__": 491 | (argc, argv) = FLAGS.ParseFlags(sys.argv[1:]) 492 | main(argc, argv) 493 | -------------------------------------------------------------------------------- /source/dump_sections.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.6 2 | 3 | """Dumps information regarding individual sections of TTYD DOL and REL files. 4 | 5 | Takes input DOL and REL files, and dumps the following information: 6 | - Each section, unlinked and linked for RELs 7 | - Each entire file, unlinked and linked for RELs 8 | - A .csv containing information on section ids, types, addresses and lengths.""" 9 | # Jonathan Aldrich 2021-01-18 ~ 2021-03-02 10 | 11 | import glob 12 | import math 13 | import os 14 | import sys 15 | import numpy as np 16 | import pandas as pd 17 | from pathlib import Path 18 | 19 | import jdalibpy.bindatastore as bd 20 | import jdalibpy.flags as flags 21 | 22 | FLAGS = flags.Flags() 23 | 24 | # Output directory. 25 | FLAGS.DefineString("out_path", "") 26 | 27 | # Input patterns for .DOL and .REL files for a single version of TTYD. 28 | # REL pattern must contain a single asterisk that matches the REL's area name. 29 | FLAGS.DefineString("dol", "") 30 | FLAGS.DefineString("rel", "") 31 | 32 | # Base address to use for linked RELs; must be in range [0x80000000, 0x81000000) 33 | # or 0. If set to 0, will not output linked versions of the REL files/sections. 34 | FLAGS.DefineInt("link_address", 0x80600000) 35 | # Base addresses to use for specific RELs, specified as comma-delimited 36 | # rel_name:HEX_ADDRESS pairs; e.g. "jon:80c779a0,gor:805ba9a0". 37 | # Falls back to the value in --link_address if unspecified. 38 | FLAGS.DefineString("link_address_overrides", "") 39 | # Base address to use for REL bss sections; must be set in range 40 | # [0x80000000, 0x81000000) if --link_address is non-zero. 41 | FLAGS.DefineInt("rel_bss_address", 0x80a00000) 42 | 43 | # Whether to display debug strings when each file starts being processed. 44 | FLAGS.DefineInt("debug_level", 1) 45 | 46 | class DumpSectionsError(Exception): 47 | def __init__(self, message=""): 48 | self.message = message 49 | 50 | def _GetOutputPath(filepath, create_parent=True): 51 | path = Path(FLAGS.GetFlag("out_path")) / filepath 52 | if create_parent and not os.path.exists(os.path.dirname(path)): 53 | os.makedirs(os.path.dirname(path)) 54 | return path 55 | 56 | def _LookupSymbolAddress(store, link_address, module_id, section_id, addend): 57 | if module_id > 0: 58 | section_table = store.view(0)[0x10] 59 | section_addr = section_table.ru32(section_id * 8) & ~3 60 | if section_addr: 61 | return link_address + section_addr + addend 62 | else: 63 | return FLAGS.GetFlag("rel_bss_address") + addend 64 | else: 65 | return addend 66 | 67 | def _LinkRelImpEntry(store, link_address, rel_table, module_id): 68 | section_view = None 69 | current_section = 0 70 | section_offset = 0 71 | while True: 72 | offset = rel_table.ru16(0) 73 | type = rel_table.ru8(2) 74 | section = rel_table.ru8(3) 75 | addend = rel_table.ru32(4) 76 | if type == 203: 77 | break 78 | elif type == 202: 79 | current_section = section 80 | section_table = store.view(0)[0x10] 81 | section_addr = section_table.ru32(section * 8) & ~3 82 | section_view = store.view(section_addr) 83 | section_offset = 0 84 | else: 85 | section_offset += offset 86 | symbol_addr = _LookupSymbolAddress( 87 | store, link_address, module_id, section, addend) 88 | relocation_addr = link_address + section_addr + section_offset 89 | 90 | if type == 1: 91 | # Write the symbol's 32-bit address. 92 | section_view.w32(symbol_addr, section_offset) 93 | elif type == 2: 94 | # Write the symbol's 24-bit address / 4 shifted up two bits. 95 | mask = 0xFFFFFC 96 | value = section_view.ru32(section_offset) & ~mask 97 | value |= (symbol_addr & mask) 98 | section_view.w32(value, section_offset) 99 | elif type == 3: 100 | # Write the symbol's 16-bit address. 101 | section_view.w16(symbol_addr, section_offset) 102 | elif type == 4: 103 | # Write the low 16 bits of the symbol address. 104 | section_view.w16(symbol_addr, section_offset) 105 | elif type == 5: 106 | # Write the high 16 bits of the symbol address. 107 | section_view.w16(symbol_addr >> 16, section_offset) 108 | elif type == 6: 109 | # Write the high 16 bits of the symbol address + 0x8000. 110 | section_view.w16((symbol_addr + 0x8000) >> 16, section_offset) 111 | elif 7 <= type <= 9: 112 | # Write the symbol's 14-bit address / 4 shifted up two bits. 113 | mask = 0x3FFC 114 | value = section_view.ru32(section_offset) & ~mask 115 | value |= (symbol_addr & mask) 116 | section_view.w32(value, section_offset) 117 | elif type == 10: 118 | # Write the 24-bit address minus the relocation address, 119 | # divided by four and shifted up two bits. 120 | mask = 0xFFFFFC 121 | value = section_view.ru32(section_offset) & ~mask 122 | value |= ((symbol_addr - relocation_addr) & mask) 123 | section_view.w32(value, section_offset) 124 | elif 11 <= type <= 13: 125 | # Write the 14-bit address minus the relocation address, 126 | # divided by four and shifted up two bits. 127 | mask = 0x3FFC 128 | value = section_view.ru32(section_offset) & ~mask 129 | value |= ((symbol_addr - relocation_addr) & mask) 130 | section_view.w32(value, section_offset) 131 | # Advance to next relocation entry. 132 | rel_table = rel_table.at(8) 133 | 134 | def _LinkRel(store, link_address): 135 | header = store.view(0) 136 | imp_table = header[0x28] 137 | imp_size = header.ru32(0x2c) 138 | imp_offset = 0 139 | while imp_offset < imp_size: 140 | module_id = imp_table.ru32(imp_offset) 141 | rel_table = imp_table[imp_offset + 4] 142 | _LinkRelImpEntry(store, link_address, rel_table, module_id) 143 | imp_offset += 8 144 | 145 | def _ProcessRel(area, filepath, link_address): 146 | def _CreateSectionDf(id, name, area, link_address, columns, section_tbl): 147 | file_start = section_tbl.ru32(8 * id) & ~3 148 | size = section_tbl.ru32(8 * id + 4) 149 | type = "data" if id > 3 else "text" 150 | if file_start == 0: 151 | type = "bss" 152 | file_start = np.nan 153 | file_end = np.nan 154 | ram_start = np.nan 155 | ram_end = np.nan 156 | else: 157 | ram_start = file_start + link_address 158 | file_end = file_start + size 159 | ram_end = ram_start + size 160 | if not link_address: 161 | ram_start = np.nan 162 | ram_end = np.nan 163 | return pd.DataFrame( 164 | [[area, id, name, type, file_start, file_end, 165 | ram_start, ram_end, size]], 166 | columns=columns) 167 | 168 | def _OutputSections(area, store, linked_folder_name): 169 | f = open(_GetOutputPath("%s/%s.rel" % (linked_folder_name, area)), "wb") 170 | f.write(store.mem[0].data) 171 | f.close() 172 | 173 | section_tbl = store.view(0)[0x10] 174 | for id in range(1, 7): 175 | file_offset = section_tbl.ru32(8 * id) & ~3 176 | size = section_tbl.ru32(8 * id + 4) 177 | if size and id < 6: 178 | f = open(_GetOutputPath("sections/%s/%s/%02d.raw" 179 | % (linked_folder_name, area, id)), "wb") 180 | f.write(store.view(0).rbytes(size, file_offset)) 181 | f.close() 182 | 183 | if FLAGS.GetFlag("debug_level"): 184 | print("Processing %s REL at %s..." % (area, filepath)) 185 | 186 | store = bd.BDStore(big_endian=True) 187 | store.RegisterFile(filepath, offset=0) 188 | section_tbl = store.view(0)[0x10] 189 | 190 | # Output REL and its sections, unlinked and linked. 191 | _OutputSections(area, store, linked_folder_name="rel_unlinked") 192 | if link_address: 193 | _LinkRel(store, link_address) 194 | _OutputSections(area, store, linked_folder_name="rel_linked") 195 | 196 | # Construct DataFrame of REL section info. 197 | columns = [ 198 | "area", "id", "name", "type", 199 | "file_start", "file_end", "ram_start", "ram_end", "size"] 200 | dfs = [] 201 | for (id, name) in { 202 | 1: ".text", 2: ".ctors", 3: ".dtors", 203 | 4: ".rodata", 5: ".data", 6: ".bss" 204 | }.items(): 205 | dfs.append(_CreateSectionDf( 206 | id, name, area, link_address, columns, section_tbl)) 207 | df = pd.concat(dfs, ignore_index=True) 208 | df = df.set_index(["area", "id", "name", "type"]) 209 | 210 | return df 211 | 212 | def _ProcessDol(filepath): 213 | def _CreateSectionDf(id, name, section_info, columns): 214 | file_start = section_info[id][0] 215 | ram_start = section_info[id][1] 216 | size = section_info[id][2] 217 | return pd.DataFrame( 218 | [["_main", id, name, "text" if id < 7 else "data", file_start, 219 | file_start + size, ram_start, ram_start + size, size]], 220 | columns=columns) 221 | 222 | def _CreateBssDf(id, name, section_info, columns, bss_end=None): 223 | ram_start = section_info[id][1] + section_info[id][2] 224 | ram_end = bss_end if bss_end else section_info[id + 1][1] 225 | size = ram_end - ram_start 226 | return pd.DataFrame( 227 | [["_main", id + 90, name, "bss", np.nan, np.nan, 228 | ram_start, ram_end, size]], 229 | columns=columns) 230 | 231 | if FLAGS.GetFlag("debug_level"): 232 | print("Processing _main DOL at %s..." % str(filepath)) 233 | 234 | store = bd.BDStore(big_endian=True) 235 | store.RegisterFile(filepath, offset=0) 236 | view = store.view(0) 237 | 238 | # Put together list of (file_start, ram_start, size) tuples. 239 | sections = [None for x in range(18)] 240 | for x in range(18): 241 | sections[x] = (view.ru32(0), view.ru32(0x48), view.ru32(0x90)) 242 | view = view.at(4) 243 | # Get start / end / size of .bss range. 244 | view = store.view(0) 245 | bss_start = view.ru32(0xd8) 246 | bss_size = view.ru32(0xdc) 247 | bss_end = bss_start + bss_size 248 | 249 | # Output DOL in its entirety, and its individual sections. 250 | f = open(_GetOutputPath("_main.dol"), "wb") 251 | f.write(store.mem[0].data) 252 | f.close() 253 | for id in range(18): 254 | if sections[id][2]: # size > 0 255 | f = open(_GetOutputPath("sections/_main/%02d.raw" % id), "wb") 256 | f.write(view.rbytes(sections[id][2], sections[id][0])) 257 | f.close() 258 | 259 | # Construct DataFrame of DOL section info. 260 | columns = [ 261 | "area", "id", "name", "type", 262 | "file_start", "file_end", "ram_start", "ram_end", "size"] 263 | dfs = [] 264 | dfs.append(_CreateSectionDf(0, ".init", sections, columns)) 265 | dfs.append(_CreateSectionDf(1, ".text", sections, columns)) 266 | dfs.append(_CreateSectionDf(7, ".ctors", sections, columns)) 267 | dfs.append(_CreateSectionDf(8, ".dtors", sections, columns)) 268 | dfs.append(_CreateSectionDf(9, ".rodata", sections, columns)) 269 | dfs.append(_CreateSectionDf(10, ".data", sections, columns)) 270 | dfs.append(_CreateBssDf(10, ".bss", sections, columns)) 271 | dfs.append(_CreateSectionDf(11, ".sdata", sections, columns)) 272 | dfs.append(_CreateBssDf(11, ".sbss", sections, columns)) 273 | dfs.append(_CreateSectionDf(12, ".sdata2", sections, columns)) 274 | dfs.append(_CreateBssDf(12, ".sbss2", sections, columns, bss_end=bss_end)) 275 | df = pd.concat(dfs, ignore_index=True) 276 | df = df.set_index(["area", "id", "name", "type"]) 277 | return df 278 | 279 | def main(argc, argv): 280 | if not FLAGS.GetFlag("out_path"): 281 | raise DumpSectionsError("Must provide a directory for --out_path.") 282 | elif not os.path.exists(Path(FLAGS.GetFlag("out_path"))): 283 | os.makedirs(Path(FLAGS.GetFlag("out_path"))) 284 | 285 | link_address_overrides = {} 286 | for kv in filter(None, FLAGS.GetFlag("link_address_overrides").split(",")): 287 | (key, value) = kv.split(":") 288 | link_address_overrides[key] = int(value, 16) 289 | 290 | # Create list of section_info dataframes to be later merged. 291 | section_info = [] 292 | 293 | # Process the DOL, outputting it and its individual sections. 294 | dol_path = Path(FLAGS.GetFlag("dol")) 295 | if not dol_path.exists(): 296 | raise DumpSectionsError("--dol must point to a valid .DOL file.") 297 | section_info.append(_ProcessDol(dol_path)) 298 | 299 | # Process RELs, outputting them and their sections (linked and unlinked). 300 | rel_pattern = FLAGS.GetFlag("rel") 301 | normalized_pattern = str(Path(rel_pattern)) 302 | # Verify exactly one asterisk wildcard exists. 303 | lpos = normalized_pattern.find("*") 304 | rpos = normalized_pattern.rfind("*") 305 | if lpos != rpos or lpos == -1: 306 | raise DumpSectionsError( 307 | "--rel pattern must contain exactly one wildcard asterisk.") 308 | # Verify any files are matched. 309 | if not glob.glob(rel_pattern): 310 | raise DumpSectionsError("--rel pattern matched no files.") 311 | # For each file, get the full path and the part of the string 312 | # that replaced the asterisk (which should be the area's name). 313 | for fn in sorted(glob.glob(rel_pattern)): 314 | filepath = str(fn) 315 | area = filepath[lpos:rpos+1-len(normalized_pattern)] 316 | 317 | link_address = FLAGS.GetFlag("link_address") 318 | if link_address and area in link_address_overrides: 319 | link_address = link_address_overrides[area] 320 | if link_address: 321 | if not (0x80000000 <= link_address < 0x81000000): 322 | raise DumpSectionsError( 323 | "Link address must be 0 or in range [0x8000,0x8100)0000.") 324 | if not (0x80000000 <= FLAGS.GetFlag("rel_bss_address") < 325 | 0x81000000): 326 | raise DumpSectionsError( 327 | "REL bss address must be in range [0x8000,0x8100)0000.") 328 | section_info.append(_ProcessRel(area, filepath, link_address)) 329 | 330 | # Finalize section_info.csv. 331 | # Concatenate section_info tables from DOL and RELs. 332 | df = pd.concat(section_info) 333 | # Convert values to eight-digit hex strings, with empty strings for NaNs. 334 | df = df.applymap(lambda val: "" if math.isnan(val) else "%08x" % int(val)) 335 | # Export to csv. 336 | df.to_csv(_GetOutputPath("section_info.csv")) 337 | 338 | if __name__ == "__main__": 339 | (argc, argv) = FLAGS.ParseFlags(sys.argv[1:]) 340 | main(argc, argv) 341 | -------------------------------------------------------------------------------- /source/export_classes.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.6 2 | 3 | """Dumps fields / byte representation of objects of given types from TTYD. 4 | 5 | This file must be run after dump_sections.py using the same --out_path. 6 | 7 | Given a symbols .csv annotated with type information, will export the fields 8 | and raw byte representations of all instances of the class types defined in 9 | export_classes_parsers.py.""" 10 | # Jonathan Aldrich 2021-03-03 ~ 2021-03-04 11 | 12 | import codecs 13 | import os 14 | import sys 15 | import numpy as np 16 | import pandas as pd 17 | from collections import defaultdict 18 | from pathlib import Path 19 | 20 | from export_classes_parsers import * 21 | import jdalibpy.bindatastore as bd 22 | import jdalibpy.flags as flags 23 | 24 | FLAGS = flags.Flags() 25 | 26 | # Output directory; should contain the outputs from dump_sections.py beforehand. 27 | # Dumped object info will go into "/classes" and "/classes_raw" dirs underneath. 28 | FLAGS.DefineString("out_path", "") 29 | # Input symbols file. 30 | FLAGS.DefineString("symbols_path", "") 31 | 32 | # Whether to display debug strings. 33 | FLAGS.DefineInt("debug_level", 1) 34 | 35 | class ExportClassesError(Exception): 36 | def __init__(self, message=""): 37 | self.message = message 38 | 39 | def _GetOutputPath(path, create_parent=True): 40 | if create_parent and not os.path.exists(os.path.dirname(path)): 41 | os.makedirs(os.path.dirname(path)) 42 | return path 43 | 44 | def _LoadSectionRamAddrDict(in_path, section_info): 45 | """Constructs a dict of REL/section : ram base address.""" 46 | if FLAGS.GetFlag("debug_level"): 47 | print("Loading section ram addresses...") 48 | 49 | section_info = section_info.set_index(["area", "id"]) 50 | 51 | section_data = {} 52 | for sec_id in range(8, 13): 53 | ram_addr = int(section_info.loc[("_main", sec_id)]["ram_start"], 16) 54 | section_data["_main-%02d" % sec_id] = ram_addr 55 | 56 | rels_dir = in_path / "sections/rel_linked" 57 | areas = [f.name for f in os.scandir(rels_dir) if f.is_dir()] 58 | for area in areas: 59 | for sec_id in range(4, 6): 60 | ram_addr = int(section_info.loc[(area, sec_id)]["ram_start"], 16) 61 | section_data["%s-%02d" % (area, sec_id)] = ram_addr 62 | return section_data 63 | 64 | def _LoadSectionDataDict(out_path, section_info): 65 | """Constructs a dict of area name : BDStore of that area's data sections.""" 66 | if FLAGS.GetFlag("debug_level"): 67 | print("Loading section data...") 68 | 69 | res = defaultdict(lambda: bd.BDStore(big_endian=True)) 70 | areas = list(section_info.area.unique()) 71 | for (index, row) in section_info.iterrows(): 72 | if row["type"] != "data": 73 | continue 74 | if row["area"] == "_main": 75 | # Include data sections from _main in all areas' BDStores. 76 | for area in areas: 77 | path = out_path / "sections/_main" / ("%02d.raw" % row["id"]) 78 | res[area].RegisterFile(path, offset=int(row["ram_start"], 16)) 79 | else: 80 | # Otherwise, include only in this area's BDStore. 81 | area = row["area"] 82 | path = out_path / "sections/rel_linked" / ( 83 | "%s/%02d.raw" % (area, row["id"])) 84 | res[area].RegisterFile(path, offset=int(row["ram_start"], 16)) 85 | return res 86 | 87 | def _GetSymbolsToDump(symbol_table, section_addrs): 88 | """Creates a table of symbols to dump w/ export_classes_parsers.""" 89 | if FLAGS.GetFlag("debug_level"): 90 | print("Finding symbols to dump...") 91 | 92 | columns = ["area", "name", "namespace", "address", "type"] 93 | data = [] 94 | # Load struct defs from export_classes_parsers. 95 | struct_defs = GetStructDefs() 96 | for (index, row) in symbol_table[symbol_table.sec_type == "data"].iterrows(): 97 | # No type / unsupported type, skip. 98 | if row["type"] not in struct_defs: 99 | continue 100 | type_def = struct_defs[row["type"]] 101 | 102 | # See how many instances there are, if the type can appear in arrays. 103 | arr_count = type_def.array 104 | if arr_count == SINGLE_INSTANCE: 105 | arr_count = 1 106 | elif arr_count == ZERO_TERMINATED: 107 | # If array is a single null entry, dump it, otherwise ignore it. 108 | arr_count = max(int(row["size"], 16) // type_def.size - 1, 1) 109 | elif arr_count == UNKNOWN_LENGTH: 110 | arr_count = int(row["size"], 16) // type_def.size 111 | 112 | # Add a row to the output dataframe per instance. 113 | ram_addr = section_addrs[ 114 | "%s-%02d" % (row["area"], row["sec_id"]) 115 | ] + int(row["sec_offset"], 16) 116 | for x in range(arr_count): 117 | name = row["name"] 118 | # Append the hexadecimal index in the array, if > 1 instance. 119 | if arr_count > 1: 120 | name += ("_%02x" if arr_count <= 256 else "_%03x") % x 121 | # Add to the output dataframe. 122 | data.append([ 123 | row["area"], name, row["namespace"], 124 | ram_addr + x * type_def.size, row["type"]]) 125 | # Add substructures to output dataframe, if necessary. 126 | # TODO: Implement support for recursive substructures? 127 | if type_def.substructs is None: 128 | continue 129 | for subtype_def in type_def.substructs: 130 | subname = name + "_" + subtype_def.name 131 | subtype_ram_addr = ram_addr + subtype_def.offset 132 | data.append([ 133 | row["area"], subname, row["namespace"], 134 | subtype_ram_addr, subtype_def.datatype]) 135 | 136 | return pd.DataFrame(data, columns=columns) 137 | 138 | def _CreateSymbolLookupTable(symbol_table, symbols_to_dump): 139 | """Creates a table for replacing pointer fields w/what they point to.""" 140 | if FLAGS.GetFlag("debug_level"): 141 | print("Creating symbol lookup table...") 142 | 143 | # Filter out bss symbols, and add numeric "address" column. 144 | symbol_table = symbol_table[symbol_table.sec_type != "bss"].copy() 145 | symbol_table["address"] = symbol_table["ram_addr"].apply(lambda x: int(x,16)) 146 | # Filter to only necessary columns. 147 | symbol_table = pd.DataFrame(symbol_table, columns=[ 148 | "area", "name", "namespace", "address", "type"]) 149 | # Add symbols from symbols_to_dump, removing existing matches if necessary. 150 | lookup_table = pd.concat([symbol_table, symbols_to_dump]) 151 | lookup_table = lookup_table.drop_duplicates( 152 | keep="last", subset=["area", "address"]) 153 | # Index by area + address for easy lookup. 154 | return lookup_table.set_index(["area", "address"]) 155 | 156 | def _DumpSymbols(out_path, symbols_to_dump, lookup_table, stores): 157 | """Dumps all symbols of supported types to .csv files in out_path.""" 158 | for classtype in sorted(symbols_to_dump.type.unique()): 159 | if FLAGS.GetFlag("debug_level"): 160 | print("Dumping instances of %s..." % classtype) 161 | 162 | dfs = [] 163 | dfs_raw = [] 164 | # Dump each instance of the class, both to fields and raw bytes. 165 | instances = symbols_to_dump.loc[symbols_to_dump["type"] == classtype] 166 | for (index, row) in instances.iterrows(): 167 | view = stores[row["area"]].view(row["address"]) 168 | dfs.append(ParseClass(view, row, lookup_table)) 169 | dfs_raw.append(ParseClassRawBytes(view, row)) 170 | # Merge dataframes and save to .csv files. 171 | pd.concat(dfs).to_csv( 172 | _GetOutputPath(out_path / "classes" / (classtype + ".csv")), 173 | encoding="utf-8", index=False) 174 | pd.concat(dfs_raw).to_csv( 175 | _GetOutputPath(out_path / "classes_raw" / (classtype + ".csv")), 176 | encoding="utf-8", index=False) 177 | 178 | def main(argc, argv): 179 | out_path = FLAGS.GetFlag("out_path") 180 | if not out_path or not os.path.exists(Path(out_path)): 181 | raise ExportClassesError("--out_path must point to a valid directory.") 182 | out_path = Path(out_path) 183 | 184 | if not os.path.exists(out_path / "section_info.csv"): 185 | raise ExportClassesError( 186 | "You must first run dump_sections.py using the same --out_path.") 187 | section_info = pd.read_csv(out_path / "section_info.csv") 188 | 189 | symbols_path = FLAGS.GetFlag("symbols_path") 190 | if not symbols_path or not os.path.exists(Path(symbols_path)): 191 | raise ExportClassesError( 192 | "--symbols_path must point to a valid symbols csv.") 193 | symbol_table = pd.read_csv(Path(symbols_path)) 194 | 195 | # Create inputs necessary for dumping symbols. 196 | 197 | section_addrs = _LoadSectionRamAddrDict(out_path, section_info) 198 | symbols_to_dump = _GetSymbolsToDump(symbol_table, section_addrs) 199 | lookup_table = _CreateSymbolLookupTable(symbol_table, symbols_to_dump) 200 | stores = _LoadSectionDataDict(out_path, section_info) 201 | 202 | _DumpSymbols(out_path, symbols_to_dump, lookup_table, stores) 203 | 204 | 205 | if __name__ == "__main__": 206 | (argc, argv) = FLAGS.ParseFlags(sys.argv[1:]) 207 | main(argc, argv) 208 | -------------------------------------------------------------------------------- /source/export_events.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.6 2 | 3 | """Exports TTYD event scripts using PistonMiner's TTYDASM tool. 4 | 5 | This file must be run after the following binaries, using the same --out_path: 6 | - dump_sections.py (to generate section_info and linked section .raw files) 7 | - symbol_to_maps.py (to generate ttydasm symbols) 8 | 9 | Given a symbols .csv and a binary of PistonMiner's TTYDASM tool, this program 10 | will then generate .txt dumps of all symbols with the type "evt", filling in 11 | symbol references with the names (or values, for string typed symbols) 12 | contained in the previously generated TTYDASM symbol maps.""" 13 | # Jonathan Aldrich 2021-02-09 ~ 2021-03-02 14 | 15 | import codecs 16 | import os 17 | import sys 18 | import numpy as np 19 | import pandas as pd 20 | import subprocess 21 | from pathlib import Path 22 | 23 | import jdalibpy.flags as flags 24 | 25 | FLAGS = flags.Flags() 26 | 27 | # Output directory; should contain these outputs from previous binaries: 28 | # - Section_info and linked sections from dump_sections.py 29 | # - Ttydasm symbol maps from symbol_to_maps.py 30 | # Dumped events will go into "/events" dir underneath. 31 | FLAGS.DefineString("out_path", "") 32 | # Input symbols file. 33 | FLAGS.DefineString("symbols_path", "") 34 | # Path to TTYDASM binary. 35 | FLAGS.DefineString("ttydasm_exe", "") 36 | 37 | # Whether to display debug strings. 38 | FLAGS.DefineInt("debug_level", 1) 39 | 40 | class ExportEventsError(Exception): 41 | def __init__(self, message=""): 42 | self.message = message 43 | 44 | def _LoadTtydasmPathDict(in_path, areas): 45 | """Constructs a dict of area name : ttydasm symbol paths.""" 46 | if FLAGS.GetFlag("debug_level"): 47 | print("Loading ttydasm symbols...") 48 | ttydasm_symbols = {} 49 | for area in areas: 50 | filepath = in_path / "ttydasm" / ("%s.sym" % area) 51 | if not os.path.exists(filepath): 52 | raise ExportEventsError( 53 | "You must first run symbol_to_maps.py using the same --out_path.") 54 | ttydasm_symbols[area] = str(filepath) 55 | return ttydasm_symbols 56 | 57 | def _LoadSectionDataDict(in_path, section_info): 58 | """Constructs a dict of area/section : data filepath + ram base address.""" 59 | if FLAGS.GetFlag("debug_level"): 60 | print("Loading section data...") 61 | 62 | section_data = {} 63 | for sec_id in (0, 1, 7, 8, 9, 10, 11, 12): 64 | dat_path = in_path / ("sections/_main/%02d.raw" % sec_id) 65 | ram_addr = int(section_info.loc[("_main", sec_id)]["ram_start"], 16) 66 | section_data["_main-%02d" % sec_id] = (str(dat_path), ram_addr) 67 | 68 | rels_dir = in_path / "sections/rel_linked" 69 | areas = [f.name for f in os.scandir(rels_dir) if f.is_dir()] 70 | for area in areas: 71 | for sec_id in range(1, 6): 72 | dat_path = rels_dir / area / ("%02d.raw" % sec_id) 73 | ram_addr = int(section_info.loc[(area, sec_id)]["ram_start"], 16) 74 | section_data["%s-%02d" % (area, sec_id)] = (str(dat_path), ram_addr) 75 | return section_data 76 | 77 | def _ExportEvents(out_path, symbols, ttydasm_symbols, section_map): 78 | def _GetEventName(row): 79 | if row["type"] != "evt": 80 | return None 81 | # Strip namespace down to last object name without its extension. 82 | ns = row["namespace"].split(" ")[-1] 83 | if ns.find("."): 84 | ns = ns[:ns.find(".")] 85 | return "%s_%s_%s" % (row["area"], ns, row["name"]) 86 | 87 | if FLAGS.GetFlag("debug_level"): 88 | print("Exporting events...") 89 | 90 | ttydasm_exe = FLAGS.GetFlag("ttydasm_exe") 91 | if not ttydasm_exe or not os.path.exists(Path(ttydasm_exe)): 92 | raise ExportEventsError("--ttydasm_exe must point to a valid binary.") 93 | if not os.path.exists(out_path / "events"): 94 | os.makedirs(out_path / "events") 95 | 96 | for (index, row) in symbols.iterrows(): 97 | event_name = _GetEventName(row) 98 | if not event_name: 99 | continue 100 | if FLAGS.GetFlag("debug_level"): 101 | print("Processing %s..." % event_name) 102 | outfile = codecs.open( 103 | out_path / "events" / ("%s.txt" % event_name), "w", encoding="utf-8") 104 | (ram_filepath, base_address) = section_map[ 105 | "%s-%02d" % (row["area"], row["sec_id"])] 106 | ram_addr = base_address + int(row["sec_offset"], 16) 107 | subprocess.check_call([ 108 | str(Path(ttydasm_exe)), 109 | "--base-address=0x%08x" % base_address, 110 | "--start-address=0x%08x" % ram_addr, 111 | "--symbol-file=%s" % ttydasm_symbols[row["area"]], 112 | ram_filepath], 113 | stdout=outfile) 114 | outfile.flush() 115 | 116 | def main(argc, argv): 117 | out_path = FLAGS.GetFlag("out_path") 118 | if not out_path or not os.path.exists(Path(out_path)): 119 | raise ExportEventsError("--out_path must point to a valid directory.") 120 | out_path = Path(out_path) 121 | 122 | if not os.path.exists(out_path / "section_info.csv"): 123 | raise ExportEventsError( 124 | "You must first run dump_sections.py using the same --out_path.") 125 | section_info = pd.read_csv(out_path / "section_info.csv") 126 | section_info = section_info.set_index(["area", "id"]) 127 | 128 | symbols_path = FLAGS.GetFlag("symbols_path") 129 | if not symbols_path or not os.path.exists(Path(symbols_path)): 130 | raise ExportEventsError( 131 | "--symbols_path must point to a valid symbols csv.") 132 | symbols = pd.read_csv(Path(symbols_path)) 133 | 134 | ttydasm_symbols = _LoadTtydasmPathDict(out_path, symbols.area.unique()) 135 | section_map = _LoadSectionDataDict(out_path, section_info) 136 | _ExportEvents(out_path, symbols, ttydasm_symbols, section_map) 137 | 138 | if __name__ == "__main__": 139 | (argc, argv) = FLAGS.ParseFlags(sys.argv[1:]) 140 | main(argc, argv) 141 | -------------------------------------------------------------------------------- /source/jdalibpy/bindatastore.py: -------------------------------------------------------------------------------- 1 | """Class for viewing, retrieving and mutating data in binary memory dumps.""" 2 | # Jonathan Aldrich 2020-12-15 3 | 4 | import ctypes # for floating-point conversions 5 | import enum # for enumerations 6 | 7 | # Custom error class. 8 | class BDError(Exception): 9 | def __init__(self, message=""): 10 | self.message = message 11 | 12 | # Enumeration of supported primitive types. 13 | class BDType(enum.Enum): 14 | INVALID = 0 15 | S8 = 1 16 | S16 = 2 17 | S32 = 3 18 | S64 = 4 19 | U8 = 5 20 | U16 = 6 21 | U32 = 7 22 | U64 = 8 23 | FLOAT = 9 24 | DOUBLE = 10 25 | CSTRING = 11 26 | BYTES = 12 27 | POINTER = 13 28 | 29 | # Represents a view into a BDStore. 30 | # TODO: Add __repr__ function for printing the view to a string? 31 | class BDView(object): 32 | def __init__(self, datastore, address): 33 | self.dat = datastore 34 | self.address = address 35 | 36 | # Functions that read from memory. 37 | 38 | def _read_byte(self, offset): 39 | """Reads a single byte from the underlying BDStore at address + offset.""" 40 | offset += self.address 41 | for b in self.dat.mem: 42 | if offset >= b.offset and offset < b.offset + len(b.data): 43 | return b.data[offset - b.offset] 44 | raise BDError("Read from unmapped offset: 0x%x" % offset) 45 | 46 | def _read_bytes(self, offset, size=1): 47 | """Reads `size` bytes from the underlying BDStore at address + offset.""" 48 | if size < 1: 49 | raise BDError("Cannot read an integer of non-positive length.") 50 | offset += self.address 51 | # Attempt to read bytes contiguously, if possible. 52 | for b in self.dat.mem: 53 | if offset >= b.offset and offset + size <= b.offset + len(b.data): 54 | return bytes(b.data[offset - b.offset : offset - b.offset + size]) 55 | # Otherwise, try to read bytes one at a time (much slower). 56 | bs = b"" 57 | offset -= self.address 58 | for x in range(size): 59 | b = self._read_byte(offset + x) 60 | bs += bytes([b]) 61 | return bs 62 | 63 | def _read_bytes_endian(self, offset, size=1, big_endian=True): 64 | """Reads `size` bytes from the underlying BDStore at address + offset, 65 | flipping the byte order to the desired endianness.""" 66 | bs = self._read_bytes(offset, size) 67 | if self.dat.big_endian == big_endian: 68 | return bs 69 | else: 70 | return bs[::-1] 71 | 72 | def _read_integer(self, offset, size=1, signed=False): 73 | """Reads an arbitrary-size integer value from the underlying BDStore 74 | at address + offset, respecting the BDStore's endianness.""" 75 | bs = self._read_bytes_endian(offset, size, big_endian=True) 76 | res = 0 77 | for x in range(size): 78 | res = 256 * res + bs[x] 79 | if res >= (1 << (size*8 - 1)) and signed: 80 | res -= (1 << size*8) 81 | return res 82 | 83 | def rs8(self, offset=0): 84 | return self._read_integer(offset, size=1, signed=True) 85 | 86 | def rs16(self, offset=0): 87 | return self._read_integer(offset, size=2, signed=True) 88 | 89 | def rs32(self, offset=0): 90 | return self._read_integer(offset, size=4, signed=True) 91 | 92 | def rs64(self, offset=0): 93 | return self._read_integer(offset, size=8, signed=True) 94 | 95 | def ru8(self, offset=0): 96 | return self._read_integer(offset, size=1, signed=False) 97 | 98 | def ru16(self, offset=0): 99 | return self._read_integer(offset, size=2, signed=False) 100 | 101 | def ru32(self, offset=0): 102 | return self._read_integer(offset, size=4, signed=False) 103 | 104 | def ru64(self, offset=0): 105 | return self._read_integer(offset, size=8, signed=False) 106 | 107 | def rf32(self, offset=0): 108 | u = self.ru32(offset) 109 | cp = ctypes.pointer(ctypes.c_uint32(u)) 110 | fp = ctypes.cast(cp, ctypes.POINTER(ctypes.c_float)) 111 | return fp.contents.value 112 | 113 | def rf64(self, offset=0): 114 | u = self.ru64(offset) 115 | cp = ctypes.pointer(ctypes.c_uint64(u)) 116 | fp = ctypes.cast(cp, ctypes.POINTER(ctypes.c_double)) 117 | return fp.contents.value 118 | 119 | def rfloat(self, offset=0): 120 | return self.rf32(offset) 121 | 122 | def rdouble(self, offset=0): 123 | return self.rf64(offset) 124 | 125 | def rcstring(self, offset=0): 126 | bs = b"" 127 | while True: 128 | b = self._read_byte(offset) 129 | if b == 0: break 130 | bs += bytes([b]) 131 | offset += 1 132 | return bs 133 | 134 | def rbytes(self, count, offset=0): 135 | return self._read_bytes(offset, count) 136 | 137 | def rptr(self, offset=0): 138 | return self._read_integer(offset, size=self.dat.ptrsize, signed=False) 139 | 140 | def read(self, type, offset=0): 141 | return { 142 | BDType.S8 : self.rs8, 143 | BDType.S16 : self.rs16, 144 | BDType.S32 : self.rs32, 145 | BDType.S64 : self.rs64, 146 | BDType.U8 : self.ru8, 147 | BDType.U16 : self.ru16, 148 | BDType.U32 : self.ru32, 149 | BDType.U64 : self.ru64, 150 | BDType.FLOAT : self.rf32, 151 | BDType.DOUBLE : self.rf64, 152 | BDType.CSTRING : self.rcstring, 153 | BDType.POINTER : self.rptr 154 | }[type](offset) 155 | 156 | # Functions that write to memory. 157 | 158 | def _write_byte(self, byte, offset): 159 | """Writes a single byte to the underlying BDStore at address + offset.""" 160 | offset += self.address 161 | for b in self.dat.mem: 162 | if offset >= b.offset and offset < b.offset + len(b.data): 163 | b.data[offset - b.offset] = byte 164 | return 165 | raise BDError("Write to unmapped offset: 0x%x" % offset) 166 | 167 | def _write_bytes(self, bs, offset): 168 | """Writes a byte string to the underlying BDStore at address + offset.""" 169 | offset += self.address 170 | # Attempt to write bytes contiguously, if possible. 171 | for b in self.dat.mem: 172 | if offset >= b.offset and offset + len(bs) <= b.offset + len(b.data): 173 | for x in range(len(bs)): 174 | b.data[offset - b.offset + x] = bs[x] 175 | return 176 | # Otherwise, try to write bytes one at a time (much slower). 177 | offset -= self.address 178 | for b in bs: 179 | self._write_byte(b, offset + x) 180 | return bs 181 | 182 | def _write_bytes_endian(self, bs, offset, big_endian=True): 183 | """Writes a byte string to the underlying BDStore at address + offset, 184 | flipping the byte order to the desired endianness.""" 185 | if self.dat.big_endian == big_endian: 186 | self._write_bytes(bs, offset) 187 | else: 188 | self._write_bytes(bs[::-1], offset) 189 | 190 | def _write_integer(self, value, offset, size=1): 191 | """Writes an arbitrary-size integer value to the underlying BDStore 192 | at address + offset, respecting the BDStore's endianness.""" 193 | bs = b"" 194 | value &= 2 ** (size*8) - 1 195 | for _ in range(size): 196 | bs += bytes([value & 0xff]) 197 | value >>= 8 198 | self._write_bytes_endian(bs, offset, big_endian=False) 199 | 200 | def w8(self, value, offset=0): 201 | self._write_integer(value, offset, size=1) 202 | 203 | def w16(self, value, offset=0): 204 | self._write_integer(value, offset, size=2) 205 | 206 | def w32(self, value, offset=0): 207 | self._write_integer(value, offset, size=4) 208 | 209 | def w64(self, value, offset=0): 210 | self._write_integer(value, offset, size=8) 211 | 212 | def wf32(self, value, offset=0): 213 | cp = ctypes.pointer(ctypes.c_float(value)) 214 | up = ctypes.cast(cp, ctypes.POINTER(ctypes.c_uint32)) 215 | self._write_integer(up.contents.value, offset, size=4) 216 | 217 | def wf64(self, value, offset=0): 218 | cp = ctypes.pointer(ctypes.c_double(value)) 219 | up = ctypes.cast(cp, ctypes.POINTER(ctypes.c_uint64)) 220 | self._write_integer(up.contents.value, offset, size=8) 221 | 222 | def wfloat(self, value, offset=0): 223 | self.wf32(value, offset) 224 | 225 | def wdouble(self, value, offset=0): 226 | self.wf64(value, offset) 227 | 228 | def wbytes(self, value, offset=0): 229 | self._write_bytes(value, offset) 230 | 231 | def wptr(self, value, offset=0): 232 | self._write_integer(value, offset, size=self.dat.ptrsize) 233 | 234 | def write(self, type, value, offset=0): 235 | return { 236 | BDType.S8 : self.w8, 237 | BDType.S16 : self.w16, 238 | BDType.S32 : self.w32, 239 | BDType.S64 : self.w64, 240 | BDType.U8 : self.w8, 241 | BDType.U16 : self.w16, 242 | BDType.U32 : self.w32, 243 | BDType.U64 : self.w64, 244 | BDType.FLOAT : self.wf32, 245 | BDType.DOUBLE : self.wf64, 246 | BDType.BYTES : self.wbytes, 247 | BDType.POINTER : self.wptr 248 | }[type](value, offset) 249 | 250 | # Functions that return new BDViews relative to this BDView. 251 | 252 | def offset(self, o): 253 | """Returns a new BDView at this view's address + the given offset.""" 254 | return BDView(self.dat, self.address + o) 255 | 256 | def at(self, o): 257 | """Returns a new BDView at this view's address + the given offset.""" 258 | return self.offset(o) 259 | 260 | def indirect(self, o): 261 | """Returns a new BDView at the address specified by rptr(offset).""" 262 | return BDView(self.dat, self.rptr(o)) 263 | 264 | def __getitem__(self, key): 265 | """Returns a new BDView at the address specified by rptr(offset).""" 266 | return self.indirect(key) 267 | 268 | # A single range of read/write memory stored in a BDStore. 269 | class BDRange(object): 270 | def __init__(self, data, offset, bounds=None): 271 | """Constructs a BDRange from an external data source. 272 | 273 | Args: 274 | - data (bytes) - The data to be stored in this range. 275 | - offset (int) - The offset used to reference this range in a BDStore. 276 | - bounds (tuple of 2 ints) - Optional; if provided, will construct 277 | the range from a slice of the provided data (if the second value is 278 | 0, uses a left-sided slice; e.g. (-4, 0) -> data[-4:]). 279 | """ 280 | if not bounds: 281 | self.data = bytearray(data) 282 | elif len(bounds) == 2: 283 | if bounds[1]: 284 | self.data = bytearray(data[bounds[0]:bounds[1]]) 285 | else: 286 | self.data = bytearray(data[bounds[0]:]) 287 | else: 288 | raise BDError("BDRange bounds must be a 2-tuple of integers:" + 289 | str(bounds)) 290 | self.offset = offset 291 | 292 | # The main class meant for public interface; holds a list of BDRanges, 293 | # each representing a read/writable range of memory starting at a given offset. 294 | class BDStore(object): 295 | def __init__(self, big_endian=False, ptrsize=4): 296 | self.mem = [] # List of BDRanges. 297 | self.big_endian = big_endian 298 | self.ptrsize = ptrsize 299 | 300 | def _list_ranges(self): 301 | bss = [] 302 | max_offset = max([b.offset + len(b.data) for b in self.mem]) 303 | max_digits = max(8, len(hex(max_offset))-2) 304 | for b in self.mem: 305 | # Print the start and end offset of the range. 306 | bs = "{start:0{digits:d}x} {end:0{digits:d}x} ".format( 307 | digits=max_digits, start=b.offset, end=b.offset + len(b.data)) 308 | # Print a preivew of bytes in the range. 309 | if len(b.data) <= 16: 310 | # If the range is short, print the entire range. 311 | bs += " ".join(["%02x" % ch for ch in b.data]) 312 | else: 313 | # Else, print the first few bytes and the last few bytes. 314 | bs += " ".join(["%02x" % ch for ch in b.data[:10]]) 315 | bs += " ..... " 316 | bs += " ".join(["%02x" % ch for ch in b.data[-4:]]) 317 | bss.append(bs) 318 | return "\n".join(bss) 319 | 320 | def _str_representation(self): 321 | s = "" 322 | bl = self._list_ranges() 323 | if bl: s += ", Ranges:\n%s" % bl 324 | return s 325 | 326 | def __str__(self): 327 | return self._str_representation() 328 | 329 | def __repr__(self): 330 | return self._str_representation() 331 | 332 | def view(self, address): 333 | """Returns a BDView on this store at a given address.""" 334 | return BDView(self, address) 335 | 336 | def at(self, address): 337 | """Returns a BDView on this store at a given address.""" 338 | return BDView(self, address) 339 | 340 | def RegisterData(self, data, offset=0, ranges=None): 341 | """Registers the provided data as one or more BDRanges. 342 | 343 | Args: 344 | - data (bytes) - The data to create ranges over. 345 | - offset (int) - The offset used to reference this data's range. 346 | - ranges (list of (offset, bounds-start, bounds-end) tuples) - Optional; 347 | if provided, will construct multiple ranges over the data. 348 | 349 | Will throw an error if the mapped offsets of a newly created range 350 | overlaps any existing range's mapped offsets. 351 | """ 352 | bdranges = [] 353 | if ranges is None: 354 | bdranges.append(BDRange(data, offset)) 355 | else: 356 | for t in ranges: 357 | if len(t) != 3: 358 | raise BDError( 359 | "'ranges' must be a list of tuples of the form " 360 | "(offset, bounds-start, bounds-end).") 361 | bdranges.append(BDRange(data, t[0], (t[1], t[2]))) 362 | for b in bdranges: 363 | if len(b.data) < 1: 364 | continue 365 | for bb in self.mem: 366 | # If ranges overlap, throw error. 367 | if (b.offset < bb.offset + len(bb.data) and 368 | bb.offset < b.offset + len(b.data)): 369 | raise BDError("Cannot create overlapping BDRanges.") 370 | self.mem.append(b) 371 | 372 | def RegisterFile(self, filename, offset=0, ranges=None): 373 | """Registers the provided file's data as one or more BDRanges. 374 | 375 | Args: 376 | - filename (str) - The filename whose data to create ranges over. 377 | - offset (int) - The offset used to reference this data's range. 378 | - ranges (list of (offset, bounds-start, bounds-end) tuples) - Optional; 379 | if provided, will construct multiple ranges over the data. 380 | 381 | Will throw an error if the mapped offsets of a newly created range 382 | overlaps any existing range's mapped offsets. 383 | """ 384 | data = open(filename, "rb").read() 385 | self.RegisterData(data, offset, ranges) 386 | -------------------------------------------------------------------------------- /source/jdalibpy/bindump.py: -------------------------------------------------------------------------------- 1 | """Utility class for retrieving data from binary memory dumps.""" 2 | # Jonathan Aldrich 2016-09-03 3 | 4 | import ctypes # for raw bytes -> floating-point conversion 5 | 6 | # Custom error class. 7 | class BinaryDumpError(Exception): 8 | def __init__(self, message=""): 9 | self.message = message 10 | 11 | # Memory region struct. 12 | class _MemoryRegion(object): 13 | def __init__(self, data, offset): 14 | self.data = data # bytes 15 | self.offset = offset # starting offset (int) 16 | 17 | # Memory dump class. 18 | class BinaryDump(object): 19 | def __init__(self, big_endian=False, ptr_bytes=4): 20 | self.mem = [] # List of _MemoryRegions. 21 | self.be = big_endian 22 | self.ptrsize = ptr_bytes 23 | 24 | def _list_blocks(self): 25 | bl = "" 26 | for region in self.mem: 27 | if bl: bl += "\n" 28 | bl += "0x%08x: %d bytes" % (region.offset, len(region.data)) 29 | return bl 30 | 31 | def _str_representation(self): 32 | s = "" 33 | bl = self._list_blocks() 34 | if bl: s += ", Blocks:\n%s" % bl 35 | return s 36 | 37 | def __str__(self): 38 | return self._str_representation() 39 | 40 | def __repr__(self): 41 | return self._str_representation() 42 | 43 | def register_block(self, data, offset=0, regions=None): 44 | blocks = [] 45 | if regions is None: 46 | blocks.append(_MemoryRegion(data, offset)) 47 | else: 48 | for t in regions: 49 | if t[1] < t[0] or t[1] >= len(data) or t[0] < 0: 50 | raise BinaryDumpError("Bounds error on region: %s" % t) 51 | return 52 | blocks.append(_MemoryRegion(data[t[0]:t[1]], t[2])) 53 | for b in blocks: 54 | self.mem.append(b) 55 | 56 | def register_file(self, filename, offset=0, regions=None): 57 | data = open(filename, "rb").read() 58 | self.register_block(data, offset, regions) 59 | 60 | def _read_byte(self, offset): 61 | for block in self.mem: 62 | if offset >= block.offset and offset < block.offset + len(block.data): 63 | return block.data[offset - block.offset] 64 | raise BinaryDumpError("Read from unmapped offset: 0x%x" % offset) 65 | 66 | def _read_contiguous_bytes(self, offset, sz): 67 | for block in self.mem: 68 | if offset >= block.offset and offset+sz <= block.offset + len(block.data): 69 | return block.data[offset-block.offset : offset-block.offset+sz] 70 | # Don't error out; couldn't read block contiguously. 71 | return None 72 | 73 | def _read_integer(self, offset, sz=1): 74 | if sz < 1: 75 | raise BinaryDumpError("Cannot read an integer of non-positive length.") 76 | res = 0 77 | for x in range(0,sz): 78 | d = x if self.be else sz-1-x 79 | res = 256 * res + self._read_byte(offset + d) 80 | return res 81 | 82 | def _read_indirect_offsets(self, offset, indirect, indirect_offset): 83 | for d in indirect: 84 | offset = self._read_integer(offset + d, self.ptrsize) 85 | return offset + indirect_offset 86 | 87 | def read_u8(self, offset, indirect=None, indirect_offset=0): 88 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 89 | return self._read_byte(offset) 90 | 91 | def read_u16(self, offset, indirect=None, indirect_offset=0): 92 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 93 | return self._read_integer(offset, 2) 94 | 95 | def read_u32(self, offset, indirect=None, indirect_offset=0): 96 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 97 | return self._read_integer(offset, 4) 98 | 99 | def read_u64(self, offset, indirect=None, indirect_offset=0): 100 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 101 | return self._read_integer(offset, 8) 102 | 103 | def read_s8(self, offset, indirect=None, indirect_offset=0): 104 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 105 | u = self.read_u8(offset) 106 | return u if u < (1<<7) else u - (1<<8) 107 | 108 | def read_s16(self, offset, indirect=None, indirect_offset=0): 109 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 110 | u = self.read_u16(offset) 111 | return u if u < (1<<15) else u - (1<<16) 112 | 113 | def read_s32(self, offset, indirect=None, indirect_offset=0): 114 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 115 | u = self.read_u32(offset) 116 | return u if u < (1<<31) else u - (1<<32) 117 | 118 | def read_s64(self, offset, indirect=None, indirect_offset=0): 119 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 120 | u = self.read_u64(offset) 121 | return u if u < (1<<63) else u - (1<<64) 122 | 123 | def read_float(self, offset, indirect=None, indirect_offset=0): 124 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 125 | u = self.read_u32(offset) 126 | cp = ctypes.pointer(ctypes.c_uint32(u)) 127 | fp = ctypes.cast(cp, ctypes.POINTER(ctypes.c_float)) 128 | return fp.contents.value 129 | 130 | def read_double(self, offset, indirect=None, indirect_offset=0): 131 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 132 | u = self.read_u64(offset) 133 | cp = ctypes.pointer(ctypes.c_uint64(u)) 134 | fp = ctypes.cast(cp, ctypes.POINTER(ctypes.c_double)) 135 | return fp.contents.value 136 | 137 | def read_char(self, offset, indirect=None, indirect_offset=0): 138 | """Reads a single byte and returns it in bytestring format.""" 139 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 140 | return bytes([self._read_byte(offset)]) 141 | 142 | def read_cstring(self, offset, indirect=None, indirect_offset=0): 143 | """Reads a zero-terminated string of bytes.""" 144 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 145 | bs = b"" 146 | d = 0 147 | while True: 148 | b = self._read_byte(offset + d) 149 | if b == 0: break 150 | bs += bytes([b]) 151 | d += 1 152 | return bs 153 | 154 | def read_bytes(self, count, offset, indirect=None, indirect_offset=0): 155 | """Reads a string of bytes of length count.""" 156 | if indirect is not None: offset = self._read_indirect_offsets(offset, indirect, indirect_offset) 157 | # First try reading as one contiguous block, since that's much faster. 158 | bs = self._read_contiguous_bytes(offset, count) 159 | if bs: 160 | return bs 161 | # Else, read one byte at a time. (TODO: speed up later if necessary) 162 | bs = b"" 163 | d = 0 164 | for x in range(count): 165 | b = self._read_byte(offset + d) 166 | bs += bytes([b]) 167 | d += 1 168 | return bs -------------------------------------------------------------------------------- /source/jdalibpy/conv.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.4 2 | 3 | """Converts between various number and datetime formats.""" 4 | # Jonathan Aldrich 2016-09-03 5 | 6 | import calendar 7 | import ctypes 8 | import datetime 9 | import sys 10 | import time 11 | 12 | HELP_STRING = """Modes: 13 | hex2dec, dec2hex, hex2float, float2hex - Numeric conversion 14 | unix2utc, utc2unix, date2unix, now2unix - Datetime conversion""" 15 | 16 | SYNONYMS = { 17 | "hex2dec": ["hex2dec", "h2d", "hd", "2d", "2dec", "dec", "d",], 18 | "dec2hex": ["dec2hex", "d2h", "dh", "2h", "2hex", "hex", "h", "x",], 19 | "hex2float": ["hex2float", "h2f", "hf", "2f", "float", "f",], 20 | "float2hex": ["float2hex", "f2h", "fh", "f2", "bits",], 21 | "unix2utc": ["unix2utc", "u2u", "utc", "2utc", "uu",], 22 | "utc2unix": ["unix2utc", "time", "time2", "timestamp", "ts", "tm", "t",], 23 | "date2unix": ["date2unix", "date", "date2", "day",], 24 | "now2unix": ["now2unix", "now2u", "n2u", "nu", "nows", "now", "today",], 25 | "help": ["help",], 26 | } 27 | 28 | def hex2dec(h): 29 | return int(h, 16) 30 | 31 | def dec2hex(d): 32 | if d < (1 << 32): 33 | return "0x%08x" % d 34 | if d < (1 << 64): 35 | return "0x%016x" % d 36 | return "0x%x" % d 37 | 38 | def hex2float(h): 39 | u = int(h, 16) 40 | up = ctypes.pointer(ctypes.c_uint32(u)) 41 | fp = ctypes.cast(up, ctypes.POINTER(ctypes.c_float)) 42 | return fp.contents.value 43 | 44 | def float2hex(f): 45 | fp = ctypes.pointer(ctypes.c_float(f)) 46 | up = ctypes.cast(fp, ctypes.POINTER(ctypes.c_uint32)) 47 | return "0x%08x" % up.contents.value 48 | 49 | def unix2utc(u): 50 | return (datetime.datetime.utcfromtimestamp(int(u)) 51 | .strftime("%Y-%m-%d %H:%M:%S")) 52 | 53 | def utc2unix(s): 54 | d = datetime.datetime.strptime(s, "%Y-%m-%d %H:%M:%S") 55 | return calendar.timegm(d.timetuple()) 56 | 57 | def date2unix(s): 58 | d = datetime.datetime.strptime(s, "%Y-%m-%d") 59 | return calendar.timegm(d.timetuple()) 60 | 61 | def now2unix(): 62 | return int(time.time()) 63 | 64 | def main(argc, argv): 65 | if argc < 1: print(HELP_STRING) 66 | elif argv[0] in SYNONYMS["help"]: print(HELP_STRING) 67 | elif argc == 1: 68 | if argv[0] in SYNONYMS["now2unix"]: print(now2unix()) 69 | else: print(HELP_STRING) 70 | elif argc > 1: 71 | if argv[0] in SYNONYMS["hex2dec"]: print(hex2dec(argv[1])) 72 | elif argv[0] in SYNONYMS["dec2hex"]: print(dec2hex(int(argv[1]))) 73 | elif argv[0] in SYNONYMS["hex2float"]: print(hex2float(argv[1])) 74 | elif argv[0] in SYNONYMS["float2hex"]: print(float2hex(float(argv[1]))) 75 | elif argv[0] in SYNONYMS["unix2utc"]: print(unix2utc(argv[1])) 76 | elif argv[0] in SYNONYMS["utc2unix"]: print(utc2unix(" ".join(argv[1:]))) 77 | elif argv[0] in SYNONYMS["date2unix"]: print(date2unix(argv[1])) 78 | else: print(HELP_STRING) 79 | 80 | if __name__ == "__main__": 81 | main(len(sys.argv) - 1, sys.argv[1:]) 82 | -------------------------------------------------------------------------------- /source/jdalibpy/flags.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.4 2 | 3 | """Command-line flag parsing utilities.""" 4 | # Jonathan Aldrich 2016-04-28 5 | 6 | class FlagMismatchError(Exception): 7 | def __init__(self, message=""): 8 | self.message = message 9 | 10 | class FlagParseError(Exception): 11 | def __init__(self, message=""): 12 | self.message = message 13 | 14 | class Flags: 15 | """Contains definitions and values for command-line flags.""" 16 | 17 | def __init__(self): 18 | self.flag_defs = {} 19 | self.flag_vals = {} 20 | 21 | def __del__(self): 22 | pass 23 | 24 | def DefineFlag(self, flag_name, value_type = "string", default_value = None): 25 | self.flag_defs[flag_name] = value_type 26 | if default_value is not None: 27 | self.flag_vals[flag_name] = default_value 28 | 29 | def DefineInt(self, flag_name, default_value = None): 30 | self.DefineFlag(flag_name, "int", default_value) 31 | 32 | def DefineFloat(self, flag_name, default_value = None): 33 | self.DefineFlag(flag_name, "float", default_value) 34 | 35 | def DefineBool(self, flag_name, default_value = None): 36 | self.DefineFlag(flag_name, "bool", default_value) 37 | 38 | def DefineString(self, flag_name, default_value = None): 39 | self.DefineFlag(flag_name, "string", default_value) 40 | 41 | def SetFlag(self, flag_name, value): 42 | if flag_name in self.flag_defs: 43 | t = self.flag_defs[flag_name] 44 | if t == "int": 45 | try: 46 | val = int(value, 0) 47 | self.flag_vals[flag_name] = val 48 | except: 49 | raise FlagMismatchError("Flag %s != int: %s" % (flag_name, value)) 50 | elif t == "float": 51 | try: 52 | self.flag_vals[flag_name] = float(value) 53 | except: 54 | raise FlagMismatchError("Flag %s != float: %s" % (flag_name, value)) 55 | elif t == "bool": 56 | try: 57 | self.flag_vals[flag_name] = bool(value) 58 | except: 59 | raise FlagMismatchError("Flag %s != bool: %s" % (flag_name, value)) 60 | else: # string 61 | self.flag_vals[flag_name] = value 62 | else: 63 | FlagParseError("Flag %s not found." % flag_name) 64 | 65 | def GetFlag(self, flag_name): 66 | return self.flag_vals[flag_name] if flag_name in self.flag_vals else None 67 | 68 | def HasFlag(self, flag_name): 69 | return flag_name in self.flag_vals 70 | 71 | def ListFlags(self): 72 | return str(self.flag_defs) 73 | 74 | def ListFlagValues(self): 75 | return str(self.flag_vals) 76 | 77 | def ParseFlags(self, argv): 78 | ret_argc = 0 79 | ret_argv = [] 80 | flag_name = "" 81 | for arg in argv: 82 | if flag_name: 83 | self.SetFlag(flag_name, arg) 84 | flag_name = "" 85 | elif arg[:2] == "--": 86 | eq = arg.find("=") 87 | if eq != -1: 88 | self.SetFlag(arg[2:eq], arg[eq+1:]) 89 | else: 90 | flag_name = arg[2:] 91 | if flag_name in self.flag_defs: 92 | if self.flag_defs[flag_name] == "bool": 93 | self.SetFlag(flag_name, True) 94 | flag_name = "" 95 | elif (flag_name[:2] == "no" and flag_name[2:] in self.flag_defs 96 | and self.flag_defs[flag_name[2:]] == "bool"): 97 | self.SetFlag(flag_name[2:], False) 98 | flag_name = "" 99 | else: 100 | ret_argc += 1 101 | ret_argv.append(arg) 102 | if flag_name: 103 | raise FlagParseError("Flag %s has no value." % flag_name) 104 | return (ret_argc, ret_argv) -------------------------------------------------------------------------------- /source/jdalibpy/rngutil.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.4 2 | 3 | # Jonathan Aldrich 2017-03-31 4 | """RNG prediction / manipulation utilities for various games. 5 | 6 | Usage: 7 | rngutil GAME MODE ARGS 8 | 9 | Games: 10 | sm64 (SM64); ttyd (PM:TTYD); pm, spm (PM64 / SPM); pit, bis (PiT / BIS) 11 | Modes / Args: 12 | -d initial_state final_state (limit): 13 | Finds the number of RNG calls between states. 14 | Stops checking after limit calls, or 32768 if no limit is specified. 15 | -a initial_state num (range): 16 | Advances the RNG state forward / back num times. 17 | If range is specified, also gives the value returned by rand[0, range). 18 | -n initial_state range val (val2) (limit) 19 | Determines how many calls needed for rand[0, range) to return 20 | a value in range [val, val2], or val exactly if no val2 is specified. 21 | Stops checking after limit calls, or 32768 if no limit is specified. 22 | -z range val (val2) (limit) 23 | Randomly tries to find the best starting state to produce the longest 24 | run of states where rand[0, range) is in range [val, val2]. 25 | Stops checking after limit calls, or 99999 if no limit is specified. 26 | -i state (For LCG random functions only): 27 | Returns the distance between the given state and the RNG's starting state.""" 28 | 29 | import math 30 | import random 31 | import sys 32 | 33 | gGameSynonyms = { 34 | "sm64": ["sm64", "sm", "64"], 35 | "pm64": ["pm", "pm64", "spm"], 36 | "ttyd": ["ttyd", "pm2"], 37 | "ml23": ["ml", "ml2", "ml3", "mlrpg2", "mlrpg3", 38 | "mlbis", "bis", "mlpit", "pit"] 39 | } 40 | gModeSynonyms = { 41 | "distance": ["find", "f", "-f", "distance", "d", "-d"], 42 | "advance": ["advance", "add", "a", "-a", "adv"], 43 | "nearest": ["nearest", "next", "n", "-n", 44 | "result", "rand", "range", "r", "-r"], 45 | "freeze": ["freeze", "fr", "frz", "z", "-z"], 46 | "index": ["index", "position", "i", "-i"], 47 | "help": ["help", "h", "-h"] 48 | } 49 | 50 | # Finds the distance of a full-period 32-bit LCG where LCG(s) = a * s + c. 51 | def _LcgDistance(final_state, a, c, start_state): 52 | candidate_state = start_state 53 | idx = 0 54 | for bit in range(32): 55 | if (final_state - (a * candidate_state + c)) % (2<> 1) ^ 0xff80) ^ (0x8180 if s4 & 1 == 1 else 0x1ff4) 84 | 85 | # Advances the state as determined by a game's specific RNG function. 86 | def Increment(self, state): 87 | state = state % 2**16 88 | if state == 21674: return 0 89 | if state == 22026: return 57460 90 | s1 = (state & 0xff) << 8 91 | s2 = state ^ s1 92 | s3 = ((s2 & 0xff) << 8) + (s2 >> 8) 93 | s4 = ((s2 & 0xff) << 1) ^ s3 94 | s5 = (s4 >> 1) ^ 0xff80 95 | s0 = s5 ^ (0x8180 if s4 & 1 == 1 else 0x1ff4) 96 | return s0 97 | 98 | # Inverse of Increment; i.e. Increment(result) = `state`. 99 | def Decrement(self, state): 100 | state = state % 2**16 101 | if state == 0: return 21674 102 | s0 = state 103 | s4 = ((s0 ^ 0xe074) & 0x7fff) << 1 104 | if self._s4_to_s6(s4) != s0: s4 = (((s0 ^ 0xe074) & 0x7fff) << 1) + 1 105 | if self._s4_to_s6(s4) != s0: s4 = (((s0 ^ 0x7e00) & 0x7fff) << 1) 106 | if self._s4_to_s6(s4) != s0: s4 = (((s0 ^ 0x7e00) & 0x7fff) << 1) + 1 107 | if self._s4_to_s6(s4) != s0: return 0 108 | s2 = s4 & 0xfe00 109 | s2 += ((s2 >> 7) ^ s4) & 0x100 110 | s2 = (((s4 ^ (s2 >> 7)) & 0xff) << 8) + (s2 >> 8) 111 | s0 = s2 & 0xff 112 | s0 += (s2 & 0xff00) ^ (s0 << 8) 113 | return s0 114 | 115 | # Given a state, returns a value in [0, rn) as determined by 116 | # a specific game's RNG. Will optionally increment `state` beforehand. 117 | def Rand(self, state, rn, increment=False): 118 | state = state % 2**16 119 | if increment: state = self.Increment(state) 120 | return state * rn // 65536 121 | 122 | class PaperMario64Rng(RngBase): 123 | """Represents the RNG functions used in Paper Mario & Super Paper Mario.""" 124 | 125 | # Advances the state as determined by a game's specific RNG function. 126 | def Increment(self, state): 127 | state = state % 2**32 128 | return (0x5D588B65 * state + 1) % 2**32 129 | 130 | # Inverse of Increment; i.e. Increment(result) = `state`. 131 | def Decrement(self, state): 132 | state = state % 2**32 133 | return (0x6A76AE6D * state + 0x95895193) % 2**32 134 | 135 | # Given a state, returns a value in [0, rn) as determined by 136 | # a specific game's RNG. Will optionally increment `state` beforehand. 137 | def Rand(self, state, rn, increment=False): 138 | state = state % 2**32 139 | if increment: state = self.Increment(state) 140 | if rn == 2: 141 | anti_range = 0xffffffff // 1001 142 | fin = state // anti_range 143 | if fin >= 1001: 144 | return self.Rand(self.Increment(state), rn) 145 | return 1 if fin < 501 else 0 146 | elif rn == 101: 147 | anti_range = 0xffffffff // 1010 148 | fin = state // anti_range 149 | if fin >= 1010: 150 | return self.Rand(self.Increment(state), rn) 151 | return 0x66666667*fin//2**34 152 | else: 153 | anti_range = 0xffffffff // rn 154 | fin = state // anti_range 155 | if fin > rn - 1: 156 | return self.Rand(self.Increment(state), rn) 157 | return fin 158 | 159 | class PaperMarioTtydRng(RngBase): 160 | """Represents the RNG functions used in Paper Mario: TTYD.""" 161 | 162 | # Advances the state as determined by a game's specific RNG function. 163 | def Increment(self, state): 164 | state = state % 2**32 165 | return (0x41C64E6D * state + 0x3039) % 2**32 166 | 167 | # Inverse of Increment; i.e. Increment(result) = `state`. 168 | def Decrement(self, state): 169 | state = state % 2**32 170 | return (0xEEB9EB65 * state + 0xFC77A683) % 2**32 171 | 172 | # Given a state, returns a value in [0, rn) as determined by 173 | # a specific game's RNG. Will optionally increment `state` beforehand. 174 | def Rand(self, state, rn, increment=False): 175 | state = state % 2**32 176 | if increment: state = self.Increment(state) 177 | return ((state >> 16) & 0x7fff) % rn 178 | 179 | # Returns the number of calls between initial_state and state. 180 | def DistanceDirect(self, state, initial_state=1): 181 | return _LcgDistance(state, 0x41C64E6D, 0x3039, initial_state) 182 | 183 | class MarioAndLuigi2Rng(RngBase): 184 | """Represents the RNG functions used in the DS Mario & Luigi games.""" 185 | 186 | # Advances the state as determined by a game's specific RNG function. 187 | def Increment(self, state): 188 | state = state % 2**16 189 | if state == 0: return 0x3ff3 190 | return (((state * 41) >> 1) & 0x7fff) + 0x8000 * (state & 1) 191 | 192 | # Inverse of Increment; i.e. Increment(result) = `state`. 193 | def Decrement(self, state): 194 | state = state % 2**16 195 | x = state % 0x10000 196 | for z in range(0,41): 197 | ix = math.ceil(((x&0x7fff)+(z*0x10000))/20.5)&0xffff 198 | if self.Increment(int(ix)) == x: return ix 199 | return -1 200 | 201 | # Given a state, returns a value in [0, rn) as determined by 202 | # a specific game's RNG. Will optionally increment `state` beforehand. 203 | def Rand(self, state, rn, increment=False): 204 | state = state % 2**16 205 | if increment: state = self.Increment(state) 206 | return state % rn 207 | 208 | def _Distance(rng, initial_state, final_state, max_calls): 209 | if rng.DistanceDirect: 210 | distance = rng.DistanceDirect(final_state, initial_state) 211 | if distance * 3 / 2 > 2**32: 212 | distance = distance - 2**32 213 | print("Num calls: %i" % (distance)) 214 | return 215 | fwd = initial_state 216 | bck = initial_state 217 | for x in range(1, max_calls+1): 218 | fwd = rng.Increment(fwd) 219 | bck = rng.Decrement(bck) 220 | if (fwd == final_state): 221 | print("Num calls: +%i" % (x)) 222 | return 223 | if (bck == final_state): 224 | print("Num calls: -%i" % (x)) 225 | return 226 | print("More than %i calls away." % (max_calls)) 227 | 228 | def _Advance(rng, state, num_calls, rn): 229 | if num_calls >= 0: 230 | for _ in range(0, num_calls): 231 | state = rng.Increment(state) 232 | else: 233 | for _ in range(0, -num_calls): 234 | state = rng.Decrement(state) 235 | if rn != None and rn > 0: 236 | print("Final state: 0x%08x, rand(range): %i" % 237 | (state, rng.Rand(state, rn))) 238 | else: 239 | print("Final state: 0x%08x" % (state)) 240 | 241 | def _Nearest(rng, state, rn, low, high, max_calls): 242 | for x in range(1, max_calls+1): 243 | state = rng.Increment(state) 244 | dec = rng.Rand(state, rn) 245 | if dec >= low and dec <= high: 246 | print("Num calls: %i, Final state: 0x%08x" % (x, state)) 247 | return 248 | print("More than %i calls away." % (max_calls)) 249 | 250 | def _Freeze(rng, bits, rn, low, high, max_calls): 251 | bstate = 0 252 | bcount = 0 253 | for x in range(max_calls): 254 | state = random.randrange(2**bits) 255 | state_nx = state 256 | count = 0 257 | while True: 258 | state_nx = rng.Increment(state_nx) 259 | dec = rng.Rand(state_nx, rn) 260 | if dec < low or dec > high: 261 | break 262 | count += 1 263 | if count > bcount: 264 | bstate = state 265 | bcount = count 266 | print("Best frozen state / length: 0x%08x (%i calls)." % (bstate, bcount)) 267 | 268 | def _Index(rng, state): 269 | if not rng.DistanceDirect: 270 | print("Direct index not supported for non-LCG RNG functions.") 271 | return 272 | index = rng.DistanceDirect(state) 273 | print("Index of state 0x%08x: %i" % (state, index)) 274 | 275 | def main(argc, argv): 276 | if argc < 3: 277 | print(__doc__) 278 | return 279 | if argv[0] in gModeSynonyms["help"]: 280 | print(__doc__) 281 | return 282 | 283 | # Select the RNG implementation based on the game parameter. 284 | rng = None 285 | bits = 0 286 | if argv[0] in gGameSynonyms["sm64"]: 287 | rng = SuperMario64Rng() 288 | bits = 16 289 | if argv[0] in gGameSynonyms["pm64"]: 290 | rng = PaperMario64Rng() 291 | bits = 32 292 | if argv[0] in gGameSynonyms["ttyd"]: 293 | rng = PaperMarioTtydRng() 294 | bits = 32 295 | if argv[0] in gGameSynonyms["ml23"]: 296 | rng = MarioAndLuigi2Rng() 297 | bits = 16 298 | if not rng: 299 | print(__doc__) 300 | return 301 | 302 | # Call one of the main functions based on the mode parameter. 303 | if argv[1] in gModeSynonyms["distance"]: 304 | if argc < 4: # doesn't have enough arguments 305 | print(__doc__) 306 | return 307 | max_calls = 32768 308 | if argc > 4: max_calls = int(argv[4], 0) # opt. limit 309 | _Distance(rng, int(argv[2], 0), int(argv[3], 0), max_calls) 310 | return 311 | if argv[1] in gModeSynonyms["advance"]: 312 | if argc < 4: # doesn't have enough arguments 313 | print(__doc__) 314 | return 315 | rn = None 316 | if argc > 4: rn = int(argv[4], 0) # opt. range 317 | _Advance(rng, int(argv[2], 0), int(argv[3], 0), rn) 318 | return 319 | if argv[1] in gModeSynonyms["nearest"]: 320 | if argc < 5: # doesn't have enough arguments 321 | print(__doc__) 322 | return 323 | end_range = int(argv[4], 0) 324 | max_calls = 32768 325 | if argc > 6: max_calls = int(argv[6], 0) # opt. limit 326 | if argc > 5: end_range = int(argv[5], 0) # opt. range end 327 | _Nearest(rng, int(argv[2], 0), int(argv[3], 0), int(argv[4], 0), 328 | end_range, max_calls) 329 | return 330 | if argv[1] in gModeSynonyms["freeze"]: 331 | if argc < 4: # doesn't have enough arguments 332 | print(__doc__) 333 | return 334 | end_range = int(argv[3], 0) 335 | max_calls = 99999 336 | if argc > 5: max_calls = int(argv[5], 0) # opt. limit 337 | if argc > 4: end_range = int(argv[4], 0) # opt. range end 338 | _Freeze(rng, bits, int(argv[2], 0), int(argv[3], 0), 339 | end_range, max_calls) 340 | return 341 | if argv[1] in gModeSynonyms["index"]: 342 | _Index(rng, int(argv[2], 0)) 343 | return 344 | print(__doc__) 345 | 346 | if __name__ == "__main__": 347 | main(len(sys.argv) - 1, sys.argv[1:]) -------------------------------------------------------------------------------- /source/map_to_symbols.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.6 2 | 3 | """Converts .MAP files into tables of symbol + predicted type information. 4 | 5 | This program takes in .MAP files for the .DOL and .RELs in a single version of 6 | TTYD, and produces a single .csv containing the areas, names/namespaces, 7 | section offsets and sizes of all contained symbols.""" 8 | # Jonathan Aldrich 2021-01-22 ~ 2021-03-04 9 | 10 | import codecs 11 | import glob 12 | import os 13 | import sys 14 | import numpy as np 15 | import pandas as pd 16 | from pathlib import Path 17 | 18 | import jdalibpy.flags as flags 19 | 20 | FLAGS = flags.Flags() 21 | 22 | # Input pattern for the TTYD demo DOL .MAP file and REL .MAP files. 23 | # REL pattern must contain a single asterisk that matches the REL name. 24 | FLAGS.DefineString("dol_map", "") 25 | FLAGS.DefineString("rel_map", "") 26 | FLAGS.DefineString("encoding", "utf-8") # Change if MAPs in different encoding. 27 | 28 | # Output directory. 29 | FLAGS.DefineString("out_path", "") 30 | 31 | # Whether to display debug strings. 32 | FLAGS.DefineInt("debug_level", 1) 33 | 34 | class MapToSymbolError(Exception): 35 | def __init__(self, message=""): 36 | self.message = message 37 | 38 | def _ProcessMap(section_info, filepath, area): 39 | def _CreateSymbolDf(area, sec_id, sec_name, sec_type, line): 40 | # TODO: Add local / global information? Needs to be parsed from header. 41 | columns = [ 42 | "area", "sec_id", "sec_name", "sec_type", "ram_addr", "file_addr", 43 | "name", "namespace", "sec_offset", "size", "align" 44 | ] 45 | 46 | tokens = list(filter(None, line.split())) 47 | # Not a symbol, or symbol is unused / size 0; return an empty DataFrame. 48 | if (len(tokens) < 6 or tokens[4][0] == "." or tokens[0] == "UNUSED" or 49 | not int(tokens[1], 16)): 50 | return pd.DataFrame(columns=columns) 51 | 52 | name = tokens[4] 53 | namespace = " ".join(tokens[5:]) 54 | sec_offset = tokens[0] 55 | size = "%08x" % (int(tokens[1], 16)) 56 | alignment = int(tokens[3], 10) 57 | 58 | # Calculate RAM and file-level addresses based on section_info table. 59 | section = section_info.loc[(area, sec_id)] 60 | ram_addr = section["ram_start"] 61 | ram_addr = ( 62 | "%08x" % (int(ram_addr, 16) + int(sec_offset, 16)) 63 | if isinstance(ram_addr, str) and ram_addr else np.nan 64 | ) 65 | file_addr = section["file_start"] 66 | file_addr = ( 67 | "%08x" % (int(file_addr, 16) + int(sec_offset, 16)) 68 | if isinstance(file_addr, str) and file_addr else np.nan 69 | ) 70 | 71 | return pd.DataFrame([[ 72 | area, sec_id, sec_name, sec_type, ram_addr, file_addr, 73 | name, namespace, sec_offset, size, alignment 74 | ]], columns=columns) 75 | 76 | dol_sections = { 77 | ".init" : 0, ".text": 1, ".ctors": 7, ".dtors": 8, 78 | ".rodata": 9, ".data": 10, ".sdata": 11, ".sdata2": 12, 79 | ".bss": 100, ".sbss": 101, ".sbss2": 102 80 | } 81 | rel_sections = { 82 | ".text": 1, ".ctors": 2, ".dtors": 3, 83 | ".rodata": 4, ".data": 5, ".bss": 6 84 | } 85 | 86 | # Loop over file's lines, adding symbol information. 87 | dfs = [] 88 | sec_id = -1 89 | for line in codecs.open(filepath, "r", FLAGS.GetFlag("encoding")).readlines(): 90 | if "Memory map" in line: 91 | break 92 | elif "section layout" in line: 93 | sec_name = line[:line.find(" ")] 94 | if area == "_main": 95 | if sec_name in dol_sections: 96 | sec_id = dol_sections[sec_name] 97 | if sec_id < 7: 98 | sec_type = "text" 99 | elif sec_id < 100: 100 | sec_type = "data" 101 | else: 102 | sec_type = "bss" 103 | else: 104 | sec_id = -1 105 | else: 106 | if sec_name in rel_sections: 107 | sec_id = rel_sections[sec_name] 108 | if sec_id < 4: 109 | sec_type = "text" 110 | elif sec_id < 6: 111 | sec_type = "data" 112 | else: 113 | sec_type = "bss" 114 | else: 115 | sec_id = -1 116 | if sec_id != -1: 117 | if FLAGS.GetFlag("debug_level"): 118 | print("Extracting symbols from %s%s..." % (area, sec_name)) 119 | elif sec_id != -1: 120 | # In a supported section; create DataFrame from line. 121 | dfs.append(_CreateSymbolDf(area, sec_id, sec_name, sec_type, line)) 122 | 123 | df = pd.concat(dfs, ignore_index=True) 124 | df = df.set_index(["area", "sec_id", "sec_offset"]) 125 | df = df.sort_index() 126 | return df 127 | 128 | def main(argc, argv): 129 | out_path = Path(FLAGS.GetFlag("out_path")) 130 | if not out_path: 131 | raise MapToSymbolError("Must provide a directory for --out_path.") 132 | elif not os.path.exists(out_path): 133 | os.makedirs(out_path) 134 | 135 | if not FLAGS.GetFlag("encoding"): 136 | raise MapToSymbolError("--encoding must be set (defaults to utf-8).") 137 | 138 | # Load section info table for file/RAM-relative addresses. 139 | if not os.path.exists(out_path / "section_info.csv"): 140 | raise MapToSymbolError( 141 | "--out_path must contain results of dump_sections.py.") 142 | section_info = pd.read_csv(out_path / "section_info.csv") 143 | section_info = section_info.set_index(["area", "id"]) 144 | 145 | # Create list of symbol_info dataframes to be later merged. 146 | symbol_info = [] 147 | 148 | # Parse symbols from .DOL map. 149 | dol_path = Path(FLAGS.GetFlag("dol_map")) 150 | if not dol_path.exists(): 151 | raise MapToSymbolError("--dol_map file does not exist." % version) 152 | symbol_info.append(_ProcessMap(section_info, dol_path, "_main")) 153 | 154 | # Parse symbols from .REL maps. 155 | rel_pattern = FLAGS.GetFlag("rel_map") 156 | normalized_pattern = str(Path(rel_pattern)) 157 | # Verify exactly one asterisk wildcard exists. 158 | lpos = normalized_pattern.find("*") 159 | rpos = normalized_pattern.rfind("*") 160 | if lpos != rpos or lpos == -1: 161 | raise MapToSymbolError( 162 | "--rel_map pattern must contain exactly one wildcard asterisk.") 163 | # Verify any files are matched. 164 | if not glob.glob(rel_pattern): 165 | raise MapToSymbolError("--rel_map pattern matched no files.") 166 | # For each file, get the full path and the part of the string 167 | # that replaced the asterisk (which should be the area's name). 168 | for fn in sorted(glob.glob(rel_pattern)): 169 | # Skip the DOL's map if it matches the REL pattern. 170 | if fn == dol_path: 171 | continue 172 | # Skip if the filepath contains "_all". 173 | if "_all" in str(fn): 174 | continue 175 | filepath = str(fn) 176 | area = filepath[lpos:rpos+1-len(normalized_pattern)] 177 | symbol_info.append(_ProcessMap(section_info, filepath, area)) 178 | 179 | if not len(symbol_info): 180 | raise MapToSymbolError("No DOLs or RELs were processed.") 181 | # Concatenate all areas' symbols into one DataFrame. 182 | df = pd.concat(symbol_info) 183 | df = df.sort_index() 184 | df.to_csv(out_path / "map_symbols.csv", encoding="utf-8") 185 | 186 | if __name__ == "__main__": 187 | (argc, argv) = FLAGS.ParseFlags(sys.argv[1:]) 188 | main(argc, argv) 189 | -------------------------------------------------------------------------------- /source/old_utils/ttyd_exporteventscripts.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.4 2 | 3 | """Exports EventScripts from diffs data using PistonMiner's ttydasm tool.""" 4 | # Jonathan "jdaster64" Aldrich 2019-09-30 5 | 6 | import codecs 7 | import os 8 | import sys 9 | import subprocess 10 | import numpy as np 11 | import pandas as pd 12 | 13 | import jdalibpy.flags as flags 14 | import ttyd_maplib 15 | 16 | FLAGS = flags.Flags() 17 | # Filepath to a file in the format of REPO/resources/ttyd_*_symboldiffs_*.csv. 18 | FLAGS.DefineString("input_diffs", "") 19 | # Filepattern matching RAM dumps from every area in the game, named by their 20 | # internal area code (e.g. "aji" = X-Naut fortress), with the area replaced with 21 | # the wildcard *; Example: "path/to/file/*.raw" 22 | FLAGS.DefineString("input_ram_pattern", "") 23 | # Filepath to PistonMiner's ttydasm tool. 24 | FLAGS.DefineString("ttydasm_exe", "") 25 | # Filepattern matching ttydasm symbol files for every area in the game. 26 | # Can alternatively use a single file with no wildcard. 27 | FLAGS.DefineString("ttydasm_symbols_pattern", "") 28 | # Directory to output all script text files to. 29 | FLAGS.DefineString("output_dir", "") 30 | 31 | class ExportEventScriptsError(Exception): 32 | def __init__(self, message=""): 33 | self.message = message 34 | 35 | def _ParseScript(df_row): 36 | area = df_row["area"] 37 | sym_area = "tik" if area == "_MS" else area 38 | eventname = df_row["eventname"] 39 | symbol_file = FLAGS.GetFlag("ttydasm_symbols_pattern").replace("*",sym_area) 40 | ram_filepath = FLAGS.GetFlag("input_ram_pattern").replace("*", sym_area) 41 | if not os.path.exists(ram_filepath): 42 | print("Skipping event %s_%s (no RAM dump)" % (area, eventname,)) 43 | return 44 | else: 45 | print("Parsing event %s_%s" % (area, eventname,)) 46 | outfile = codecs.open( 47 | os.path.join( 48 | FLAGS.GetFlag("output_dir"), "%s_%s.txt" % (area, eventname) 49 | ), "w", encoding="utf-8") 50 | subprocess.check_call([ 51 | FLAGS.GetFlag("ttydasm_exe"), 52 | "--base-address=0x80000000", 53 | "--start-address=0x%x" % (df_row["address"],), 54 | "--symbol-file=%s" % (symbol_file,), 55 | ram_filepath], 56 | stdout=outfile) 57 | outfile.flush() 58 | 59 | def main(argc, argv): 60 | if not FLAGS.GetFlag("input_diffs"): 61 | raise ExportEventScriptsError("No input diffs CSV provided.") 62 | if not FLAGS.GetFlag("input_ram_pattern"): 63 | raise ExportEventScriptsError("No input ram filepattern provided.") 64 | if not FLAGS.GetFlag("output_dir"): 65 | raise ExportEventScriptsError("No output script directory provided.") 66 | if (not FLAGS.GetFlag("ttydasm_exe") or 67 | not FLAGS.GetFlag("ttydasm_symbols_pattern")): 68 | raise ExportEventScriptsError( 69 | "Both ttydasm executable and symbol filepattern must be provided.") 70 | 71 | # Get a DataFrame of symbol information from the input diffs file. 72 | df = ttyd_maplib.GetSymbolInfoFromDiffsCsv(FLAGS.GetFlag("input_diffs")) 73 | 74 | # Store the name of the output event file in the dataframe. 75 | def get_event_name(row): 76 | if row["file"]: 77 | # TODO: Add an option to include area in the "uniqueness" key. 78 | return "%s_%s" % (row["file"][:-2], row["name"]) 79 | else: 80 | return row["fullname"] 81 | 82 | # Keep only event scripts. 83 | df = df.loc[df["class"] == "EventScript_t"] 84 | df["eventname"] = df.apply(get_event_name, axis=1) 85 | # Remove non-events and deduplicate by event name. 86 | df.drop_duplicates(subset=["eventname"], inplace=True) 87 | # Sort by area and address for debugging convenience's sake. 88 | df.sort_values(["area", "address"], kind="mergesort", inplace=True) 89 | 90 | # Process each event using ttydasm. 91 | for idx, row in df.iterrows(): 92 | _ParseScript(row) 93 | 94 | if __name__ == "__main__": 95 | (argc, argv) = FLAGS.ParseFlags(sys.argv[1:]) 96 | main(argc, argv) -------------------------------------------------------------------------------- /source/old_utils/ttyd_generatesymbolmaps.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.4 2 | 3 | """Generates symbol maps given input diffs, in standard or ttydasm format.""" 4 | # Jonathan "jdaster64" Aldrich 2019-09-29 5 | 6 | import codecs 7 | import os 8 | import sys 9 | import re 10 | import numpy as np 11 | import pandas as pd 12 | 13 | import jdalibpy.flags as flags 14 | import ttyd_maplib 15 | 16 | FLAGS = flags.Flags() 17 | # Filepath to a file in the format of REPO/resources/ttyd_*_symboldiffs_*.csv. 18 | FLAGS.DefineString("input_diffs", "") 19 | # Filepath to a ttydasm symbol file to combine with the symbols generated from 20 | # the diffs file. 21 | FLAGS.DefineString("input_ttydasm_symbase", "") 22 | # Output directory for MAP files. 23 | FLAGS.DefineString("output_maps_dir", "") 24 | # Output directory for ttydasm symbol files. 25 | FLAGS.DefineString("output_ttydasm_maps_dir", "") 26 | 27 | class GenerateSymbolMapsError(Exception): 28 | def __init__(self, message=""): 29 | self.message = message 30 | 31 | def main(argc, argv): 32 | if not FLAGS.GetFlag("input_diffs"): 33 | raise GenerateSymbolMapsError("No input diffs CSV provided.") 34 | if (not FLAGS.GetFlag("output_maps_dir") and 35 | not FLAGS.GetFlag("output_ttydasm_maps_dir")): 36 | raise GenerateSymbolMapsError( 37 | "Neither a standard nor ttydasm output dir provided.") 38 | 39 | # Get a DataFrame of symbol information from the input diffs file. 40 | df = ttyd_maplib.GetSymbolInfoFromDiffsCsv(FLAGS.GetFlag("input_diffs")) 41 | 42 | # Store symbol info in map file and ttydasm map file formats in dataframe. 43 | def get_mapfile_row(row): 44 | return "0x%08x 0x%06x 0x%08x 0 %s\n" % ( 45 | row["address"], row["length"], row["address"], row["fullname"]) 46 | def get_ttydasm_row(row): 47 | return "%08X:%s\n" % (row["address"], row["fullname"]) 48 | 49 | df["maprow"] = df.apply(get_mapfile_row, axis=1) 50 | df["ttydasmrow"] = df.apply(get_ttydasm_row, axis=1) 51 | 52 | # Sort by section and remove duplicate symbol addresses in the same area. 53 | sections = [".text", ".rodata", ".data", ".sdata"] 54 | df["section"] = pd.Categorical(df["section"], sections) 55 | df.sort_values(["area", "section"], kind="mergesort", inplace=True) 56 | # Probably can be removed; artifact of when this supported a base map. 57 | df.drop_duplicates(subset=["area", "fullname", "address"], inplace=True) 58 | 59 | # Output standard maps for the DOL & each REL represented in the diffs file. 60 | for area in df.area.unique(): 61 | print("Generating symbol map file for %s..." % (area,)) 62 | 63 | filename = "boot.map" if area == "_MS" else "%s.map" % (area,) 64 | outfile = open( 65 | os.path.join(FLAGS.GetFlag("output_maps_dir"), filename), "w") 66 | df_area = df.loc[df["area"].isin(["_MS", area])] 67 | for section in sections: 68 | df_section = df_area.loc[df_area["section"] == section] 69 | if len(df_section.index) > 0: 70 | outfile.write("%s section layout\n" % (section,)) 71 | for text in df_section["maprow"]: 72 | outfile.write(text) 73 | outfile.write("\n") 74 | 75 | # Sort by area and address only, regardless of section for ttydasm symbols. 76 | df.sort_values(["area", "address"], kind="mergesort", inplace=True) 77 | 78 | # Output ttydasm maps for the DOL & each REL represented in the diffs file. 79 | for area in df.area.unique(): 80 | print("Generating ttydasm symbols for %s..." % (area,)) 81 | 82 | filename = "%s.sym" % ("Start_us" if area == "_MS" else area,) 83 | outfile = codecs.open(os.path.join( 84 | FLAGS.GetFlag("output_ttydasm_maps_dir"), filename), 85 | "w", encoding="utf-8") 86 | 87 | # Deduplicate with base ttydasm symbols. (TODO: Clean this up!) 88 | symbols = [] 89 | if FLAGS.GetFlag("input_ttydasm_symbase"): 90 | for sym in codecs.open( 91 | FLAGS.GetFlag("input_ttydasm_symbase"), "r", encoding="utf-8" 92 | ).readlines(): 93 | symbols.append(sym) 94 | base_symbol_addresses = [sym[:sym.find(":")] for sym in symbols] 95 | 96 | df_area = df.loc[df["area"].isin(["_MS", area])] 97 | for sym in df_area["ttydasmrow"]: 98 | if sym[:sym.find(":")] not in base_symbol_addresses: 99 | symbols.append(sym) 100 | 101 | for sym in sorted(symbols): 102 | outfile.write(sym) 103 | 104 | if __name__ == "__main__": 105 | (argc, argv) = FLAGS.ParseFlags(sys.argv[1:]) 106 | main(argc, argv) -------------------------------------------------------------------------------- /source/old_utils/ttyd_maplib.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.4 2 | 3 | """Various utilities for dealing with TTYD symbol information.""" 4 | # Jonathan "jdaster64" Aldrich 2019-09-29 5 | 6 | import os 7 | import sys 8 | import re 9 | import numpy as np 10 | import pandas as pd 11 | 12 | import jdalibpy.flags as flags 13 | 14 | FLAGS = flags.Flags() 15 | # Filepath to a file in the format of REPO/resources/ttyd_*_symboldiffs_*.csv. 16 | FLAGS.DefineString("input_diffs", "") 17 | 18 | class TtydSymbolLibError(Exception): 19 | def __init__(self, message=""): 20 | self.message = message 21 | 22 | def GetClassSize(classname): 23 | """ 24 | Returns the size in bytes of a member of class `classname`, or -1 25 | if the class does not have a consistent size. 26 | """ 27 | kClassNameSizes = { 28 | "AttackParams_t": 0xc0, 29 | "AudienceItemWeight_t": 0x8, 30 | "BattleLoadoutParams_t": 0x20, 31 | "BattleObjectData_t": 0x18, 32 | "BattleSetup_t": 0x44, 33 | "BattleSetupNoTbl_t": 0x8, 34 | "BattleStageData_t": 0x1b4, 35 | "BattleUnitDefense_t": 0x5, 36 | "BattleUnitDefenseAttr_t": 0x5, 37 | "BattleUnitEntry_t": 0x30, 38 | "BattleUnitParams_t": 0xc4, 39 | "BattleUnitParts_t": 0x4c, 40 | "BattleUnitStatusVulnerability_t": 0x16, 41 | "BattleWeightedLoadout_t": 0xc, 42 | "CookingRecipe_t": 0xc, 43 | "ItemData_t": 0x28, 44 | "ItemDropWeight_t": 0x8, 45 | "PointDropWeights_t": 0x50, 46 | "ShopItemList_t": 0x8, 47 | "ShopSellPriceList_t": 0x8, 48 | } 49 | return kClassNameSizes[classname] if classname in kClassNameSizes else -1 50 | 51 | 52 | # TODO: Implement different base REL addresses per region, and add an option 53 | # to keep symbols starting with "@" to at least delineate them in Dolphin. 54 | def GetSymbolInfoFromDiffsCsv(filepath, region="U"): 55 | """ 56 | Input: filepath to a CSV file containing at least the following columns: 57 | "Sec" - Section symbol is from, e.g. .text, .rodata, .data 58 | "Area" - .rel file the symbol is from, or "_MS" for the main dol. 59 | "Symbol" - Name / object file (namespace) the symbol is from. 60 | "Actual-B" - Offset the symbol is found at, from the main dol when 61 | loaded in the game for "_MS" symbols, or from the start 62 | of the rel file for others. 63 | "Len-B" - Size of the symbol in bytes. 64 | "Class" - Type of the underlying structure; may not be of a 65 | fixed size (e.g. "EventScript_t" for event scripts). 66 | 67 | Output: pandas.DataFrame with the following columns: 68 | area, fullname, name, file, section, address, offset, length, class 69 | """ 70 | df = pd.read_csv(filepath, header=0).dropna( 71 | axis=0, subset=["Actual-B", "Len-B"]) 72 | 73 | def convert_diffs(row): 74 | section = row["Sec"] 75 | area = row["Area"] 76 | symbol_full_name = row["Symbol"] 77 | offset = int(row["Actual-B"], 0) 78 | length = int(row["Len-B"], 0) 79 | class_name = row["Class"] 80 | 81 | # Split full name into symbol name and namespace / object file. 82 | if symbol_full_name[-2:] == ".o": 83 | symbol_name = symbol_full_name[:symbol_full_name.rfind(" ")] 84 | symbol_file = symbol_full_name[symbol_full_name.rfind(" ")+1:] 85 | else: 86 | symbol_name = symbol_full_name 87 | symbol_file = "" 88 | # Calculate the absolute address based on the area / offset. 89 | # TODO: These aren't technically fixed addresses (especially jon). 90 | address = offset 91 | if row["Area"] == "jon": 92 | address += 0x80c779a0 93 | elif row["Area"] != "_MS": 94 | address += 0x805ba9a0 95 | 96 | return pd.Series( 97 | data=[area, symbol_full_name, symbol_name, symbol_file, section, 98 | address, offset, length, class_name], 99 | index=["area", "fullname", "name", "file", "section", 100 | "address", "offset", "length", "class"]) 101 | 102 | # Filter out symbols w/unknown name or location, and sort by area + address. 103 | filter = (df["Actual-B"] != "UNUSED") & ~(df["Symbol"].str.contains("@")) 104 | df = df.loc[filter].apply(convert_diffs, axis=1) 105 | df.sort_values(["area", "address"], kind="mergesort", inplace=True) 106 | return df 107 | 108 | def LookupSymbolName(df, area, address, classname=""): 109 | """ 110 | Returns the full name of a symbol in from a DataFrame of symbol information 111 | (as returned by GetSymbolInfoFromDiffsCsv) based on its area and address. 112 | Matches can be in the area provided or in the main DOL ("_MS"). 113 | 114 | If a classname is provided, will ensure that the symbol also matches it, and 115 | if the class has a defined size, will look for individual instances with 116 | a matching address in arrays of that class type, and return the name/index. 117 | 118 | If no match is found, returns the address as a hex string. 119 | """ 120 | 121 | # TODO: Improve runtime further for common case. 122 | filter = df["area"].isin([area, "_MS"]) & df["address"] < address 123 | if classname: 124 | filter = filter & (df["class"] == classname) 125 | for idx, row in df.loc[filter].iloc[::-1].iterrows(): 126 | class_size = GetClassSize(classname) 127 | if class_size > 0: 128 | array_len = row["length"] // class_size 129 | for array_idx in range(array_len): 130 | if row["address"] + class_size * array_idx == address: 131 | format_sp = "_%03x" if array_len > 255 else "_%02x" 132 | return row["fullname"] + (format_sp % array_idx) 133 | elif row["address"] == address: 134 | return row["fullname"] 135 | return "0x%08x" % address 136 | 137 | def main(argc, argv): 138 | # If main() is called, test the library on an input CSV. 139 | if not FLAGS.GetFlag("input_diffs"): 140 | raise TtydSymbolLibError("No input diffs CSV provided.") 141 | 142 | # Read symbol data from diffs file. 143 | df = GetSymbolInfoFromDiffsCsv(FLAGS.GetFlag("input_diffs")) 144 | # Print some sample data. 145 | print("Shape: %s" % (str(df.shape),)) 146 | print(df.sample(n=5)) 147 | print(df.dtypes) 148 | 149 | if __name__ == "__main__": 150 | (argc, argv) = FLAGS.ParseFlags(sys.argv[1:]) 151 | main(argc, argv) -------------------------------------------------------------------------------- /source/setup.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import os 3 | import hashlib 4 | import sys 5 | from pathlib import Path 6 | import jdalibpy.flags as flags 7 | 8 | def _ExitWithHelp(message): 9 | print(f"\n{message}") 10 | print(""" 11 | Required Args: 12 | --ver=version (supported: us, jp, pal) 13 | --out="path_to_out_folder" 14 | --rels_directory="path_to_rel_folder" 15 | --dol_filepath="path_to_main_dol" 16 | --ttydasm="path_to_ttydasm_exe" 17 | Optional Args: 18 | --ttyd_utils="path_to_ttyd_utils" (Defaults to parent dir)""") 19 | print(""" 20 | Example Invocation from ttyd-utils/source/ directory: 21 | python3 ./setup.py --ver="us" --out="../../tools-out" --dol_filepath="../../iso/sys/main.dol" --rels_directory="../../iso/files/rel" --ttydasm="../../tools/ttydasm.exe" 22 | """) 23 | sys.exit(1) 24 | 25 | # Define all flags 26 | flag_mgr = flags.Flags() 27 | flag_mgr.DefineString("ver", "") 28 | flag_mgr.DefineString("out", "") 29 | flag_mgr.DefineString("dol_filepath", "") 30 | flag_mgr.DefineString("rels_directory", "") 31 | flag_mgr.DefineString("ttydasm", "") 32 | flag_mgr.DefineString("ttyd_utils", "..") 33 | 34 | # Parse command-line arguments 35 | argc, argv = flag_mgr.ParseFlags(sys.argv[1:]) 36 | 37 | # Get values of flags 38 | ver = flag_mgr.GetFlag("ver").lower() 39 | out_path = flag_mgr.GetFlag("out") 40 | dol_filepath = flag_mgr.GetFlag("dol_filepath") 41 | rels_directory = flag_mgr.GetFlag("rels_directory") 42 | ttydasm = flag_mgr.GetFlag("ttydasm") 43 | ttyd_utils = flag_mgr.GetFlag("ttyd_utils") 44 | ttyd_utils_src_dir = f"{ttyd_utils}/source/" 45 | 46 | if ver not in ("us", "jp", "pal"): 47 | _ExitWithHelp("Error: No or bad --ver provided.") 48 | if not out_path: 49 | _ExitWithHelp("Error: No --out provided.") 50 | if not dol_filepath: 51 | _ExitWithHelp("Error: No --dol_filepath provided.") 52 | if not rels_directory: 53 | _ExitWithHelp("Error: No --rels_directory provided.") 54 | if not ttydasm: 55 | _ExitWithHelp("Error: No --ttydasm provided.") 56 | 57 | if (ver == "us"): 58 | csv = "us_symbols.csv" 59 | link_addr = '--link_address=0x805ba9a0' 60 | link_addr_overrides = '--link_address_overrides=jon:0x80c779a0,tst:0x80c779a0' 61 | rel_bss_addr = '--rel_bss_address=0x803c8460' 62 | elif (ver == "jp"): 63 | csv = "jp_symbols.csv" 64 | link_addr = '--link_address=0x805b4420' 65 | link_addr_overrides = '--link_address_overrides=jon:0x80c71420,tst:0x80c71420' 66 | rel_bss_addr = '--rel_bss_address=0x803c48e0' 67 | elif (ver == "pal"): 68 | csv = "eu_symbols.csv" 69 | link_addr = '--link_address=0x805fb8c0' 70 | link_addr_overrides = '--link_address_overrides=jon:0x80cb88c0,tst:0x80cb88c0' 71 | rel_bss_addr = '--rel_bss_address=0x803d44c0' 72 | 73 | try: 74 | import pandas as pd 75 | except ModuleNotFoundError: 76 | print("Pandas is not installed. Installing now...") 77 | subprocess.check_call(["pip", "install", "pandas"]) 78 | import pandas as pd 79 | 80 | dump_sections_args = [ 81 | 'python', 82 | str(Path(f"{ttyd_utils_src_dir}dump_sections.py")), 83 | f'--out_path={out_path}', 84 | f'--dol={dol_filepath}', 85 | f'--rel={rels_directory}/*.rel', 86 | f'{link_addr}', 87 | f'{link_addr_overrides}', 88 | f'{rel_bss_addr}' 89 | ] 90 | 91 | symbol_to_maps_args = [ 92 | 'python', 93 | str(Path(f"{ttyd_utils_src_dir}symbol_to_maps.py")), 94 | f'--out_path={out_path}', 95 | f'--symbols_path={ttyd_utils}/resources/{csv}', 96 | f'{rel_bss_addr}' 97 | ] 98 | 99 | export_events_args = [ 100 | 'python', 101 | str(Path(f"{ttyd_utils_src_dir}export_events.py")), 102 | f'--out_path={out_path}', 103 | f'--symbols_path={ttyd_utils}/resources/{csv}', 104 | f'--ttydasm_exe={ttydasm}' 105 | ] 106 | 107 | export_classes_args = [ 108 | 'python', 109 | str(Path(f"{ttyd_utils_src_dir}export_classes.py")), 110 | f'--out_path={out_path}', 111 | f'--symbols_path={ttyd_utils}/resources/{csv}', 112 | ] 113 | 114 | combine_event_dumps_args = [ 115 | 'python', 116 | str(Path(f"{ttyd_utils_src_dir}combine_event_dumps.py")), 117 | f'--out_path={out_path}', 118 | ] 119 | 120 | sort_events_by_prefix_args = [ 121 | 'python', 122 | str(Path(f"{ttyd_utils_src_dir}sort_events_by_prefix.py")), 123 | f"{out_path}/events", 124 | ] 125 | 126 | us_dol_md5 = '9800df7555210cb392e0cbc9f88488c5' 127 | us_rel_md5s = { 128 | "aaa": "f3ca3df5425a7eae7d65ecd43f9863b7", 129 | "aji": "2c123a7e426d334ffe4456ac94bff1d5", 130 | "bom": "7c54f5ee5eb30f24791ab3f969e93c56", 131 | "dmo": "eb07479364d1053cf63bf16ac6ae49fa", 132 | "dou": "8021d89909032192ec0c268d0119aa79", 133 | "eki": "95429d3a408e6b7423d6815a9de15f88", 134 | "end": "065a7f464a043d8253ffe69085a7355c", 135 | "gon": "d5e494ccdbfdb4b9e74959bed81512a4", 136 | "gor": "375b3ce1e1fa8ac865c94b828c10ea28", 137 | "gra": "6ca6acfec9993d3c3dabbed36d1a303c", 138 | "hei": "0c7a940e8a93908f73f85fd1faeec4ba", 139 | "hom": "4ab70b1215adeeba3e176dbbb2ddf6d2", 140 | "jin": "8b309402f80e1e4597f70089a70ae68c", 141 | "jon": "7440b6821e095093dd1cacc3d8cad045", 142 | "kpa": "75946e7686ce6f624bbdf1a60bec91d3", 143 | "las": "157527d44e48a7530ca79aec568eebaa", 144 | "moo": "e5cc511f622f31ea2ecfcf675844b624", 145 | "mri": "e3e8874fa43a72988c9c305493e07c42", 146 | "muj": "4a903ad264e44c5ef64f549899bcd649", 147 | "nok": "a44ee0df5d6fb9207ff70659cfdd9817", 148 | "pik": "6961bf10d8eeed70ffce58712551bd2f", 149 | "rsh": "88eaa0bc867e3b24724fcf22edcc60e5", 150 | "sys": "e5655488ff599c5cfc3f42fd30df7451", 151 | "tik": "ac17d353639983147231e2a0d6cf80b5", 152 | "tou": "d9ff80367fb5f9a43efe2f73a620ca54", 153 | "tou2": "ac7039fd4c377077ee0d69124fef3d89", 154 | "usu": "7607bb4ca39ed0576175cd6dce840095", 155 | "win": "6cc3cf0dc6d26787ea0e0a11cc3254d5", 156 | "yuu": "124e474373a7965e0e45fc845f9c9b3b" 157 | } 158 | 159 | pal_dol_md5 = '9ac2fd3b4d3d0efb00d9ee2563c7da24' 160 | pal_rel_md5s = { 161 | "aaa": "07a3a6cf5ff42e0a1453418cb0620f94", 162 | "aji": "c11f9cc01ef57110615ed84a50cf7c72", 163 | "bom": "927581c71ac05a9c6e18ae09f30c08f2", 164 | "dig": "6dd7be05ae9d6e15c30896045432fe3e", 165 | "dmo": "c05cc4633721adc0205d5e718d182b64", 166 | "dou": "def7fa886824c1165c8dd97676ed643f", 167 | "eki": "ecc5774997dc52cebe98b3666267c377", 168 | "end": "9a6f892c08291b57331e4e4b3d048291", 169 | "gon": "326e4af6b775b3729dc205f2f32d5ace", 170 | "gor": "62c32160f3b90c715122c5cb30d262f7", 171 | "gra": "6f11fe4b0b701ace7c0a7f88b21264ea", 172 | "hei": "b467afbe1067142110c47e7692b899b9", 173 | "hom": "14c73e0c89599b56fe93a3d31e21b67b", 174 | "jin": "50628475c16bf438003e188ddfbc7856", 175 | "jon": "77eb3e11526600b9e11941a5d44705c5", 176 | "kpa": "420537465433c7b9f96485d75cb04679", 177 | "las": "f722a153d7b1f20f665e5e7d5946a3a8", 178 | "moo": "6c3b639ce286e2a12a0d644dd7611f0b", 179 | "mri": "b8d4c2785fe4448db6d4a7ffa21d2a5c", 180 | "muj": "40cc8087fa5dffba022d5e6812069cd5", 181 | "nok": "ccfcfd86ce75beffffb82fe4dc329a3e", 182 | "pik": "8f37ee4e040f760eb95043844cec6fde", 183 | "qiz": "9abc39cf495e93534fd663df404eda7c", 184 | "rsh": "a249851048681037c9e8cf92ea84c19f", 185 | "sys": "6b451e92553956f11390dfad5091f060", 186 | "tik": "7230b8ea8b36c1492519c7a80b7d0e36", 187 | "tou": "69845d6c8ee11435edb3e642fab54a79", 188 | "tou2": "7e030a82e2758edecfe7b8d60a3a9662", 189 | "tst": "8b631f373f1601ec3af00eb8f86fd825", 190 | "usu": "15d1a4c8e31666a6d01bcd2de59a5957", 191 | "win": "25ebdd4978ef4b39d751ccb3236cda2b", 192 | "yuu": "4db17ad0ac333e7523deddf9f2f8bac6" 193 | } 194 | 195 | jp_dol_md5 = '67b32111d4855254149542440ea24070' 196 | jp_rel_md5s = { 197 | "aaa": "408fc74378eb669b0c07c3d5f7ed3bf7", 198 | "aji": "fc18c58f9d3a03fd978265f64f9e6ca8", 199 | "bom": "0e0a507355e949aa08f1f7dfc47f7705", 200 | "dig": "079fa33ddde37627f9970c836302f74b", 201 | "dmo": "ee8362bf9a676184fb24c29b829f19b2", 202 | "dou": "ebd99038d9896c2c4528fabcdd40cae6", 203 | "eki": "4ca5127bdda82e8be8c14faa544e0397", 204 | "end": "6dd060a56c579169c7146df1e9a6821d", 205 | "gon": "8aeff996005dfd457e6852407a6f038e", 206 | "gor": "b5f78ea4e6a51d398ac5142c096bc8c2", 207 | "gra": "d454d744a49c21ae743ee2fab60d2ba5", 208 | "hei": "6c89b1abd62ab7a420608522b4b6facc", 209 | "hom": "646339544d80047b0fd572864052e405", 210 | "jin": "e9bdea6bcdeb06375ba439a571f24559", 211 | "jon": "0ddda07c5cf76e8ced8c0b704e5917c3", 212 | "kpa": "6df4cd281211026a2de8706509292468", 213 | "las": "284092810a870aff12666059fdf5eb9a", 214 | "moo": "f8462bd42e6f66bded476b2df3bb7a01", 215 | "mri": "a80e2dbed49582b91166faff069ff63b", 216 | "muj": "d62666c9053cbba81a4bc87740405723", 217 | "nok": "c9f68697351a68b46033cc68a78d1c7c", 218 | "pik": "0630bdf8b783bae0365f3023f453e55c", 219 | "qiz": "1e3fb8d804d39ed3225db7bfe525a515", 220 | "rsh": "a921cebbcd9160cb3f4b15c20249b78e", 221 | "sys": "17ca61a0a8690d5e6ad196b4f95ee4b9", 222 | "tik": "d5a0db39419005cb9d93ac07c5ca3d4f", 223 | "tou": "0b8f7cfc4ef173db0461539290080d94", 224 | "tou2": "a332196692e93ec8bfdd9d3fe9b3fcb1", 225 | "usu": "29b98c270ab0930f5683e8e198cd292a", 226 | "win": "4500720279c007fd200b568ff1e9ed1a", 227 | "yuu": "412e5d0bbcd5ee4c879250cec7929977" 228 | } 229 | 230 | if (ver == "us"): 231 | rel_md5s = us_rel_md5s 232 | dol_md5 = us_dol_md5 233 | elif (ver == "jp"): 234 | rel_md5s = jp_rel_md5s 235 | dol_md5 = jp_dol_md5 236 | elif (ver == "pal"): 237 | rel_md5s = pal_rel_md5s 238 | dol_md5 = pal_dol_md5 239 | 240 | failed_file_count = 0 241 | 242 | # Open the DOL and read its contents 243 | with open(dol_filepath, 'rb') as f: 244 | data = f.read() 245 | # Calculate the md5 hash of the file contents 246 | md5 = hashlib.md5(data).hexdigest() 247 | if md5 == dol_md5: 248 | print(f"main.dol - \033[1;32mSuccess:\033[0m {md5}") 249 | else: 250 | print(f"main.dol - \033[1;31mFail:\033[0m {md5}") 251 | print("main.dol md5 does not match") 252 | print("Press any key to exit...") 253 | input() 254 | sys.exit(1) 255 | 256 | # Loop through each rel file in the directory 257 | for rel, expected_md5 in rel_md5s.items(): 258 | # Create the full path to the rel file 259 | filepath = os.path.join(rels_directory, rel + '.rel') 260 | 261 | # Open the file and read its contents 262 | with open(filepath, 'rb') as f: 263 | data = f.read() 264 | 265 | # Calculate the md5 hash of the file contents 266 | md5 = hashlib.md5(data).hexdigest() 267 | 268 | # Compare the calculated md5 to the expected md5 for this rel file 269 | if md5 == expected_md5: 270 | print(f"{rel}.rel - \033[1;32mSuccess:\033[0m {md5}") 271 | else: 272 | print(f"{rel}.rel - \033[1;31mFail:\033[0m {md5}") 273 | failed_file_count += 1 274 | 275 | if failed_file_count != 0: 276 | print("A non-matching rel file was found") 277 | print("Press any key to exit...") 278 | input() 279 | sys.exit(1) 280 | 281 | 282 | # Use the os.makedirs() function to create the folder 283 | os.makedirs(out_path, exist_ok=True) 284 | 285 | # Use subprocess.run() to call the script and wait for it to finish 286 | print("\nRunning dump_sections...") 287 | completed_process = subprocess.run(dump_sections_args, check=True) 288 | if completed_process.returncode != 0: 289 | print(f"Error: the script exited with status {completed_process.returncode}") 290 | sys.exit(1) 291 | 292 | print("\nRunning symbol_to_maps...") 293 | completed_process = subprocess.run(symbol_to_maps_args, check=True) 294 | if completed_process.returncode != 0: 295 | print(f"Error: the script exited with status {completed_process.returncode}") 296 | sys.exit(1) 297 | 298 | print("\nRunning export_events...") 299 | completed_process = subprocess.run(export_events_args, check=True) 300 | if completed_process.returncode != 0: 301 | print(f"Error: the script exited with status {completed_process.returncode}") 302 | sys.exit(1) 303 | 304 | print("\nRunning export_classes...") 305 | completed_process = subprocess.run(export_classes_args, check=True) 306 | if completed_process.returncode != 0: 307 | print(f"Error: the script exited with status {completed_process.returncode}") 308 | sys.exit(1) 309 | 310 | print("\nRunning combine_event_dumps...") 311 | completed_process = subprocess.run(combine_event_dumps_args, check=True) 312 | if completed_process.returncode != 0: 313 | print(f"Error: the script exited with status {completed_process.returncode}") 314 | sys.exit(1) 315 | 316 | print("\nRunning sort_events_by_prefix...") 317 | completed_process = subprocess.run(sort_events_by_prefix_args, check=True) 318 | if completed_process.returncode != 0: 319 | print(f"Error: the script exited with status {completed_process.returncode}") 320 | sys.exit(1) 321 | -------------------------------------------------------------------------------- /source/sort_events_by_prefix.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import sys 4 | 5 | if len(sys.argv) < 2: 6 | print("Usage: python3 ./sort_events_by_prefix.py [directory]") 7 | sys.exit(1) 8 | 9 | path_to_files = os.path.abspath(sys.argv[1]) 10 | 11 | # Define your dictionary 12 | prefix_subdirs = { 13 | "_main_battle": [ 14 | "attack_audience", "audience_msg", "break_slot", "event_default", "event_subset", "item_data", 15 | "mario_marioAttackEvent", "mario", "seq_end", "stage_object" 16 | ], 17 | "_main_eff": [], 18 | "_main_evt": [ 19 | "bero", "damage_evt", "door", "kinopio", "lecture_evt", "map_evt", 20 | "memcard_evt", "mobj_evt", "mobj_kpa", "movefloor", "npc", "shop", "sub_evt", 21 | "sub_mail", 22 | ], 23 | "_main_mot": [], 24 | "_main_npc": [ 25 | "event_barriern", "event_basabasa", "event_bubble", "event_chorobon", "event_dokan2D", "event_dougassu", 26 | "event_enemy", "event_fall2D", "event_gesso2D", "event_hannya", "event_hbom", "event_hbross", 27 | "event_honenoko", "event_kamec", "event_kamec2", "event_karon", "event_killer", "event_kuriboo2D", 28 | "event_kuriboo", "event_mahoon", "event_met", "event_nokonoko", "event_npc", "event_pakkun", 29 | "event_pansy", "event_patakuri", "event_patamet", "event_piders", "event_patapata", "event_sambo", 30 | "event_sinemon", "event_sinnosuke", "event_teresa", "event_testenemynpc", "event_togedaruma", "event_togemet", 31 | "event_togenoko", "event_twinkling_pansy", "event_unk", "event_wanwan", "event_zako2D", "event_zakoM2D", 32 | "event_zakowiz", "event_basabasa2", "event_dokugassun", "event_honenoko2" 33 | ], 34 | "_main_party": [], 35 | "_main_sac" : [ 36 | "bakugame", "common_sac", "deka", "genki", "muki", "suki", 37 | "zubastar" 38 | ], 39 | "_main_seq" : [], 40 | "_main_unit" : [ 41 | "bomzou", "koura", "mario", "object_switch", "object_tree", "party_christine", 42 | "party_chuchurina", "party_clauda", "party_nokotarou", "party_sanders", "party_vivian", "party_yoshi", "system" 43 | ], 44 | "_main_win" : [], 45 | "aaa": ["aaa_00"], 46 | "aji": [ 47 | "aji_00", "aji_01", "aji_02", "aji_03", "aji_04", "aji_05", 48 | "aji_06", "aji_07","aji_08", "aji_09", "aji_10", "aji_11", 49 | "aji_12", "aji_13", "aji_14", "aji_15", "aji_16", "aji_17", 50 | "aji_18", "aji_19", "evt_shuryolight", "unit_barriern_z", "unit_barriern", "unit_boss_magnum", 51 | "unit_gundan_zako", "battle_database", 52 | ], 53 | "bom": [ 54 | "bom_00", "bom_01", "bom_02", "bom_03", "bom_04", "unit_bllizard", 55 | "unit_ice_pakkun", "unit_kuriboo", "battle_database" 56 | ], 57 | "dmo": ["dmo_00"], 58 | "dou": [ 59 | "dou_00", "dou_01", "dou_02", "dou_03", "dou_04", "dou_05", 60 | "dou_06", "dou_07", "dou_08", "dou_09", "dou_10", "dou_11", 61 | "dou_12", "dou_13", "dou_dou", "evt_lect", "unit_bubble", "unit_heavy_bom", 62 | "unit_hermos", "unit_killer", "unit_kuriboo", "unit_patamet", "battle_database" 63 | ], 64 | "eki": [ 65 | "eki_00", "eki_01", "eki_02", "eki_03", "eki_04", "eki_05", 66 | "eki_06", "unit_kuriboo", "unit_kurokumorn", "unit_patatogemet", "unit_sambo", "evt_lect" 67 | ], 68 | "end": [], 69 | "gon": [ 70 | "gon_00", "gon_01", "gon_02", "gon_03", "gon_04", "gon_05", 71 | "gon_06", "gon_07", "gon_08", "gon_09", "gon_10", "gon_11", 72 | "gon_12", "gon_13", "unit_boss_gonbaba", "evt_lect", "unit_honenoko", "unit_kuriboo", 73 | "unit_nokonoko", "unit_patakuri", "unit_patapata", "unit_red_honenoko", "unit_togekuri" 74 | ], 75 | "gor": [ 76 | "gor_00", "gor_01", "gor_02", "gor_03", "gor_04", "gor_10", 77 | "gor_11", "gor_12", "gor_irai", "unit_boss_kanbu1", "unit_gundan_zako", "unit_kuriboo", 78 | "unit_lecture_christine", "unit_lecture_frankli", "unit_monban", "unit_npc_christine", "evt_lect" 79 | ], 80 | "gra": [ 81 | "gra_00", "gra_01", "gra_02", "gra_03", "gra_04", "gra_05", 82 | "gra_06", "unit_faker_mario", "unit_hyper_kuriboo", "unit_hyper_patakuri", "unit_hyper_sinemon", "unit_kuriboo", 83 | "unit_pansy", "unit_twinkling_pansy", "unit_hyper_togekuri" 84 | ], 85 | "hei": [ 86 | "hei_00", "hei_01", "hei_02", "hei_03", "hei_04", "hei_05", 87 | "hei_06", "hei_07", "hei_08", "hei_09", "hei_10", "hei_11", 88 | "hei_12", "hei_13", "unit_chorobon", "unit_gold_chorobon", "unit_kuriboo", "unit_monochrome_sinemon", 89 | "unit_nokonoko", "unit_patakuri", "unit_patapata", "unit_sinemon", "unit_sinnosuke", "unit_togedaruma", 90 | "unit_togekuri", "evt_lect" 91 | ], 92 | "hom": ["hom_00", "hom_10", "hom_11", "hom_12"], 93 | "jin": [ 94 | "jin_00", "jin_01", "jin_02", "jin_03", "jin_04", "jin_05", 95 | "jin_06", "jin_07", "jin_08", "jin_09", "jin_10", "jin_11", 96 | "unit_atmic", "unit_basabasa", "unit_boss_rampell", "unit_faker_mario", "unit_gullible_christine", "unit_gullible_clauda", 97 | "unit_gullible_nokotarou", "unit_gullible_yoshi", "unit_met", "unit_teresa", "unit_togemet", "evt_kagemario" 98 | ], 99 | "jon": [ 100 | "jon_evt", "jon_gonbaba", "jon_iri_12", "jon_jon", "unit_badge_borodo", "unit_basabasa", "unit_bllizard", 101 | "unit_bomhei", "unit_borodo", "unit_boss_zonbaba", "unit_bubble", "unit_burst_wanwan", "unit_chorobon", 102 | "unit_churantalar_piders", "unit_churantalar_renzoku", "unit_churantalar", "unit_dark_keeper", "unit_dokugassun", "unit_flower_chorobon", 103 | "unit_giant_bomb", "unit_hannya", "unit_heavy_bom", "unit_hennya", "unit_hinnya", "unit_honenoko", 104 | "unit_hyper_jyugem", "unit_hyper_sinemon", "unit_hyper_togezo", "unit_ice_pakkun", "unit_jyugem", "unit_karon", 105 | "unit_mahorn", "unit_monochrome_kurokumorn", "unit_monochrome_sinemon", "unit_pakkun_flower", "unit_patamet", "unit_patatogemet", 106 | "unit_phantom", "unit_piders", "unit_purple_teresa", "unit_sambo_mummy", "unit_sambo", "unit_sinemon", 107 | "unit_super_mahorn", "unit_teresa", "unit_togenoko", "unit_togezo", "unit_twinkling_pansy", "unit_ura_noko", 108 | "unit_wanwan", "unit_yamitogedaruma", "unit_togekuri", "unit_yami_kuriboo", "unit_yami_noko", "unit_yami_pata", 109 | "unit_yami_patakuri", "unit_yami_togekuri", "jon" 110 | ], 111 | "kpa": [ 112 | "kpa_00", "kpa_01", "kpa_02", "kpa_03", "kpa_04", "kpa_05", 113 | "kpa_06", "kpa_07" 114 | ], 115 | "las": [ 116 | "las_00", "las_01", "las_02", "las_03", "las_04", "las_05", 117 | "las_06", "las_07", "las_08", "las_09", "las_10", "las_11", 118 | "las_12", "las_13", "las_14", "las_15", "las_16", "las_17", 119 | "las_18", "las_19", "las_20", "las_21", "las_22", "las_23", 120 | "las_24", "las_25", "las_26", "las_27", "las_28", "las_29", 121 | "las_30", "unit_basabasa", "unit_black_karon", "unit_boss_batten_leader", "unit_boss_batten_satellite", "unit_boss_black_peach", 122 | "unit_boss_bunbaba", "unit_boss_kamec", "unit_boss_koopa", "unit_boss_majolyne", "unit_boss_marilyn", "unit_boss_rampell", 123 | "unit_heavy_bom", "unit_honenoko", "unkt_karon", "unit_phantom", "unit_red_honenoko", "unit_super_killer", 124 | "unit_super_mahorn", "unit_wanwan", "battle_database", "evt_shuryolight", "unit_karon" 125 | ], 126 | "moo": [ 127 | "moo_00", "moo_01", "moo_02", "moo_03", "moo_04", "moo_05", "moo_06", "moo_07", "unit_barriern_z", "unit_hyper_sinemon", "unit_kuriboo", "unit_sinemon" 128 | ], 129 | "mri": [ 130 | "mri_00", "mri_01", "mri_02", "mri_03", "mri_04", "mri_05", 131 | "mri_06", "mri_07", "mri_08", "mri_09", "mri_10", "mri_11", 132 | "mri_12", "mri_13", "mri_14", "mri_15", "mri_16", "mri_17", 133 | "mri_18", "mri_19", "mri_20", "mri_evt", "mri_mri", "mri_puni", 134 | "unit_barriern", "unit_boss_kanbu2", "unit_boss_magnum", "unit_dokugassun", "unit_gundan_zako", "unit_kuriboo", 135 | "unit_monochrome_kurokumorn", "unit_monochrome_pakkun", "unit_pakkun_flower", "unit_piders", "battle_database", "evt_lect" 136 | ], 137 | "muj": [ 138 | "muj_00", "muj_01", "muj_02", "muj_03", "muj_04", "muj_05", 139 | "muj_06", "muj_07", "muj_08", "muj_09", "muj_10", "muj_11", 140 | "muj_12", "muj_20", "muj_21", "muj_battle_database", "muj_evt", "muj_korutesu", 141 | "muj_muj", "unit_boss_cortez", "unit_boss_gundan_zako_group1", "unit_boss_honeduka", "unit_boss_kanbu3", "unit_flower_chorobon", 142 | "unit_green_chorobon", "unit_boss_gundan_zako_group2", "unit_boss_gundan_zako_group3", "unit_boss_gundan_zako_magician", "unit_gundan_zako", "unit_hermos", 143 | "unit_kuriboo", "unit_pakkun_flower", "unit_poison_pakkun", "battle_database", "evt_lect" 144 | ], 145 | "nok": [ 146 | "nok_00", "nok_01", "nok_nokonoko", "unit_act_kinopio", "unit_act_mario", "unit_act_teresa", 147 | "unit_act_atmic", "unit_act_clauda", "unit_act_kinopiko" 148 | ], 149 | "pik": [ 150 | "pik_00", "pik_01", "pik_02", "pik_03", "pik_04", "unit_purple_teresa" 151 | ], 152 | "rsh": [ 153 | "rsh_00", "rsh_01", "rsh_02", "rsh_03", "rsh_04", "rsh_05", 154 | "rsh_06", "rsh_07", "rsh_08", "rsh_evt", "rsh_kami", "rsh_simi", 155 | "unit_boss_moamoa", "battle_database" 156 | ], 157 | "sys": [], 158 | "tik": [ 159 | "tik_00", "tik_01", "tik_02", "tik_03", "tik_04", "tik_05", 160 | "tik_06", "tik_07", "tik_08", "tik_09", "tik_10", "tik_11", 161 | "tik_12", "tik_13", "tik_14", "tik_15", "tik_16", "tik_17", 162 | "tik_18", "tik_19", "tik_20", "tik_21", "unit_boss_gesso", "unit_hammer_bros", 163 | "unit_hannya", "unit_hennya", "unit_hinnya", "unit_kamec", "unit_kuriboo", "unit_lecture_frankli", 164 | "unit_nokonoko", "unit_patakuri", "unit_patapata", "unit_togekuri", "unit_togenoko", "battle_database", 165 | "evt_lect" 166 | ], 167 | 168 | "tou2": [ 169 | "tou_03", "unit_basabasa", "unit_bomhei", "unit_boomerang_bros", "unit_borodo", "unit_boss_champion", 170 | "unit_boss_koopa", "unit_boss_macho", "unit_burst_wanwan", "unit_chorobon", "unit_chrimson_togemet", "unit_dark_keeper", 171 | "unit_fire_bros", "unit_flower_chorobon", "unit_green_chorobon", "unit_hammer_bros", "unit_hannya", "unit_hennya", 172 | "unit_hinnya", "unit_honenoko", "unit_hyper_jyugem", "unit_hyper_sinnosuke", "unit_hyper_togezo", "unit_iron_sinemon", 173 | "unit_iron_sinemon2", "unit_jyugem", "unit_kamec", "unit_monochrome_kurokumorn", "unit_monochrome_pakkun", "unit_nokonoko", 174 | "unit_patapata", "unit_piders", "unit_sambo", "unit_togedaruma", "unit_togezo", "unit_ura_noko", 175 | "unit_ura_pata", "unit_wanawana", "unit_crimson_togemet", "unit_kurikuri", "unit_togenoko" 176 | ], 177 | "tou": [ 178 | "tou_00", "tou_01", "tou_02", "tou_03", "tou_04", "tou_05", 179 | "tou_06", "tou_07", "tou_08", "tou_09", "tou_10", "tou_11", 180 | "tou_12", "tou_13", "tou_20", "tou_dummy", "tou_evt", "evt_lect" 181 | ], 182 | "tst": [], #only not blank on PAL 183 | "usu": [ 184 | "usu_00", "usu_01", "evt_lect", "evt_kagemario" 185 | ], 186 | "win": [ 187 | "win_00", "win_01", "win_02", "win_03", "win_04", "win_05", 188 | "win_06", "win_evt", "unit_boss_majolyne", "unit_boss_marilyn", "unit_boss_vivian", "unit_dokugassun", 189 | "unit_gundan_zako", "unit_kuriboo", "unit_monochrome_kurokumorn", "unit_monochrome_pakkun", "unit_monochrome_sinemon", "unit_pakkun_flower", 190 | "win_win", "evt_lect" 191 | ], 192 | "yuu": [ 193 | "yuu_00", "yuu_01", "yuu_02", "yuu_03", "evt_yuuminigame", "evt_yuunpc" 194 | ] 195 | } 196 | 197 | # Create directories for keys and elements 198 | for key, elements in prefix_subdirs.items(): 199 | key_path = os.path.join(path_to_files, key) 200 | os.makedirs(key_path, exist_ok=True) 201 | 202 | for element in elements: 203 | element_path = os.path.join(key_path, element) 204 | os.makedirs(element_path, exist_ok=True) 205 | 206 | # Sort files into appropriate folders 207 | for filename in os.listdir(path_to_files): 208 | file_path = os.path.join(path_to_files, filename) 209 | 210 | if os.path.isfile(file_path): 211 | moved = False 212 | for key, elements in prefix_subdirs.items(): 213 | if moved: 214 | break 215 | 216 | if filename.startswith(key): 217 | for element in elements: 218 | if filename.startswith(f"{key}_{element}_"): 219 | destination_path = os.path.join(path_to_files, key, element, filename) 220 | shutil.move(file_path, destination_path) 221 | moved = True 222 | break 223 | 224 | if not moved: 225 | destination_path = os.path.join(path_to_files, key, filename) 226 | shutil.move(file_path, destination_path) 227 | moved = True 228 | -------------------------------------------------------------------------------- /source/symbol_to_maps.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3.6 2 | 3 | """Creates symbol maps and ttydasm maps from a csv of symbol info. 4 | 5 | This program takes a .csv of symbol information, and dumps the following: 6 | - .MAP files for the .DOL and each .REL on its own 7 | - Combined .MAP files per .REL area that include the .DOL symbols, 8 | and the .REL symbols at their linked addresses. 9 | - TTYDASM symbol map files per area (using linked addresses); will use the names 10 | of symbols if provided, or the literal value of symbols marked as strings.""" 11 | # Jonathan Aldrich 2021-02-09 ~ 2021-03-02 12 | 13 | import codecs 14 | import os 15 | import sys 16 | import numpy as np 17 | import pandas as pd 18 | from collections import defaultdict 19 | from pathlib import Path 20 | 21 | import jdalibpy.flags as flags 22 | 23 | FLAGS = flags.Flags() 24 | 25 | # Output directory (will create "maps" and "ttydasm" dirs underneath). 26 | FLAGS.DefineString("out_path", "") 27 | # Input symbols file. 28 | FLAGS.DefineString("symbols_path", "") 29 | # Rel bss location (if not provided, will not output REL .bss to combined maps.) 30 | FLAGS.DefineInt("rel_bss_address") 31 | 32 | # Whether to display debug strings. 33 | FLAGS.DefineInt("debug_level", 1) 34 | 35 | class SymbolToMapError(Exception): 36 | def __init__(self, message=""): 37 | self.message = message 38 | 39 | def _GetOutputPath(path, create_parent=True): 40 | if create_parent and not os.path.exists(os.path.dirname(path)): 41 | os.makedirs(os.path.dirname(path)) 42 | return path 43 | 44 | def _LoadSectionRamAddrDict(in_path, section_info): 45 | """Constructs a dict of REL/section : ram base address.""" 46 | if FLAGS.GetFlag("debug_level"): 47 | print("Loading section ram addresses...") 48 | 49 | section_data = {} 50 | for sec_id in (0, 1, 7, 8, 9, 10, 11, 12, 100, 101, 102): 51 | ram_addr = int(section_info.loc[("_main", sec_id)]["ram_start"], 16) 52 | section_data["_main-%02d" % sec_id] = ram_addr 53 | 54 | rels_dir = in_path / "sections/rel_linked" 55 | areas = [f.name for f in os.scandir(rels_dir) if f.is_dir()] 56 | for area in areas: 57 | for sec_id in range(1, 6): 58 | ram_addr = int(section_info.loc[(area, sec_id)]["ram_start"], 16) 59 | section_data["%s-%02d" % (area, sec_id)] = ram_addr 60 | return section_data 61 | 62 | def _GetMapLines(symbols, section_addrs): 63 | """Returns a dict of area/sec name : list of lines for the map file.""" 64 | res = defaultdict(list) 65 | rel_bss_addr = FLAGS.GetFlag("rel_bss_address") 66 | if FLAGS.GetFlag("debug_level"): 67 | print("Processing map lines...") 68 | for (index, row) in symbols.iterrows(): 69 | if FLAGS.GetFlag("debug_level") and not index % 1000: 70 | print("Processing line %d..." % index) 71 | if row["area"] == "_main": 72 | text = " %08x %08x %08x %2d %s %s\n" % ( 73 | int(row["sec_offset"], 16), int(row["size"], 16), 74 | int(row["ram_addr"], 16), int(row["align"]), 75 | row["name"], row["namespace"]) 76 | else: 77 | if row["sec_type"] == "bss": 78 | if not rel_bss_addr: 79 | continue 80 | ram_addr = rel_bss_addr + int(row["sec_offset"], 16) 81 | else: 82 | ram_addr = section_addrs[ 83 | "%s-%02d" % (row["area"], row["sec_id"]) 84 | ] + int(row["sec_offset"], 16) 85 | text = " %08x %08x %08x %2d %s %s\n" % ( 86 | int(row["sec_offset"], 16), int(row["size"], 16), 87 | int(row["sec_offset"], 16), int(row["align"]), 88 | row["name"], row["namespace"]) 89 | combined_text = " %08x %08x %08x %2d %s %s\n" % ( 90 | int(row["sec_offset"], 16), int(row["size"], 16), 91 | ram_addr, int(row["align"]), 92 | row["name"], row["namespace"]) 93 | res[(row["area"] + "_all", row["sec_name"])].append(combined_text) 94 | res[(row["area"], row["sec_name"])].append(text) 95 | return res 96 | 97 | def _CreateMapFiles(out_path, map_lines, area, section_info): 98 | # Create a .MAP file with only symbols from the given area. 99 | lines = [] 100 | sections = [ 101 | ".init", ".text", ".ctors", ".dtors", ".rodata", ".data", ".bss", 102 | ".sdata", ".sbss", ".sdata2", ".sbss2"] 103 | for section in sections: 104 | if (area, section) in map_lines: 105 | lines.append("%s section layout\n" % section) 106 | lines.append(" Starting Virtual\n") 107 | lines.append(" address Size address\n") 108 | lines.append(" -------------------------\n") 109 | for line in map_lines[(area, section)]: 110 | lines.append(line) 111 | lines.append("\n") 112 | f = codecs.open( 113 | _GetOutputPath(out_path / "maps" / (area + ".map")), "w", "utf-8") 114 | for line in lines: 115 | f.write(line) 116 | 117 | # Add memory map section to bottom of file. 118 | f.write("\nMemory map:\n"); 119 | f.write(" Starting Size File\n"); 120 | f.write(" address Offset\n"); 121 | ram_addr = 0 122 | file_addr = 0 123 | size = 0 124 | for section in sections: 125 | if (area, section) in section_info.index: 126 | info = section_info.loc[(area, section)] 127 | if area == "_main": 128 | # If not main DOL, RAM addresses should stay at 0. 129 | ram_addr = int(info["ram_start"], 16) 130 | if info["type"] == "bss": 131 | # Compute .bss 'file offset' based on last section's endpoint, 132 | # as .bss sections are by definition not stored in the file. 133 | align = 32 if area == "_main" else 8 134 | file_addr += size + align - 1 135 | file_addr -= (file_addr % align) 136 | else: 137 | file_addr = int(info["file_start"], 16) 138 | size = int(info["size"],16) 139 | f.write( 140 | "%17s %08x %08x %08x\n" % (section, ram_addr, size, file_addr)) 141 | f.write("\n") 142 | 143 | # For RELs, also export .MAP files with .dol and .rel symbols included. 144 | if area == "_main": 145 | return 146 | lines = [] 147 | for section in sections: 148 | lines.append("%s section layout\n" % section) 149 | lines.append(" Starting Virtual\n") 150 | lines.append(" address Size address\n") 151 | lines.append(" -------------------------\n") 152 | for line in map_lines[("_main", section)]: 153 | if "rel_bss" not in line or not FLAGS.GetFlag("rel_bss_address"): 154 | lines.append(line) 155 | for line in map_lines[(area + "_all", section)]: 156 | lines.append(line) 157 | lines.append("\n") 158 | f = codecs.open( 159 | _GetOutputPath(out_path / "maps" / (area + "_all.map")), "w", "utf-8") 160 | for line in lines: 161 | f.write(line) 162 | 163 | def _GetTtydasmLines(symbols, section_addrs): 164 | """Returns a dict of area : list of lines for the ttydasm symbols file.""" 165 | res = defaultdict(list) 166 | if FLAGS.GetFlag("debug_level"): 167 | print("\nProcessing ttydasm lines...") 168 | for (index, row) in symbols.iterrows(): 169 | if FLAGS.GetFlag("debug_level") and not index % 1000: 170 | print("Processing line %d..." % index) 171 | if row["area"] == "_main" or row["sec_name"] != ".bss": 172 | if ((row["name"][:4] == "str_" or row["name"][0] == "@") 173 | and row["type"] == "string"): 174 | # Use actual string in quotes, rather than name/namespace. 175 | name = '"%s"' % row["value"] 176 | else: 177 | name = "%s %s" % (row["name"], row["namespace"]) 178 | ram_addr = section_addrs[ 179 | "%s-%02d" % (row["area"], row["sec_id"]) 180 | ] + int(row["sec_offset"], 16) 181 | text = "%08X:%s\n" % (ram_addr, name) 182 | res[(row["area"], row["sec_name"])].append(text) 183 | return res 184 | 185 | def _CreateTtydasmFile(out_path, ttydasm_lines, area): 186 | lines = [] 187 | for key in ttydasm_lines: 188 | if key[0] == area or key[0] == "_main": 189 | lines += ttydasm_lines[key] 190 | f = codecs.open( 191 | _GetOutputPath(out_path / "ttydasm" / (area + ".sym")), "w", "utf-8") 192 | for line in sorted(lines): 193 | f.write(line) 194 | 195 | def main(argc, argv): 196 | in_path = FLAGS.GetFlag("symbols_path") 197 | if not os.path.exists(in_path): 198 | raise SymbolToMapError( 199 | "--symbols_path must be set to a valid CSV of symbols.") 200 | in_path = Path(in_path) 201 | 202 | out_path = FLAGS.GetFlag("out_path") 203 | if not out_path: 204 | raise SymbolToMapError("Must set an --out_path.") 205 | out_path = Path(out_path) 206 | 207 | if not os.path.exists(out_path / "section_info.csv"): 208 | raise SymbolToMapError( 209 | "You must first run dump_sections.py using the same --out_path.") 210 | section_info = pd.read_csv(out_path / "section_info.csv") 211 | section_info = section_info.set_index(["area", "id"]) 212 | 213 | symbols = pd.read_csv(in_path) 214 | section_addrs = _LoadSectionRamAddrDict(out_path, section_info) 215 | map_lines = _GetMapLines(symbols, section_addrs) 216 | ttydasm_lines = _GetTtydasmLines(symbols, section_addrs) 217 | 218 | section_info = section_info.reset_index() 219 | section_info = section_info.set_index(["area", "name"]) 220 | 221 | for area in symbols.area.unique(): 222 | _CreateMapFiles(out_path, map_lines, area, section_info) 223 | _CreateTtydasmFile(out_path, ttydasm_lines, area) 224 | 225 | if __name__ == "__main__": 226 | (argc, argv) = FLAGS.ParseFlags(sys.argv[1:]) 227 | main(argc, argv) 228 | --------------------------------------------------------------------------------