├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── README.md ├── UNLICENSE ├── asm_unpackers ├── .gitignore ├── Makefile ├── test_data.bin ├── test_data.upk ├── unpack_arm32.S ├── unpack_armv6m.S ├── unpack_jagrisc.js ├── unpack_jagrisc_fast.js └── unpack_riscv.S ├── c_library ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── Makefile ├── Readme.md ├── src │ └── lib.rs ├── upkr.c └── upkr.h ├── c_unpacker ├── .gitignore ├── Makefile ├── decode_bit_alt.c ├── main.c ├── readme.txt └── unpack.c ├── dos_unpacker ├── readme.txt ├── unpack_x86_16_DOS.asm ├── unpack_x86_16_DOS_no_relocation.asm └── unpack_x86_16_DOS_no_repeated_offset.asm ├── fuzz ├── .gitignore ├── Cargo.lock ├── Cargo.toml └── fuzz_targets │ ├── all_configs.rs │ └── unpack.rs ├── release ├── .gitignore └── Makefile ├── src ├── context_state.rs ├── greedy_packer.rs ├── heatmap.rs ├── lib.rs ├── lz.rs ├── main.rs ├── match_finder.rs ├── parsing_packer.rs └── rans.rs └── z80_unpacker ├── .gitignore ├── Makefile ├── example ├── example.asm ├── example.sna ├── screens.reversed │ ├── Grongy - ZX Spectrum (2022).scr.upk │ ├── Schafft - Poison (2017).scr.upk │ ├── diver - Back to Bjork (2015).scr.upk │ └── diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr.upk └── screens │ ├── Grongy - ZX Spectrum (2022).scr │ ├── Grongy - ZX Spectrum (2022).scr.upk │ ├── Schafft - Poison (2017).scr │ ├── Schafft - Poison (2017).scr.upk │ ├── diver - Back to Bjork (2015).scr │ ├── diver - Back to Bjork (2015).scr.upk │ ├── diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr │ └── diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr.upk ├── readme.txt └── unpack.asm /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 4 4 | 5 | [[package]] 6 | name = "anyhow" 7 | version = "1.0.98" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" 10 | 11 | [[package]] 12 | name = "autocfg" 13 | version = "1.5.0" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" 16 | 17 | [[package]] 18 | name = "bitflags" 19 | version = "2.9.1" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" 22 | 23 | [[package]] 24 | name = "cc" 25 | version = "1.2.29" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362" 28 | dependencies = [ 29 | "shlex", 30 | ] 31 | 32 | [[package]] 33 | name = "cdivsufsort" 34 | version = "2.0.0" 35 | source = "registry+https://github.com/rust-lang/crates.io-index" 36 | checksum = "edefce019197609da416762da75bb000bbd2224b2d89a7e722c2296cbff79b8c" 37 | dependencies = [ 38 | "cc", 39 | "sacabase", 40 | ] 41 | 42 | [[package]] 43 | name = "cfg-if" 44 | version = "1.0.1" 45 | source = "registry+https://github.com/rust-lang/crates.io-index" 46 | checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" 47 | 48 | [[package]] 49 | name = "crossbeam-channel" 50 | version = "0.5.15" 51 | source = "registry+https://github.com/rust-lang/crates.io-index" 52 | checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" 53 | dependencies = [ 54 | "crossbeam-utils", 55 | ] 56 | 57 | [[package]] 58 | name = "crossbeam-utils" 59 | version = "0.8.21" 60 | source = "registry+https://github.com/rust-lang/crates.io-index" 61 | checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" 62 | 63 | [[package]] 64 | name = "crossterm" 65 | version = "0.29.0" 66 | source = "registry+https://github.com/rust-lang/crates.io-index" 67 | checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" 68 | dependencies = [ 69 | "bitflags", 70 | "document-features", 71 | "parking_lot", 72 | "rustix", 73 | ] 74 | 75 | [[package]] 76 | name = "document-features" 77 | version = "0.2.11" 78 | source = "registry+https://github.com/rust-lang/crates.io-index" 79 | checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" 80 | dependencies = [ 81 | "litrs", 82 | ] 83 | 84 | [[package]] 85 | name = "errno" 86 | version = "0.3.13" 87 | source = "registry+https://github.com/rust-lang/crates.io-index" 88 | checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" 89 | dependencies = [ 90 | "libc", 91 | "windows-sys", 92 | ] 93 | 94 | [[package]] 95 | name = "lexopt" 96 | version = "0.3.1" 97 | source = "registry+https://github.com/rust-lang/crates.io-index" 98 | checksum = "9fa0e2a1fcbe2f6be6c42e342259976206b383122fc152e872795338b5a3f3a7" 99 | 100 | [[package]] 101 | name = "libc" 102 | version = "0.2.174" 103 | source = "registry+https://github.com/rust-lang/crates.io-index" 104 | checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" 105 | 106 | [[package]] 107 | name = "linux-raw-sys" 108 | version = "0.9.4" 109 | source = "registry+https://github.com/rust-lang/crates.io-index" 110 | checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" 111 | 112 | [[package]] 113 | name = "litrs" 114 | version = "0.4.1" 115 | source = "registry+https://github.com/rust-lang/crates.io-index" 116 | checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" 117 | 118 | [[package]] 119 | name = "lock_api" 120 | version = "0.4.13" 121 | source = "registry+https://github.com/rust-lang/crates.io-index" 122 | checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" 123 | dependencies = [ 124 | "autocfg", 125 | "scopeguard", 126 | ] 127 | 128 | [[package]] 129 | name = "num-traits" 130 | version = "0.2.19" 131 | source = "registry+https://github.com/rust-lang/crates.io-index" 132 | checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" 133 | dependencies = [ 134 | "autocfg", 135 | ] 136 | 137 | [[package]] 138 | name = "parking_lot" 139 | version = "0.12.4" 140 | source = "registry+https://github.com/rust-lang/crates.io-index" 141 | checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" 142 | dependencies = [ 143 | "lock_api", 144 | "parking_lot_core", 145 | ] 146 | 147 | [[package]] 148 | name = "parking_lot_core" 149 | version = "0.9.11" 150 | source = "registry+https://github.com/rust-lang/crates.io-index" 151 | checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" 152 | dependencies = [ 153 | "cfg-if", 154 | "libc", 155 | "redox_syscall", 156 | "smallvec", 157 | "windows-targets 0.52.6", 158 | ] 159 | 160 | [[package]] 161 | name = "pbr" 162 | version = "1.1.1" 163 | source = "registry+https://github.com/rust-lang/crates.io-index" 164 | checksum = "ed5827dfa0d69b6c92493d6c38e633bbaa5937c153d0d7c28bf12313f8c6d514" 165 | dependencies = [ 166 | "crossbeam-channel", 167 | "libc", 168 | "winapi", 169 | ] 170 | 171 | [[package]] 172 | name = "proc-macro2" 173 | version = "1.0.95" 174 | source = "registry+https://github.com/rust-lang/crates.io-index" 175 | checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" 176 | dependencies = [ 177 | "unicode-ident", 178 | ] 179 | 180 | [[package]] 181 | name = "quote" 182 | version = "1.0.40" 183 | source = "registry+https://github.com/rust-lang/crates.io-index" 184 | checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" 185 | dependencies = [ 186 | "proc-macro2", 187 | ] 188 | 189 | [[package]] 190 | name = "redox_syscall" 191 | version = "0.5.13" 192 | source = "registry+https://github.com/rust-lang/crates.io-index" 193 | checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" 194 | dependencies = [ 195 | "bitflags", 196 | ] 197 | 198 | [[package]] 199 | name = "rustix" 200 | version = "1.0.8" 201 | source = "registry+https://github.com/rust-lang/crates.io-index" 202 | checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" 203 | dependencies = [ 204 | "bitflags", 205 | "errno", 206 | "libc", 207 | "linux-raw-sys", 208 | "windows-sys", 209 | ] 210 | 211 | [[package]] 212 | name = "sacabase" 213 | version = "2.0.0" 214 | source = "registry+https://github.com/rust-lang/crates.io-index" 215 | checksum = "9883fc3d6ce3d78bb54d908602f8bc1f7b5f983afe601dabe083009d86267a84" 216 | dependencies = [ 217 | "num-traits", 218 | ] 219 | 220 | [[package]] 221 | name = "scopeguard" 222 | version = "1.2.0" 223 | source = "registry+https://github.com/rust-lang/crates.io-index" 224 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 225 | 226 | [[package]] 227 | name = "shlex" 228 | version = "1.3.0" 229 | source = "registry+https://github.com/rust-lang/crates.io-index" 230 | checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" 231 | 232 | [[package]] 233 | name = "smallvec" 234 | version = "1.15.1" 235 | source = "registry+https://github.com/rust-lang/crates.io-index" 236 | checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" 237 | 238 | [[package]] 239 | name = "syn" 240 | version = "2.0.104" 241 | source = "registry+https://github.com/rust-lang/crates.io-index" 242 | checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" 243 | dependencies = [ 244 | "proc-macro2", 245 | "quote", 246 | "unicode-ident", 247 | ] 248 | 249 | [[package]] 250 | name = "thiserror" 251 | version = "2.0.12" 252 | source = "registry+https://github.com/rust-lang/crates.io-index" 253 | checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" 254 | dependencies = [ 255 | "thiserror-impl", 256 | ] 257 | 258 | [[package]] 259 | name = "thiserror-impl" 260 | version = "2.0.12" 261 | source = "registry+https://github.com/rust-lang/crates.io-index" 262 | checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" 263 | dependencies = [ 264 | "proc-macro2", 265 | "quote", 266 | "syn", 267 | ] 268 | 269 | [[package]] 270 | name = "unicode-ident" 271 | version = "1.0.18" 272 | source = "registry+https://github.com/rust-lang/crates.io-index" 273 | checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" 274 | 275 | [[package]] 276 | name = "upkr" 277 | version = "0.2.3" 278 | dependencies = [ 279 | "anyhow", 280 | "cdivsufsort", 281 | "crossterm", 282 | "lexopt", 283 | "pbr", 284 | "thiserror", 285 | ] 286 | 287 | [[package]] 288 | name = "winapi" 289 | version = "0.3.9" 290 | source = "registry+https://github.com/rust-lang/crates.io-index" 291 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 292 | dependencies = [ 293 | "winapi-i686-pc-windows-gnu", 294 | "winapi-x86_64-pc-windows-gnu", 295 | ] 296 | 297 | [[package]] 298 | name = "winapi-i686-pc-windows-gnu" 299 | version = "0.4.0" 300 | source = "registry+https://github.com/rust-lang/crates.io-index" 301 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 302 | 303 | [[package]] 304 | name = "winapi-x86_64-pc-windows-gnu" 305 | version = "0.4.0" 306 | source = "registry+https://github.com/rust-lang/crates.io-index" 307 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 308 | 309 | [[package]] 310 | name = "windows-sys" 311 | version = "0.60.2" 312 | source = "registry+https://github.com/rust-lang/crates.io-index" 313 | checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" 314 | dependencies = [ 315 | "windows-targets 0.53.2", 316 | ] 317 | 318 | [[package]] 319 | name = "windows-targets" 320 | version = "0.52.6" 321 | source = "registry+https://github.com/rust-lang/crates.io-index" 322 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 323 | dependencies = [ 324 | "windows_aarch64_gnullvm 0.52.6", 325 | "windows_aarch64_msvc 0.52.6", 326 | "windows_i686_gnu 0.52.6", 327 | "windows_i686_gnullvm 0.52.6", 328 | "windows_i686_msvc 0.52.6", 329 | "windows_x86_64_gnu 0.52.6", 330 | "windows_x86_64_gnullvm 0.52.6", 331 | "windows_x86_64_msvc 0.52.6", 332 | ] 333 | 334 | [[package]] 335 | name = "windows-targets" 336 | version = "0.53.2" 337 | source = "registry+https://github.com/rust-lang/crates.io-index" 338 | checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" 339 | dependencies = [ 340 | "windows_aarch64_gnullvm 0.53.0", 341 | "windows_aarch64_msvc 0.53.0", 342 | "windows_i686_gnu 0.53.0", 343 | "windows_i686_gnullvm 0.53.0", 344 | "windows_i686_msvc 0.53.0", 345 | "windows_x86_64_gnu 0.53.0", 346 | "windows_x86_64_gnullvm 0.53.0", 347 | "windows_x86_64_msvc 0.53.0", 348 | ] 349 | 350 | [[package]] 351 | name = "windows_aarch64_gnullvm" 352 | version = "0.52.6" 353 | source = "registry+https://github.com/rust-lang/crates.io-index" 354 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 355 | 356 | [[package]] 357 | name = "windows_aarch64_gnullvm" 358 | version = "0.53.0" 359 | source = "registry+https://github.com/rust-lang/crates.io-index" 360 | checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" 361 | 362 | [[package]] 363 | name = "windows_aarch64_msvc" 364 | version = "0.52.6" 365 | source = "registry+https://github.com/rust-lang/crates.io-index" 366 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 367 | 368 | [[package]] 369 | name = "windows_aarch64_msvc" 370 | version = "0.53.0" 371 | source = "registry+https://github.com/rust-lang/crates.io-index" 372 | checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" 373 | 374 | [[package]] 375 | name = "windows_i686_gnu" 376 | version = "0.52.6" 377 | source = "registry+https://github.com/rust-lang/crates.io-index" 378 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 379 | 380 | [[package]] 381 | name = "windows_i686_gnu" 382 | version = "0.53.0" 383 | source = "registry+https://github.com/rust-lang/crates.io-index" 384 | checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" 385 | 386 | [[package]] 387 | name = "windows_i686_gnullvm" 388 | version = "0.52.6" 389 | source = "registry+https://github.com/rust-lang/crates.io-index" 390 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 391 | 392 | [[package]] 393 | name = "windows_i686_gnullvm" 394 | version = "0.53.0" 395 | source = "registry+https://github.com/rust-lang/crates.io-index" 396 | checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" 397 | 398 | [[package]] 399 | name = "windows_i686_msvc" 400 | version = "0.52.6" 401 | source = "registry+https://github.com/rust-lang/crates.io-index" 402 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 403 | 404 | [[package]] 405 | name = "windows_i686_msvc" 406 | version = "0.53.0" 407 | source = "registry+https://github.com/rust-lang/crates.io-index" 408 | checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" 409 | 410 | [[package]] 411 | name = "windows_x86_64_gnu" 412 | version = "0.52.6" 413 | source = "registry+https://github.com/rust-lang/crates.io-index" 414 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 415 | 416 | [[package]] 417 | name = "windows_x86_64_gnu" 418 | version = "0.53.0" 419 | source = "registry+https://github.com/rust-lang/crates.io-index" 420 | checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" 421 | 422 | [[package]] 423 | name = "windows_x86_64_gnullvm" 424 | version = "0.52.6" 425 | source = "registry+https://github.com/rust-lang/crates.io-index" 426 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 427 | 428 | [[package]] 429 | name = "windows_x86_64_gnullvm" 430 | version = "0.53.0" 431 | source = "registry+https://github.com/rust-lang/crates.io-index" 432 | checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" 433 | 434 | [[package]] 435 | name = "windows_x86_64_msvc" 436 | version = "0.52.6" 437 | source = "registry+https://github.com/rust-lang/crates.io-index" 438 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 439 | 440 | [[package]] 441 | name = "windows_x86_64_msvc" 442 | version = "0.53.0" 443 | source = "registry+https://github.com/rust-lang/crates.io-index" 444 | checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" 445 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "upkr" 3 | version = "0.2.3" 4 | edition = "2024" 5 | description = "Simple LZ packer with relatively small unpackers" 6 | license = "Unlicense" 7 | reepository = "https://github.com/exoticorn/upkr" 8 | 9 | [profile.release] 10 | strip = "debuginfo" 11 | 12 | [features] 13 | terminal = ["crossterm", "pbr"] 14 | 15 | [dependencies] 16 | cdivsufsort = "2" 17 | lexopt = "0.3.1" 18 | anyhow = "1" 19 | thiserror = "2.0.12" 20 | pbr = { version = "1", optional = true } 21 | crossterm = { version = "0.29.0", default-features = false, optional = true } 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Upkr 2 | 3 | Upkr is a simple general purpose lz packer designed to be used in the [MicroW8](https://github.com/exoticorn/microw8) platform. 4 | The compressed format is losely based on [Shrinkler](https://github.com/askeksa/Shrinkler) with the main difference being that 5 | Upkr doesn't differentiate between literals at odd or even addresses (by default) and that I went with rANS/rABS instead of a range coder. 6 | 7 | Compression rate is on par with Shrinkler. 8 | 9 | The differences compare to Shrinkler also makes it interesting on 8bit platforms. The z80 unpacker included in the release 10 | is both about twice as fast and smaller than the Shrinkler unpacker. 11 | 12 | ## Inspirations: 13 | 14 | * Ferris' blog about his [C64 intro packer](https://yupferris.github.io/blog/2020/08/31/c64-4k-intro-packer-deep-dive.html) 15 | * [Shrinkler](https://github.com/askeksa/Shrinkler) 16 | * Ryg's [sample rANS implementation](https://github.com/rygorous/ryg_rans) 17 | 18 | ## Unpackers 19 | 20 | The release includes a reference c unpacker, as well as some optimized asm unpackers (arm and riscv). The unpckers in 21 | c_unpacker and asm_unpackers unpack the default upkr compressed format. The z80_unpacker 22 | is based on some variations to the compressed format. (Use `upkr --z80` to select those variations.) 23 | The 16 bit dos unpacker also uses some variations. (`upkr --x86`) 24 | 25 | ### More unpackers outside this repository 26 | 27 | * [Atari Lynx](https://github.com/42Bastian/new_bll/blob/master/demos/depacker/unupkr.asm) 28 | * [Atari Jaguar](https://github.com/42Bastian/new_bjl/blob/main/exp/depacker/unupkr.js) 29 | * [8080, R800](https://github.com/ivagorRetrocomp/DeUpkr) 30 | * [6502](https://github.com/pfusik/upkr6502) 31 | 32 | ## Usage 33 | 34 | ``` 35 | upkr [-l level(0-9)] [config options] [] 36 | upkr -u [config options] [] 37 | upkr --heatmap [config options] [] 38 | upkr --margin [config options] 39 | 40 | -l, --level N compression level 0-9 41 | -0, ..., -9 short form for setting compression level 42 | -d, --decompress decompress infile 43 | --heatmap calculate heatmap from compressed file 44 | --raw-cost report raw cost of literals in heatmap 45 | (the cost of literals is spread across all matches 46 | that reference the literal by default.) 47 | --hexdump print heatmap as colored hexdump 48 | --margin calculate margin for overlapped unpacking of a packed file 49 | 50 | When no infile is given, or the infile is '-', read from stdin. 51 | When no outfile is given and reading from stdin, or when outfile is '-', write to stdout. 52 | 53 | Config presets for specific unpackers: 54 | --z80 --big-endian-bitstream --invert-bit-encoding --simplified-prob-update -9 55 | --x86 --bitstream --invert-is-match-bit --invert-continue-value-bit --invert-new-offset-bit 56 | --x86b --bitstream --invert-continue-value-bit --no-repeated-offsets -9 57 | 58 | Config options (need to match when packing/unpacking): 59 | -b, --bitstream bitstream mode 60 | -p, --parity N use N (2/4) parity contexts 61 | -r, --reverse reverse input & output 62 | 63 | Config options to tailor output to specific optimized unpackers: 64 | --invert-is-match-bit 65 | --invert-new-offset-bit 66 | --invert-continue-value-bit 67 | --invert-bit-encoding 68 | --simplified-prob-update 69 | --big-endian-bitstream (implies --bitstream) 70 | --no-repeated-offsets 71 | --eof-in-length 72 | --max-offset N 73 | --max-length N 74 | ``` 75 | 76 | ## Heatmap 77 | 78 | By default, the `--heatmap` flag writes out the heatmap data as a binary file. The heatmap file is 79 | the same size as the unpacked data. Each byte can be interpreted like this: 80 | 81 | ``` 82 | is_literal = byte & 1; // whether the byte was encoded as a literal (as opposed to a match) 83 | size_in_bits = 2.0 ** (((byte >> 1) - 64) / 8.0); // the size this byte takes up in the compressed data 84 | ``` 85 | -------------------------------------------------------------------------------- /UNLICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to -------------------------------------------------------------------------------- /asm_unpackers/.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | -------------------------------------------------------------------------------- /asm_unpackers/Makefile: -------------------------------------------------------------------------------- 1 | build/unpack_riscv64: ../c_unpacker/main.c unpack_riscv.S 2 | mkdir -p build 3 | riscv64-linux-gnu-gcc -g -static -o $@ $^ 4 | 5 | test_riscv64: build/unpack_riscv64 6 | qemu-riscv64 $< test_data.upk /tmp/out.bin 7 | cmp test_data.bin /tmp/out.bin 8 | 9 | build/unpack_riscv64.o: unpack_riscv.S 10 | mkdir -p build 11 | riscv64-linux-gnu-gcc -c -o $@ $? 12 | 13 | build/unpack_riscv64.bin: build/unpack_riscv64.o 14 | riscv64-linux-gnu-objcopy -O binary --only-section=.text $? $@ 15 | 16 | disas-riscv64: build/unpack_riscv64.o 17 | riscv64-linux-gnu-objdump -d $? 18 | 19 | build/unpack_riscv32.o: unpack_riscv.S 20 | mkdir -p build 21 | riscv64-linux-gnu-gcc -march=rv32imc -mabi=ilp32 -c -o $@ $? 22 | 23 | build/unpack_riscv32.bin: build/unpack_riscv32.o 24 | riscv64-linux-gnu-objcopy -O binary --only-section=.text $? $@ 25 | 26 | build/unpack_riscv32nc.o: unpack_riscv.S 27 | mkdir -p build 28 | riscv64-linux-gnu-gcc -march=rv32im -mabi=ilp32 -c -o $@ $? 29 | 30 | build/unpack_riscv32nc.bin: build/unpack_riscv32nc.o 31 | riscv64-linux-gnu-objcopy -O binary --only-section=.text $? $@ 32 | 33 | disas-riscv32: build/unpack_riscv32.o 34 | riscv64-linux-gnu-objdump -d $? 35 | 36 | build/unpack_armv6m: ../c_unpacker/main.c unpack_armv6m.S 37 | mkdir -p build 38 | arm-linux-gnueabihf-gcc -g -static -o $@ $^ 39 | 40 | test_armv6m: build/unpack_armv6m 41 | qemu-arm $< test_data.upk /tmp/out.bin 42 | cmp test_data.bin /tmp/out.bin 43 | 44 | build/unpack_armv6m.bin: unpack_armv6m.S 45 | mkdir -p build 46 | arm-none-eabi-gcc -march=armv6-m -c -o build/unpack_armv6m.o $? 47 | arm-none-eabi-objcopy -O binary --only-section=.text build/unpack_armv6m.o $@ 48 | 49 | build/unpack_arm32: ../c_unpacker/main.c unpack_arm32.S 50 | mkdir -p build 51 | arm-linux-gnueabihf-gcc -g -static -o $@ $^ 52 | 53 | test_arm32: build/unpack_arm32 54 | qemu-arm $< test_data.upk /tmp/out.bin 55 | cmp test_data.bin /tmp/out.bin 56 | 57 | build/unpack_arm32.bin: unpack_arm32.S 58 | mkdir -p build 59 | arm-none-eabi-gcc -c -o build/unpack_arm32.o $? 60 | arm-none-eabi-objcopy -O binary --only-section=.text build/unpack_arm32.o $@ 61 | 62 | build/unpack_c: ../c_unpacker/main.c ../c_unpacker/unpack.c 63 | mkdir -p build 64 | gcc -g -o $@ $^ 65 | 66 | test_c: build/unpack_c 67 | $< test_data.upk /tmp/out.bin 68 | cmp test_data.bin /tmp/out.bin 69 | 70 | sizes: build/unpack_armv6m.bin build/unpack_riscv64.bin build/unpack_riscv32.bin build/unpack_arm32.bin 71 | ls -l build/*.bin 72 | -------------------------------------------------------------------------------- /asm_unpackers/test_data.bin: -------------------------------------------------------------------------------- 1 | typedef unsigned char u8; 2 | typedef unsigned short u16; 3 | typedef unsigned long u32; 4 | 5 | u8* upkr_data_ptr; 6 | u8 upkr_probs[1 + 255 + 1 + 2*32 + 2*32]; 7 | #ifdef UPKR_BITSTREAM 8 | u16 upkr_state; 9 | u8 upkr_current_byte; 10 | int upkr_bits_left; 11 | #else 12 | u32 upkr_state; 13 | #endif 14 | 15 | int upkr_decode_bit(int context_index) { 16 | #ifdef UPKR_BITSTREAM 17 | while(upkr_state < 32768) { 18 | if(upkr_bits_left == 0) { 19 | upkr_current_byte = *upkr_data_ptr++; 20 | upkr_bits_left = 8; 21 | } 22 | upkr_state = (upkr_state << 1) + (upkr_current_byte & 1); 23 | upkr_current_byte >>= 1; 24 | --upkr_bits_left; 25 | } 26 | #else 27 | while(upkr_state < 4096) { 28 | upkr_state = (upkr_state << 8) | *upkr_data_ptr++; 29 | } 30 | #endif 31 | 32 | int prob = upkr_probs[context_index]; 33 | int bit = (upkr_state & 255) < prob ? 1 : 0; 34 | 35 | int tmp = prob; 36 | if(!bit) { 37 | tmp = 256 - tmp; 38 | } 39 | upkr_state = tmp * (upkr_state >> 8) + (upkr_state & 255); 40 | tmp += (256 - tmp + 8) >> 4; 41 | if(!bit) { 42 | upkr_state -= prob; 43 | tmp = 256 - tmp; 44 | } 45 | upkr_probs[context_index] = tmp; 46 | 47 | return bit; 48 | } 49 | 50 | int upkr_decode_length(int context_index) { 51 | int length = 0; 52 | int bit_pos = 0; 53 | while(upkr_decode_bit(context_index)) { 54 | length |= upkr_decode_bit(context_index + 1) << bit_pos++; 55 | context_index += 2; 56 | } 57 | return length | (1 << bit_pos); 58 | } 59 | 60 | void* upkr_unpack(void* destination, void* compressed_data) { 61 | upkr_data_ptr = (u8*)compressed_data; 62 | upkr_state = 0; 63 | #ifdef UPKR_BITSTREAM 64 | upkr_bits_left = 0; 65 | #endif 66 | for(int i = 0; i < sizeof(upkr_probs); ++i) 67 | upkr_probs[i] = 128; 68 | 69 | u8* write_ptr = (u8*)destination; 70 | 71 | int prev_was_match = 0; 72 | int offset = 0; 73 | for(;;) { 74 | if(upkr_decode_bit(0)) { 75 | if(prev_was_match || upkr_decode_bit(256)) { 76 | offset = upkr_decode_length(257) - 1; 77 | if(offset == 0) { 78 | break; 79 | } 80 | } 81 | int length = upkr_decode_length(257 + 64); 82 | while(length--) { 83 | *write_ptr = write_ptr[-offset]; 84 | ++write_ptr; 85 | } 86 | prev_was_match = 1; 87 | } else { 88 | int byte = 1; 89 | while(byte < 256) { 90 | int bit = upkr_decode_bit(byte); 91 | byte = (byte << 1) + bit; 92 | } 93 | *write_ptr++ = byte; 94 | prev_was_match = 0; 95 | } 96 | } 97 | 98 | return write_ptr; 99 | } 100 | -------------------------------------------------------------------------------- /asm_unpackers/test_data.upk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/asm_unpackers/test_data.upk -------------------------------------------------------------------------------- /asm_unpackers/unpack_arm32.S: -------------------------------------------------------------------------------- 1 | .arm 2 | 3 | .section .text 4 | 5 | .global upkr_unpack 6 | .type upkr_unpack, %function 7 | // r0 .. out_ptr (returned) 8 | // r1 .. in_ptr (returned) 9 | // r2 .. state 10 | // r3 .. offset 11 | // r4 .. prev_was_literal / decode_length ret 12 | // r5 .. context index 13 | // r6 .. decode_length temp 14 | // r7 .. probs ptr 15 | // r8-r11 .. decode_bit temp 16 | // r12 .. decode_length return address 17 | upkr_unpack: 18 | push { r3-r11, lr } 19 | 20 | mov r2, #384 21 | mov r3, #128 22 | .Lclear: 23 | subs r2, r2, #1 24 | strb r3, [sp, -r2] 25 | bne .Lclear 26 | 27 | .Lloop: 28 | mov r5, #0 29 | bl upkr_decode_bit 30 | bcc .Ldata 31 | .Lmatch: 32 | mov r5, #256 33 | rsbs r6, r4, #0 34 | blcc upkr_decode_bit 35 | bcc .Lskip_offset 36 | 37 | bl upkr_decode_length 38 | adds r3, r4, #1 39 | popeq { r3-r11, pc } 40 | .Lskip_offset: 41 | 42 | mov r5, #256+64 43 | bl upkr_decode_length 44 | .Lcopy_loop: 45 | ldrb r5, [r0, r3] 46 | .Lstore: 47 | strb r5, [r0], #1 48 | adds r4, r4, #1 49 | blt .Lcopy_loop 50 | b .Lloop 51 | 52 | .Ldata: 53 | mov r5, #1 54 | 55 | .Ldata_loop: 56 | bl upkr_decode_bit 57 | adc r5, r5, r5 58 | movs r4, r5, lsr #8 59 | beq .Ldata_loop 60 | b .Lstore 61 | 62 | .type upkr_decode_length, %function 63 | upkr_decode_length: 64 | mov r12, lr 65 | 66 | mov r4, #0 67 | mvn r6, #0 68 | .Lbit_loop: 69 | bl upkr_decode_bit_inc 70 | addcc r4, r4, r6 71 | movcc pc, r12 72 | 73 | bl upkr_decode_bit_inc 74 | addcs r4, r4, r6 75 | mov r6, r6, lsl #1 76 | b .Lbit_loop 77 | 78 | .type upkr_decode_bit, %function 79 | upkr_decode_bit_inc: 80 | add r5, r5, #1 81 | upkr_decode_bit: 82 | cmp r2, #4096 83 | ldrltb r8, [r1], #1 84 | orrlt r2, r8, r2, lsl#8 85 | blt upkr_decode_bit 86 | 87 | ldrb r8, [sp, -r5] 88 | and r9, r2, #255 89 | add r9, r9, #1 90 | cmp r8, r9 91 | rsbcs r8, r8, #256 92 | mvn r9, r2, lsr#8 93 | addcs r9, r9, #1 94 | mla r2, r8, r9, r2 95 | add r9, r8, #8 96 | sub r8, r8, r9, lsr#4 97 | rsbcs r8, r8, #256 98 | strb r8, [sp, -r5] 99 | mov pc, r14 100 | 101 | -------------------------------------------------------------------------------- /asm_unpackers/unpack_armv6m.S: -------------------------------------------------------------------------------- 1 | // armv6-m upkr unpacker by yrlf 2 | // some optimizations by exoticorn 3 | 4 | .syntax unified 5 | .thumb 6 | 7 | .section .text 8 | 9 | #define ALIGNUP(n, align) (((n) + (align) - 1) & ~((align) - 1)) 10 | #define PROB_LEN (1 + 255 + 1 + 2*32 + 2*32) 11 | #define FRAME_SIZE ALIGNUP(PROB_LEN, 4) 12 | 13 | // auto upkr_unpack(uint8_t * out, uint8_t * in) -> tuple 14 | .global upkr_unpack 15 | .type upkr_unpack, %function 16 | // r0 .. out_ptr (returned) 17 | // r1 .. in_ptr (returned) 18 | // r2 .. state 19 | // r3 .. offset 20 | // r4 .. prev_was_literal / decode_length ret 21 | // r5 .. subroutine arg (preserved) 22 | // r6 .. decode_bit ret 23 | // r7 .. probs ptr 24 | upkr_unpack: 25 | push { r4, r5, r6, r7, lr } 26 | sub sp, sp, #FRAME_SIZE 27 | 28 | mov r7, sp 29 | movs r2, #255 30 | adds r2, r2, #(PROB_LEN - 255) 31 | movs r3, #128 32 | .Lclear: 33 | subs r2, r2, #1 34 | strb r3, [r7, r2] 35 | bne .Lclear 36 | 37 | .Lloop: 38 | movs r5, #0 39 | bl upkr_decode_bit 40 | beq .Ldata 41 | .Lmatch: 42 | // r6 = 1 43 | lsls r5, r6, #8 44 | cmp r4, #0 45 | beq 1f 46 | 47 | bl upkr_decode_bit 48 | beq 2f 49 | 50 | 1: 51 | bl upkr_decode_length 52 | adds r3, r4, #1 53 | beq .Lend 54 | 2: 55 | 56 | adds r5, r5, #64 57 | bl upkr_decode_length 58 | .Lcopy_loop: 59 | ldrb r5, [r0, r3] 60 | .Lstore: 61 | strb r5, [r0] 62 | adds r0, r0, #1 63 | adds r4, r4, #1 64 | blt .Lcopy_loop 65 | b .Lloop 66 | 67 | .Ldata: 68 | movs r5, #1 69 | 70 | .Ldata_loop: 71 | bl upkr_decode_bit 72 | adcs r5, r5, r5 73 | lsrs r4, r5, #8 74 | beq .Ldata_loop 75 | b .Lstore 76 | 77 | .Lend: 78 | add sp, sp, #FRAME_SIZE 79 | pop { r4, r5, r6, r7, pc } 80 | 81 | .type upkr_decode_length, %function 82 | // r0 .. -length tmp (saved) 83 | // r1 .. 84 | // r2 .. 85 | // r3 .. 86 | // r4 .. -length (returned) 87 | // r5 .. context index (saved) 88 | // r6 .. (saved) 89 | // r7 .. 90 | upkr_decode_length: 91 | push { r0, r5, r6, lr } 92 | 93 | movs r0, #0 94 | subs r4, r0, #1 95 | .Lbit_loop: 96 | adds r5, r5, #1 97 | bl upkr_decode_bit 98 | beq 1f 99 | 100 | adds r5, r5, #1 101 | bl upkr_decode_bit 102 | beq 2f 103 | adds r0, r0, r4 104 | 2: 105 | lsls r4, r4, #1 106 | b .Lbit_loop 107 | 1: 108 | adds r4, r4, r0 109 | 110 | pop { r0, r5, r6, pc } 111 | 112 | .type upkr_decode_bit, %function 113 | // r0 .. tmp / prob (saved) 114 | // r1 .. in_ptr (modified) 115 | // r2 .. state (modified) 116 | // r3 .. scratch (saved) 117 | // r4 .. 118 | // r5 .. context index (preserved) 119 | // r6 .. bit (returned) 120 | // r7 .. probs ptr (preserved) 121 | upkr_fill_state: 122 | lsls r2, r2, #8 123 | ldrb r6, [r1] 124 | adds r1, r1, #1 125 | orrs r2, r2, r6 126 | 127 | upkr_decode_bit: 128 | lsrs r6, r2, #12 129 | beq upkr_fill_state 130 | 131 | push { r0, r1, r3, lr } 132 | 133 | ldrb r0, [r7, r5] 134 | 135 | lsrs r3, r2, #8 136 | uxtb r1, r2 137 | 138 | subs r6, r1, r0 139 | blt 1f 140 | 141 | subs r1, r2, r0 142 | rsbs r0, r0, #0 143 | 1: 144 | 145 | muls r3, r3, r0 146 | adds r2, r1, r3 147 | 148 | rsbs r3, r0, #0 149 | uxtb r3, r3 150 | lsrs r3, r3, #4 151 | adcs r0, r0, r3 152 | 153 | cmp r6, #0 154 | blt 1f 155 | 156 | rsbs r0, r0, #0 157 | 1: 158 | 159 | strb r0, [r7, r5] 160 | 161 | lsrs r6, r6, #31 162 | pop { r0, r1, r3, pc } 163 | -------------------------------------------------------------------------------- /asm_unpackers/unpack_jagrisc.js: -------------------------------------------------------------------------------- 1 | ;;; -*-asm-*- 2 | ;;; ukpr unpacker for Atari Jaguar RISC. 3 | 4 | ;;; lyxass syntax 5 | 6 | 7 | ; input: 8 | ;;; R20 : packed buffer 9 | ;;; R21 : output buffer 10 | ;;; r30 : return address 11 | ;;; 12 | ;;; Register usage (destroyed!) 13 | ;;; r0-r17,r20,r21 14 | ;;; 15 | 16 | DST REG 21 17 | SRC REG 20 18 | 19 | REGTOP 16 20 | LR_save REG 99 21 | LR_save2 REG 99 22 | GETBIT REG 99 23 | GETLENGTH REG 99 24 | LITERAL REG 99 25 | LOOP REG 99 26 | index REG 99 27 | bit_pos REG 99 28 | state REG 99 29 | prev_was_match REG 99 30 | offset REG 99 31 | prob reg 99 32 | byte REG 99 33 | PROBS reg 99 34 | tmp2 reg 2 35 | tmp1 REG 1 36 | tmp0 REG 0 37 | 38 | REGMAP 39 | 40 | upkr_probs equ $200 41 | 42 | SIZEOF_PROBS EQU 1+255+1+2*32+2*32 43 | 44 | unupkr:: 45 | move LR,LR_save 46 | moveq #0,tmp0 47 | movei #upkr_probs,PROBS 48 | bset #7,tmp0 49 | movei #SIZEOF_PROBS,tmp2 50 | move PROBS,tmp1 51 | .init storeb tmp0,(tmp1) 52 | subq #1,tmp2 53 | jr pl,.init 54 | addq #1,tmp1 55 | 56 | moveq #0,offset 57 | moveq #0,state 58 | movei #getlength,GETLENGTH 59 | movei #getbit,GETBIT 60 | .looppc move PC,LOOP 61 | addq #.loop-.looppc,LOOP 62 | move pc,LITERAL 63 | jr .start 64 | addq #6,LITERAL 65 | 66 | .literal 67 | moveq #1,byte 68 | move pc,LR 69 | jr .into 70 | addq #6,LR ; LR = .getbit 71 | .getbit 72 | addc byte,byte 73 | .into 74 | btst #8,byte 75 | jump eq,(GETBIT) 76 | move byte,index 77 | 78 | storeb byte,(DST) 79 | addq #1,DST 80 | .start 81 | moveq #0,prev_was_match 82 | 83 | .loop 84 | moveq #0,index 85 | BL (GETBIT) 86 | jump cc,(LITERAL) 87 | addq #14,LR 88 | cmpq #1,prev_was_match 89 | jr eq,.newoff 90 | shlq #8,r0 91 | jump (GETBIT) 92 | move r0,index 93 | jr cc,.oldoff 94 | shlq #8,r0 95 | .newoff 96 | addq #1,r0 ; r0 = 257 97 | BL (GETLENGTH) 98 | subq #1,r0 99 | jump eq,(LR_save) 100 | move r0,offset 101 | 102 | .oldoff 103 | movei #257+64,r0 104 | BL (GETLENGTH) 105 | 106 | move DST,r1 107 | sub offset,r1 108 | .cpymatch1 109 | loadb (r1),r2 110 | subq #1,r0 111 | addqt #1,r1 112 | storeb r2,(DST) 113 | jr ne,.cpymatch1 114 | addq #1,DST 115 | 116 | jump (LOOP) 117 | moveq #1,prev_was_match 118 | 119 | getlength: 120 | move LR,LR_save2 121 | moveq #0,byte 122 | move r0,index 123 | moveq #0,bit_pos 124 | move pc,LR 125 | jump (GETBIT) 126 | addq #6,LR 127 | .gl 128 | jr cc,.exit 129 | addq #8,LR ; => return to "sh ..." 130 | jump (GETBIT) 131 | nop 132 | sh bit_pos,r0 133 | subq #1,bit_pos ; sh < 0 => shift left! 134 | or r0,byte 135 | jump (GETBIT) 136 | subq #8,LR 137 | .exit 138 | moveq #1,r0 139 | sh bit_pos,r0 140 | jump (LR_save2) 141 | or byte,r0 142 | 143 | .newbyte: 144 | loadb (SRC),r2 145 | shlq #8,state 146 | addq #1,SRC 147 | or r2,state 148 | getbit 149 | move state,r2 150 | move PROBS,r1 151 | add index,r1 ; r1 = &probs[index] 152 | shrq #12,r2 153 | loadb (r1),prob 154 | jr eq,.newbyte 155 | move state,r2 156 | move state,r0 157 | shlq #24,r2 158 | shrq #8,r0 ; sh 159 | shrq #24,r2 ; sl 160 | cmp prob,r2 161 | addqt #1,index 162 | jr cs,.one 163 | mult prob,r0 164 | 165 | ;; state -= ((state >> 8) + 1)*prob 166 | ;; prob -= (prob+8)>>4 167 | move prob,r2 168 | add prob,r0 169 | addq #8,r2 170 | sub r0,state 171 | shrq #4,r2 172 | moveq #0,r0 173 | jr .ret 174 | sub r2,prob 175 | 176 | .one 177 | ;; state = (state >> 8)*prob+(state & 0xff) 178 | ;; prob += (256 + 8 - prob) >> 4 179 | move r2,state 180 | movei #256+8,r2 181 | add r0,state 182 | sub prob,r2 ; 256-prob+8 183 | shrq #4,r2 184 | add r2,prob 185 | 186 | moveq #3,r0 187 | .ret 188 | storeb prob,(r1) 189 | jump (LR) 190 | shrq #1,r0 ; C = 0, r0 = 1 191 | -------------------------------------------------------------------------------- /asm_unpackers/unpack_jagrisc_fast.js: -------------------------------------------------------------------------------- 1 | ;;; -*-asm-*- 2 | ;;; ukpr unpacker for Atari Jaguar RISC. (quick version) 3 | 4 | ;;; lyxass syntax 5 | 6 | 7 | ; input: 8 | ;;; R20 : packed buffer 9 | ;;; R21 : output buffer 10 | ;;; r30 : return address 11 | ;;; 12 | ;;; Register usage (destroyed!) 13 | ;;; r0-r17,r20,r21 14 | ;;; 15 | 16 | DST REG 21 17 | SRC REG 20 18 | 19 | REGTOP 17 20 | LR_save REG 99 21 | LR_save2 REG 99 22 | GETBIT REG 99 23 | GETLENGTH REG 99 24 | LITERAL REG 99 25 | LOOP REG 99 26 | index REG 99 27 | bit_pos REG 99 28 | state REG 99 29 | prev_was_match REG 99 30 | offset REG 99 31 | prob reg 99 32 | byte REG 99 33 | ndata reg 99 34 | PROBS reg 99 35 | tmp2 reg 2 36 | tmp1 REG 1 37 | tmp0 REG 0 38 | 39 | REGMAP 40 | 41 | upkr_probs equ $200 42 | 43 | SIZEOF_PROBS EQU 1+255+1+2*32+2*32 44 | 45 | unupkr:: 46 | move LR,LR_save 47 | movei #$80808080,tmp0 48 | movei #upkr_probs,PROBS 49 | movei #SIZEOF_PROBS,tmp2 50 | move PROBS,tmp1 51 | .init store tmp0,(tmp1) 52 | subq #4,tmp2 53 | jr pl,.init 54 | addq #4,tmp1 55 | 56 | loadb (SRC),ndata 57 | addq #1,SRC 58 | moveq #0,offset 59 | moveq #0,state 60 | movei #getlength,GETLENGTH 61 | movei #getbit,GETBIT 62 | .looppc move PC,LOOP 63 | addq #.loop-.looppc,LOOP 64 | move pc,LITERAL 65 | jr .start 66 | addq #6,LITERAL 67 | 68 | .literal 69 | moveq #1,byte 70 | move pc,LR 71 | jr .into 72 | addq #6,LR ; LR = .getbit 73 | .getbit 74 | addc byte,byte 75 | .into 76 | btst #8,byte 77 | jump eq,(GETBIT) 78 | move byte,index 79 | 80 | storeb byte,(DST) 81 | addq #1,DST 82 | .start 83 | moveq #0,prev_was_match 84 | 85 | .loop 86 | moveq #0,index 87 | BL (GETBIT) 88 | jump cc,(LITERAL) 89 | addq #14,LR 90 | cmpq #1,prev_was_match 91 | jr eq,.newoff 92 | shlq #8,r0 93 | jump (GETBIT) 94 | move r0,index 95 | jr cc,.oldoff 96 | shlq #8,r0 97 | .newoff 98 | addq #1,r0 ; r0 = 257 99 | BL (GETLENGTH) 100 | subq #1,r0 101 | move r0,offset 102 | jump eq,(LR_save) 103 | nop 104 | .oldoff 105 | movei #257+64,r0 106 | BL (GETLENGTH) 107 | 108 | move DST,r2 109 | move DST,r1 110 | or offset,r2 111 | btst #0,r2 112 | moveq #1,prev_was_match 113 | jr ne,.cpymatch1 114 | sub offset,r1 115 | .cpymatch2 116 | loadw (r1),r2 117 | addqt #2,r1 118 | subq #2,r0 119 | storew r2,(DST) 120 | jump eq,(LOOP) 121 | addqt #2,DST 122 | jr pl,.cpymatch2 123 | nop 124 | jump (LOOP) 125 | subq #1,DST 126 | 127 | .cpymatch1 128 | loadb (r1),r2 129 | subq #1,r0 130 | addqt #1,r1 131 | storeb r2,(DST) 132 | jr ne,.cpymatch1 133 | addq #1,DST 134 | 135 | jump (LOOP) 136 | //-> nop 137 | 138 | getlength: 139 | move LR,LR_save2 140 | moveq #0,byte 141 | move r0,index 142 | moveq #0,bit_pos 143 | move pc,LR 144 | jump (GETBIT) 145 | addq #6,LR 146 | .gl 147 | jr cc,.exit 148 | addq #8,LR ; => return to "sh ..." 149 | jump (GETBIT) 150 | nop 151 | sh bit_pos,r0 152 | subq #1,bit_pos ; sh < 0 => shift left! 153 | or r0,byte 154 | jump (GETBIT) 155 | subq #8,LR 156 | .exit 157 | moveq #1,r0 158 | sh bit_pos,r0 159 | jump (LR_save2) 160 | or byte,r0 161 | 162 | .newbyte: 163 | move ndata,r2 164 | shlq #8,state 165 | loadb (SRC),ndata 166 | or r2,state 167 | addq #1,SRC 168 | move state,r2 169 | shrq #12,r2 170 | jr ne,.done 171 | move state,r2 172 | jr .newbyte 173 | getbit 174 | move state,r2 175 | move PROBS,r1 176 | add index,r1 ; r1 = &probs[index] 177 | shrq #12,r2 178 | loadb (r1),prob 179 | jr eq,.newbyte 180 | move state,r2 181 | .done 182 | move state,r0 183 | shlq #24,r2 184 | shrq #8,r0 ; sh 185 | shrq #24,r2 ; sl 186 | cmp prob,r2 187 | addqt #1,index 188 | jr cs,.one 189 | mult prob,r0 190 | 191 | ;; state -= ((state >> 8) + 1)*prob 192 | ;; prob -= (prob+8)>>4 193 | move prob,r2 194 | add prob,r0 195 | addq #8,r2 196 | sub r0,state 197 | shrq #4,r2 198 | moveq #0,r0 199 | sub r2,prob 200 | shrq #1,r0 ; C = 0, r0 = 0 201 | jump (LR) 202 | storeb prob,(r1) 203 | 204 | .one 205 | ;; state = (state >> 8)*prob+(state & 0xff) 206 | ;; prob += (256 + 8 - prob) >> 4 207 | move r2,state 208 | movei #256+8,r2 209 | add r0,state 210 | sub prob,r2 ; 256-prob+8 211 | shrq #4,r2 212 | add r2,prob 213 | 214 | moveq #3,r0 215 | storeb prob,(r1) 216 | jump (LR) 217 | shrq #1,r0 ; C = 0, r0 = 1 218 | -------------------------------------------------------------------------------- /asm_unpackers/unpack_riscv.S: -------------------------------------------------------------------------------- 1 | .section .text 2 | 3 | // x9 prev was literal 4 | // x10 out ptr 5 | // x11 in ptr 6 | // x12 offset 7 | // x13 state 8 | // x14 context index 9 | 10 | .global upkr_unpack 11 | .type upkr_unpack, %function 12 | upkr_unpack: 13 | mv t4, ra 14 | mv x17, x8 15 | mv t6, x9 16 | li x9, 256 + 128 17 | mv x13, x9 18 | 1: 19 | sub x8, sp, x13 20 | sb x9, 0(x8) 21 | addi x13, x13, -1 22 | bnez x13, 1b 23 | 24 | .Lmainloop: 25 | li x14, 0 26 | jal upkr_decode_bit 27 | beqz x15, .Lliteral 28 | 29 | slli x14, x14, 8 30 | beqz x9, .Lread_offset_inc_x14 31 | jal upkr_decode_bit 32 | bnez x15, .Lread_offset 33 | 34 | .Lfinished_offset: 35 | addi x14, x14, 64 36 | jalr ra // jal upkr_decode_number 37 | 1: 38 | add x14, x10, t0 39 | lbu x14, (x14) 40 | .Lstore_byte: 41 | sb x14, (x10) 42 | addi x10, x10, 1 43 | addi x9, x9, 1 44 | blt x9, x0, 1b 45 | j .Lmainloop 46 | 47 | .Lliteral: 48 | jal upkr_decode_bit 49 | addi x14, x14, -1 50 | slli x14, x14, 1 51 | add x14, x14, x15 52 | srli x9, x14, 8 53 | beqz x9, .Lliteral 54 | j .Lstore_byte 55 | 56 | .Lread_offset_inc_x14: 57 | addi x14, x14, 1 58 | .Lread_offset: 59 | jalr ra // jal upkr_decode_number 60 | addi t0, x9, 1 61 | bnez t0, .Lfinished_offset 62 | .Ldone: 63 | mv x8, x17 64 | mv x9, t6 65 | jr t4 66 | 67 | upkr_load_byte: 68 | lbu x15, 0(x11) 69 | addi x11, x11, 1 70 | slli x13, x13, 8 71 | add x13, x13, x15 72 | // x8 prob array ptr 73 | // x11 in ptr 74 | // x13 state 75 | // x14 context index 76 | // return: 77 | // x14 context index + 1 78 | // x15 decoded bit 79 | upkr_decode_bit: 80 | srli x15, x13, 12 81 | beqz x15, upkr_load_byte 82 | 83 | addi x14, x14, 1 84 | 85 | sub t2, sp, x14 86 | lbu x12, (t2) 87 | 88 | andi x8, x13, 255 89 | sltu x15, x8, x12 90 | beqz x15, 1f 91 | xori x12, x12, 255 92 | addi x12, x12, 1 93 | 1: 94 | srli x8, x13, 8 95 | addi x8, x8, 1 96 | sub x8, x8, x15 97 | mul x8, x8, x12 98 | sub x13, x13, x8 99 | 100 | addi x8, x12, 8 101 | srli x8, x8, 4 102 | sub x12, x12, x8 103 | beqz x15, 1f 104 | sub x12, x0, x12 105 | 1: 106 | 107 | sb x12, (t2) 108 | 109 | jalr ra 110 | 111 | // x14 context index 112 | // return: x9 negtive decoded number 113 | upkr_decode_number: 114 | mv t3, ra 115 | mv t5, x14 116 | li x9, 0 117 | li t1, -1 118 | 1: 119 | jal upkr_decode_bit 120 | beqz x15, 1f 121 | jal upkr_decode_bit 122 | beqz x15, 2f 123 | add x9, x9, t1 124 | 2: 125 | add t1, t1, t1 126 | j 1b 127 | 1: 128 | add x9, x9, t1 129 | 130 | mv x14, t5 131 | jr t3 132 | -------------------------------------------------------------------------------- /c_library/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /upkr -------------------------------------------------------------------------------- /c_library/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "anyhow" 7 | version = "1.0.69" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" 10 | 11 | [[package]] 12 | name = "autocfg" 13 | version = "1.1.0" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 16 | 17 | [[package]] 18 | name = "cc" 19 | version = "1.0.79" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" 22 | 23 | [[package]] 24 | name = "cdivsufsort" 25 | version = "2.0.0" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "edefce019197609da416762da75bb000bbd2224b2d89a7e722c2296cbff79b8c" 28 | dependencies = [ 29 | "cc", 30 | "sacabase", 31 | ] 32 | 33 | [[package]] 34 | name = "lexopt" 35 | version = "0.2.1" 36 | source = "registry+https://github.com/rust-lang/crates.io-index" 37 | checksum = "478ee9e62aaeaf5b140bd4138753d1f109765488581444218d3ddda43234f3e8" 38 | 39 | [[package]] 40 | name = "num-traits" 41 | version = "0.2.15" 42 | source = "registry+https://github.com/rust-lang/crates.io-index" 43 | checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" 44 | dependencies = [ 45 | "autocfg", 46 | ] 47 | 48 | [[package]] 49 | name = "proc-macro2" 50 | version = "1.0.51" 51 | source = "registry+https://github.com/rust-lang/crates.io-index" 52 | checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" 53 | dependencies = [ 54 | "unicode-ident", 55 | ] 56 | 57 | [[package]] 58 | name = "quote" 59 | version = "1.0.23" 60 | source = "registry+https://github.com/rust-lang/crates.io-index" 61 | checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" 62 | dependencies = [ 63 | "proc-macro2", 64 | ] 65 | 66 | [[package]] 67 | name = "sacabase" 68 | version = "2.0.0" 69 | source = "registry+https://github.com/rust-lang/crates.io-index" 70 | checksum = "9883fc3d6ce3d78bb54d908602f8bc1f7b5f983afe601dabe083009d86267a84" 71 | dependencies = [ 72 | "num-traits", 73 | ] 74 | 75 | [[package]] 76 | name = "syn" 77 | version = "1.0.109" 78 | source = "registry+https://github.com/rust-lang/crates.io-index" 79 | checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" 80 | dependencies = [ 81 | "proc-macro2", 82 | "quote", 83 | "unicode-ident", 84 | ] 85 | 86 | [[package]] 87 | name = "thiserror" 88 | version = "1.0.39" 89 | source = "registry+https://github.com/rust-lang/crates.io-index" 90 | checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c" 91 | dependencies = [ 92 | "thiserror-impl", 93 | ] 94 | 95 | [[package]] 96 | name = "thiserror-impl" 97 | version = "1.0.39" 98 | source = "registry+https://github.com/rust-lang/crates.io-index" 99 | checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e" 100 | dependencies = [ 101 | "proc-macro2", 102 | "quote", 103 | "syn", 104 | ] 105 | 106 | [[package]] 107 | name = "unicode-ident" 108 | version = "1.0.8" 109 | source = "registry+https://github.com/rust-lang/crates.io-index" 110 | checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" 111 | 112 | [[package]] 113 | name = "upkr" 114 | version = "0.2.1" 115 | dependencies = [ 116 | "anyhow", 117 | "cdivsufsort", 118 | "lexopt", 119 | "thiserror", 120 | ] 121 | 122 | [[package]] 123 | name = "upkr_c" 124 | version = "0.0.1" 125 | dependencies = [ 126 | "upkr", 127 | ] 128 | -------------------------------------------------------------------------------- /c_library/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "upkr_c" 3 | version = "0.0.1" 4 | edition = "2021" 5 | 6 | [lib] 7 | name = "upkr" 8 | crate-type = ["staticlib"] 9 | 10 | [profile.release] 11 | opt-level = "s" 12 | strip = "debuginfo" 13 | lto = true 14 | panic = "abort" 15 | 16 | [dependencies] 17 | upkr = { path="..", default-features=false } 18 | -------------------------------------------------------------------------------- /c_library/Makefile: -------------------------------------------------------------------------------- 1 | upkr: upkr.c upkr.h target/release/libupkr.a 2 | gcc -O2 -Ltarget/release -o upkr upkr.c -lupkr -lm 3 | strip upkr 4 | 5 | target/release/libupkr.a: cargo 6 | cargo build --release 7 | 8 | .PHONY: cargo -------------------------------------------------------------------------------- /c_library/Readme.md: -------------------------------------------------------------------------------- 1 | This is a simple example of compiling upkr to a library that can be linked in a 2 | c program. It consists of a small rust crate which implements the c api and 3 | compiles to a static library and a matching c header file. As is, the rust 4 | crate offers two simple functions to compress/uncompress data with the default 5 | upkr config. 6 | 7 | The provided makefile will only work on linux. Building the example upkr.c on 8 | other platforms is left as an exercise for the reader ;) 9 | 10 | On Windows you might have to make sure to install and use the correct rust 11 | toolchain version (mingw vs. msvc) to match your c compiler. -------------------------------------------------------------------------------- /c_library/src/lib.rs: -------------------------------------------------------------------------------- 1 | use std::ffi::c_int; 2 | 3 | // the upkr config to use, this can be modified to use other configs 4 | fn config() -> upkr::Config { 5 | upkr::Config::default() 6 | } 7 | 8 | #[no_mangle] 9 | pub extern "C" fn upkr_compress( 10 | output_buffer: *mut u8, 11 | output_buffer_size: usize, 12 | input_buffer: *const u8, 13 | input_size: usize, 14 | compression_level: c_int, 15 | ) -> usize { 16 | let output_buffer = unsafe { std::slice::from_raw_parts_mut(output_buffer, output_buffer_size) }; 17 | let input_buffer = unsafe { std::slice::from_raw_parts(input_buffer, input_size) }; 18 | 19 | let packed_data = upkr::pack(input_buffer, compression_level.max(0).min(9) as u8, &config(), None); 20 | let copy_size = packed_data.len().min(output_buffer.len()); 21 | output_buffer[..copy_size].copy_from_slice(&packed_data[..copy_size]); 22 | 23 | packed_data.len() 24 | } 25 | 26 | #[no_mangle] 27 | pub extern "C" fn upkr_uncompress(output_buffer: *mut u8, output_buffer_size: usize, input_buffer: *const u8, input_size: usize) -> isize { 28 | let output_buffer = unsafe { std::slice::from_raw_parts_mut(output_buffer, output_buffer_size)}; 29 | let input_buffer = unsafe { std::slice::from_raw_parts(input_buffer, input_size)}; 30 | 31 | match upkr::unpack(input_buffer, &config(), output_buffer.len()) { 32 | Ok(unpacked_data) => { 33 | output_buffer[..unpacked_data.len()].copy_from_slice(&unpacked_data); 34 | unpacked_data.len() as isize 35 | } 36 | Err(upkr::UnpackError::OverSize { size, .. }) => size as isize, 37 | Err(other) => { 38 | eprintln!("[upkr] compressed data corrupt: {}", other); 39 | -1 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /c_library/upkr.c: -------------------------------------------------------------------------------- 1 | #include "upkr.h" 2 | #include 3 | #include 4 | #include 5 | 6 | int main(int argc, char** argv) { 7 | if(argc < 2) { 8 | fprintf(stdout, "Usage:\n upkr [compress] [-0 .. -9] []\n upkr [uncompress] []\n"); 9 | return 1; 10 | } 11 | 12 | int argi = 1; 13 | int uncompress = 0; 14 | int compression_level = 4; 15 | if(strcmp(argv[argi], "compress") == 0) { 16 | ++argi; 17 | } else if(strcmp(argv[argi], "uncompress") == 0) { 18 | uncompress = 1; 19 | ++argi; 20 | } 21 | 22 | if(argi < argc && argv[argi][0] == '-') { 23 | compression_level = atoi(argv[argi] + 1); 24 | ++argi; 25 | } 26 | 27 | if(argi == argc) { 28 | fprintf(stdout, "intput filename missing\n"); 29 | return 1; 30 | } 31 | 32 | const char* input_name = argv[argi++]; 33 | char* output_name; 34 | if(argi < argc) { 35 | output_name = argv[argi]; 36 | } else { 37 | output_name = malloc(strlen(input_name) + 5); 38 | strcpy(output_name, input_name); 39 | strcat(output_name, uncompress ? ".unp" : ".upk"); 40 | } 41 | 42 | FILE* file = fopen(input_name, "rb"); 43 | if(file == 0) { 44 | fprintf(stdout, "failed to open input file '%s'\n", file); 45 | return 1; 46 | } 47 | fseek(file, 0, SEEK_END); 48 | long input_size = ftell(file); 49 | rewind(file); 50 | 51 | char* input_buffer = (char*)malloc(input_size); 52 | long offset = 0; 53 | while(offset < input_size) { 54 | long read_size = fread(input_buffer + offset, 1, input_size - offset, file); 55 | if(read_size <= 0) { 56 | fprintf(stdout, "error reading input file\n"); 57 | return 1; 58 | } 59 | offset += read_size; 60 | } 61 | fclose(file); 62 | 63 | long output_buffer_size = input_size * 8; 64 | long output_size; 65 | char* output_buffer = (char*)malloc(output_buffer_size); 66 | for(;;) { 67 | if(uncompress) { 68 | output_size = upkr_uncompress(output_buffer, output_buffer_size, input_buffer, input_size); 69 | } else { 70 | output_size = upkr_compress(output_buffer, output_buffer_size, input_buffer, input_size, compression_level); 71 | } 72 | if(output_size < 0) { 73 | return 1; 74 | } 75 | if(output_size <= output_buffer_size) { 76 | break; 77 | } 78 | output_buffer = (char*)realloc(output_buffer, output_size); 79 | output_buffer_size = output_size; 80 | } 81 | 82 | file = fopen(output_name, "wb"); 83 | if(file == 0) { 84 | fprintf(stdout, "failed to open output file '%s'\n", output_name); 85 | return 1; 86 | } 87 | offset = 0; 88 | while(offset < output_size) { 89 | long written_size = fwrite(output_buffer + offset, 1, output_size - offset, file); 90 | if(written_size <= 0) { 91 | fprintf(stdout, "error writing output file\n"); 92 | return 1; 93 | } 94 | offset += written_size; 95 | } 96 | fclose(file); 97 | 98 | return 0; 99 | } -------------------------------------------------------------------------------- /c_library/upkr.h: -------------------------------------------------------------------------------- 1 | #ifndef UPKR_H_INCLUDED 2 | 3 | #include 4 | 5 | #ifdef __cplusplus 6 | extern "C" { 7 | #endif 8 | 9 | // input_buffer/input_size: input data to compress 10 | // output_buffer/output_buffer_size: buffer to compress into 11 | // compression_level: 0-9 12 | // returns the size of the compressed data, even if it didn't fit into the output buffer 13 | size_t upkr_compress(void* output_buffer, size_t output_buffer_size, void* input_buffer, size_t input_size, int compression_level); 14 | 15 | // input_buffer/input_size: compressed data 16 | // output_buffer/output_buffer_size: buffer to uncompress into 17 | // return value: 18 | // >= 0 : size of uncompressed data, even if it didn't fit into the output buffer 19 | // < 0 : input data corrupt, unable to decompress 20 | ptrdiff_t upkr_uncompress(void* output_buffer, size_t output_buffer_size, void* input_buffer, size_t input_size); 21 | 22 | #ifdef __cplusplus 23 | } 24 | #endif 25 | #endif -------------------------------------------------------------------------------- /c_unpacker/.gitignore: -------------------------------------------------------------------------------- 1 | unpack 2 | unpack_bitstream 3 | unpack_debug 4 | *.upk 5 | 6 | -------------------------------------------------------------------------------- /c_unpacker/Makefile: -------------------------------------------------------------------------------- 1 | all: unpack unpack_bitstream 2 | 3 | unpack: main.c unpack.c 4 | cc -O2 -o unpack main.c unpack.c 5 | 6 | unpack_bitstream: main.c unpack.c 7 | cc -O2 -D UPKR_BITSTREAM -o unpack_bitstream main.c unpack.c 8 | 9 | unpack_debug: main.c unpack.c 10 | cc -g -o unpack_debug main.c unpack.c 11 | -------------------------------------------------------------------------------- /c_unpacker/decode_bit_alt.c: -------------------------------------------------------------------------------- 1 | int upkr_decode_bit(int context_index) { 2 | #ifdef UPKR_BITSTREAM 3 | while(upkr_state < 32768) { 4 | if(upkr_bits_left == 0) { 5 | upkr_current_byte = *upkr_data_ptr++; 6 | upkr_bits_left = 8; 7 | } 8 | upkr_state = (upkr_state << 1) + (upkr_current_byte & 1); 9 | upkr_current_byte >>= 1; 10 | --upkr_bits_left; 11 | } 12 | #else 13 | while(upkr_state < 4096) { 14 | upkr_state = (upkr_state << 8) | *upkr_data_ptr++; 15 | } 16 | #endif 17 | 18 | int prob = upkr_probs[context_index]; 19 | int bit = (upkr_state & 255) < prob ? 1 : 0; 20 | 21 | if(bit) { 22 | prob = 256 - prob; 23 | } 24 | upkr_state -= prob * ((upkr_state >> 8) + (bit ^ 1)); 25 | prob -= (prob + 8) >> 4; 26 | if(bit) { 27 | prob = -prob; 28 | } 29 | upkr_probs[context_index] = prob; 30 | 31 | return bit; 32 | } 33 | 34 | -------------------------------------------------------------------------------- /c_unpacker/main.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | void* upkr_unpack(void* destination, void* compressed_data); 5 | 6 | int main(int argn, char** argv) { 7 | void* input_buffer = malloc(1024*1024); 8 | void* output_buffer = malloc(1024*1024); 9 | 10 | FILE* in_file = fopen(argv[1], "rb"); 11 | int in_size = fread(input_buffer, 1, 1024*1024, in_file); 12 | fclose(in_file); 13 | 14 | printf("Compressed size: %d\n", in_size); 15 | 16 | void* end_ptr = upkr_unpack(output_buffer, input_buffer); 17 | int out_size = (char*)end_ptr - (char*)output_buffer; 18 | 19 | printf("Uncompressed size: %d\n", out_size); 20 | 21 | FILE* out_file = fopen(argv[2], "wb"); 22 | fwrite(output_buffer, 1, out_size, out_file); 23 | fclose(out_file); 24 | 25 | return 0; 26 | } 27 | -------------------------------------------------------------------------------- /c_unpacker/readme.txt: -------------------------------------------------------------------------------- 1 | a very simple unpacker in c, as a reference for people wanting to implement their own unpacker. 2 | absolutely not production ready, it makes no effort to ensure the output buffer can actually 3 | hold the uncompressed data. 4 | !!! Never run on untrusted input !!! 5 | -------------------------------------------------------------------------------- /c_unpacker/unpack.c: -------------------------------------------------------------------------------- 1 | /* 2 | A simple C unpacker for upkr compressed data. 3 | 4 | This implements two variants, selected by the UPKR_BITSTREAM define: 5 | - normal: faster and smaller on modern hardware as whole bytes are shifted into 6 | the rANS state at a time, but requires 20bits for the state 7 | - bitstream: only single bits are shifted into the rANS state at a time 8 | which allows the state to always fit in 16bits which is a boon 9 | on very old CPUs. 10 | The encoder and decoder need to be configured to use the same varianet. 11 | 12 | upkr compressed data is a rANS byte-/bit-stream encoding a series of literal 13 | byte values and back-references as probability encoded bits. 14 | 15 | upkr_decode_bit reads one bit from the rANS stream, taking a probability context 16 | as parameter. The probability context is a byte estimating the probability of 17 | a bit encoded in this context being set. It is updated by upkr_decode_bit 18 | after each decoded bit to reflect the observed past frequencies of on/off bits. 19 | 20 | There are a number of different contexts used in the compressed format. The order in the 21 | upkr_probs array is arbitrary, the only requirement for the unpacker is that all bits 22 | that shared the same context while encoding also share the same context while decoding. 23 | The contexts are: 24 | - is match 25 | - has offset 26 | - literal bit N (0-7) with already decoded highest bits of literal == M (255 total) 27 | - offset bit N (one less than max offset bits) 28 | - has offset bit N (max offset bits) 29 | - length bit N (one less then max length bits) 30 | - has length bit N (max length bits) 31 | 32 | Literal bytes are encoded from highest to lowest bit, with the bit position and 33 | the already decoded bits as context. 34 | 35 | Offst and Length are encoded in an interlaced variant of elias gamma coding. They 36 | are encoded from lowest to highest bits. For each bit, first one bit is read in the 37 | "has offset/length bit N)". If this is set, offset/length bit N is read in it's context 38 | and the decoding continues with the next bit. If the "has bit N" is read as false, a 39 | fixed 1 bit is added as the top bit at this position. 40 | 41 | The highlevel decode loop then looks like this: 42 | loop: 43 | if read_bit(IS_MATCH): 44 | if prev_was_match || read_bit(HAS_OFFSET): 45 | offset = read_length_or_offset(OFFSET) - 1 46 | if offset == 0: 47 | break 48 | length = read_length_or_offset(LENGTH) 49 | copy_bytes_from_offset(length, offset) 50 | else: 51 | read_and_push(literal) 52 | */ 53 | 54 | typedef unsigned char u8; 55 | typedef unsigned short u16; 56 | typedef unsigned long u32; 57 | 58 | u8* upkr_data_ptr; 59 | u8 upkr_probs[1 + 255 + 1 + 2*32 + 2*32]; 60 | #ifdef UPKR_BITSTREAM 61 | u16 upkr_state; 62 | u8 upkr_current_byte; 63 | int upkr_bits_left; 64 | #else 65 | u32 upkr_state; 66 | #endif 67 | 68 | int upkr_decode_bit(int context_index) { 69 | #ifdef UPKR_BITSTREAM 70 | // shift in single bits until rANS state is >= 32768 71 | while(upkr_state < 32768) { 72 | if(upkr_bits_left == 0) { 73 | upkr_current_byte = *upkr_data_ptr++; 74 | upkr_bits_left = 8; 75 | } 76 | upkr_state = (upkr_state << 1) + (upkr_current_byte & 1); 77 | upkr_current_byte >>= 1; 78 | --upkr_bits_left; 79 | } 80 | #else 81 | // shift in a full byte until rANS state is >= 4096 82 | while(upkr_state < 4096) { 83 | upkr_state = (upkr_state << 8) | *upkr_data_ptr++; 84 | } 85 | #endif 86 | 87 | int prob = upkr_probs[context_index]; 88 | int bit = (upkr_state & 255) < prob ? 1 : 0; 89 | 90 | // rANS state and context probability update 91 | // for the later, add 1/16th (rounded) of difference from either 0 or 256 92 | if(bit) { 93 | upkr_state = prob * (upkr_state >> 8) + (upkr_state & 255); 94 | prob += (256 - prob + 8) >> 4; 95 | } else { 96 | upkr_state = (256 - prob) * (upkr_state >> 8) + (upkr_state & 255) - prob; 97 | prob -= (prob + 8) >> 4; 98 | } 99 | upkr_probs[context_index] = prob; 100 | 101 | return bit; 102 | } 103 | 104 | int upkr_decode_length(int context_index) { 105 | int length = 0; 106 | int bit_pos = 0; 107 | while(upkr_decode_bit(context_index)) { 108 | length |= upkr_decode_bit(context_index + 1) << bit_pos++; 109 | context_index += 2; 110 | } 111 | return length | (1 << bit_pos); 112 | } 113 | 114 | void* upkr_unpack(void* destination, void* compressed_data) { 115 | upkr_data_ptr = (u8*)compressed_data; 116 | upkr_state = 0; 117 | #ifdef UPKR_BITSTREAM 118 | upkr_bits_left = 0; 119 | #endif 120 | // all contexts are initialized to 128 = equal probability of 0 and 1 121 | for(int i = 0; i < sizeof(upkr_probs); ++i) 122 | upkr_probs[i] = 128; 123 | 124 | u8* write_ptr = (u8*)destination; 125 | 126 | int prev_was_match = 0; 127 | int offset = 0; 128 | for(;;) { 129 | // is match 130 | if(upkr_decode_bit(0)) { 131 | // has offset 132 | if(prev_was_match || upkr_decode_bit(256)) { 133 | offset = upkr_decode_length(257) - 1; 134 | if(offset == 0) { 135 | // a 0 offset signals the end of the compressed data 136 | break; 137 | } 138 | } 139 | int length = upkr_decode_length(257 + 64); 140 | while(length--) { 141 | *write_ptr = write_ptr[-offset]; 142 | ++write_ptr; 143 | } 144 | prev_was_match = 1; 145 | } else { 146 | // byte contains the previously read bits and indicates the number of 147 | // read bits by the set top bit. Therefore it can be directly used as the 148 | // context index. The set top bit ends up at bit position 8 and is not stored. 149 | int byte = 1; 150 | while(byte < 256) { 151 | int bit = upkr_decode_bit(byte); 152 | byte = (byte << 1) + bit; 153 | } 154 | *write_ptr++ = byte; 155 | prev_was_match = 0; 156 | } 157 | } 158 | 159 | return write_ptr; 160 | } 161 | -------------------------------------------------------------------------------- /dos_unpacker/readme.txt: -------------------------------------------------------------------------------- 1 | 16 bit DOS executable stubs 2 | --------------------------- 3 | 4 | by pestis and TomCat 5 | 6 | unpack_x86_16_DOS.asm: 7 | maximum compatibility, relocates unpacked code to normal start address 8 | unpack_x86_16_DOS_no_relocation.asm: 9 | saves some bytes by not relocating, unpacked code needs to be assembled to 10 | start at 0x3FFE 11 | unpack_x86_16_DOS_no_repeated_offset.asm: 12 | removes support for repeated offsets, potentially at the cost of some compression ratio. 13 | most likely only a win in very narrow circumstances around the 1kb mark -------------------------------------------------------------------------------- /dos_unpacker/unpack_x86_16_DOS.asm: -------------------------------------------------------------------------------- 1 | ; Contributions from pestis, TomCat and exoticorn 2 | ; 3 | ; This is the 16-bit DOS x86 decompression stub for upkr, which is designed for 4 | ; maximum compatibility: it relocates the compressed data so it can be 5 | ; decompressed starting at the normal .COM starting address. In other words, 6 | ; many of the already existing .COM files should be compressable using this 7 | ; stub. 8 | ; 9 | ; How to use: 10 | ; 1) Pack your intro using upkr into data.bin with the --x86 command line 11 | ; argument: 12 | ; 13 | ; $ upkr --x86 intro.com data.bin 14 | ; 15 | ; 2) Compile this .asm file using nasm (or any compatible assembler): 16 | ; 17 | ; $ nasm unpack_x86_16_DOS.asm -fbin -o intropck.com 18 | ; 19 | ; The packed size of the intro+stub is limited by max_len (see below) bytes. 20 | ; 21 | ; In specific cases, the unpacker stub can be further optimized to save a byte 22 | ; or two: 23 | ; 1) You can remove CLC before RET, if you don't mind carry being set upon 24 | ; program entry 25 | ; 2) You can also move PUSHA before PUSH SI and put POPA as the first 26 | ; operation of the compressed code. 27 | max_len equ 16384 28 | prog_start equ (0x100+max_len+510+relocation-upkr_unpack) 29 | probs equ (((prog_start+max_len+510)+255)/256)*256 30 | 31 | org 0x100 32 | 33 | ; This is will be loaded at 0x100, but relocates the code and data to prog_start 34 | relocation: 35 | push si ; si = 0x100 at DOS start, so save it for later ret 36 | pusha ; pusha to recall all registers before starting intro 37 | push si ; for pop di to start writing the output 38 | mov di, prog_start ; the depacker & data are relocated from 0x100 to prog_start 39 | mov ch, max_len/512 40 | rep movsw 41 | jmp si ; jump to relocated upkr_unpack 42 | 43 | 44 | ; upkr_unpack unpacks the code to 0x100 and runs it when done. 45 | upkr_unpack: 46 | xchg ax, bp ; position in input bitstream (bp) = 0 47 | cwd ; upkr_state (dx) = 0; 48 | xchg ax, cx ; cx = 0x9XX 49 | mov al, 128 ; for(int i = 0; i < sizeof(upkr_probs); ++i) upkr_probs[i] = 128; 50 | rep stosb 51 | pop di ; u8* write_ptr = (u8*)destination; 52 | .mainloop: 53 | mov bx, probs 54 | call upkr_decode_bit 55 | jc .else ; if(upkr_decode_bit(0)) { 56 | mov bh, (probs+256)/256 57 | jcxz .skip_call 58 | call upkr_decode_bit 59 | jc .skipoffset 60 | .skip_call: 61 | stc 62 | call upkr_decode_number ; offset = upkr_decode_length(258) - 1; 63 | loop .notdone ; if(offset == 0) 64 | popa 65 | clc 66 | ret 67 | .notdone: 68 | mov si, di 69 | .sub: 70 | dec si 71 | loop .sub 72 | .skipoffset: 73 | mov bl, 128 ; int length = upkr_decode_length(384); 74 | call upkr_decode_number 75 | rep movsb ; *write_ptr = write_ptr[-offset]; 76 | jmp .mainloop 77 | .byteloop: 78 | call upkr_decode_bit ; int bit = upkr_decode_bit(byte); 79 | .else: 80 | adc bl, bl ; byte = (byte << 1) + bit; 81 | jnc .byteloop 82 | xchg ax, bx 83 | stosb 84 | inc si 85 | mov cl, 1 86 | jmp .mainloop ; prev_was_match = 0; 87 | 88 | 89 | ; upkr_decode_bit decodes one bit from the rANS entropy encoded bit stream. 90 | ; parameters: 91 | ; bx = memory address of the context probability 92 | ; dx = decoder state 93 | ; bp = bit position in input stream 94 | ; returns: 95 | ; dx = new decoder state 96 | ; bp = new bit position in input stream 97 | ; carry = bit 98 | ; trashes ax 99 | upkr_load_bit: 100 | bt [compressed_data-relocation+prog_start], bp 101 | inc bp 102 | adc dx, dx 103 | upkr_decode_bit: 104 | inc dx ; inc dx, dec dx is used to test the top (sign) bit of dx 105 | dec dx 106 | jns upkr_load_bit 107 | movzx ax, byte [bx] ; u16 prob = upkr_probs[context_index] 108 | neg byte [bx] 109 | push ax ; save prob, tmp = prob 110 | cmp dl, al ; int bit = (upkr_state & 255) < prob ? 1 : 0; (carry = bit) 111 | pushf ; save bit flags 112 | jc .bit ; (skip if bit) 113 | xchg [bx], al ; tmp = 256 - tmp; 114 | .bit: 115 | shr byte [bx], 4 ; upkr_probs[context_index] = tmp + (256 - tmp + 8) >> 4; 116 | adc [bx], al 117 | mul dh ; upkr_state = tmp * (upkr_state >> 8) + (upkr_state & 255); 118 | mov dh, 0 119 | add dx, ax 120 | popf 121 | pop ax 122 | jc .bit2 ; (skip if bit) 123 | neg byte [bx] ; tmp = 256 - tmp; 124 | sub dx, ax ; upkr_state -= prob; note that this will also leave carry always unset, which is what we want 125 | .bit2: 126 | ret ; return the bit in carry 127 | 128 | 129 | ; upkr_decode_number loads a variable length encoded number (up to 16 bits) from 130 | ; the compressed stream. Only numbers 1..65535 can be encoded. If the encoded 131 | ; number has 4 bits and is 1ABC, it is encoded using a kind of an "interleaved 132 | ; elias code": 0A0B0C1. The 1 in the end implies that no more bits are coming. 133 | ; parameters: 134 | ; cx = must be 0 135 | ; bx = memory address of the context probability 136 | ; dx = decoder state 137 | ; bp = bit position in input stream 138 | ; carry = must be 1 139 | ; returns: 140 | ; cx = length 141 | ; dx = new decoder state 142 | ; bp = new bit position in input stream 143 | ; carry = 1 144 | ; trashes bl, ax 145 | upkr_decode_number_loop: 146 | inc bx 147 | call upkr_decode_bit 148 | upkr_decode_number: 149 | rcr cx, 1 150 | inc bx 151 | call upkr_decode_bit 152 | jnc upkr_decode_number_loop ; 0 = there's more bits coming, 1 = no more bits 153 | .loop2: 154 | rcr cx, 1 155 | jnc .loop2 156 | ret 157 | 158 | 159 | compressed_data: 160 | incbin "data.bin" 161 | -------------------------------------------------------------------------------- /dos_unpacker/unpack_x86_16_DOS_no_relocation.asm: -------------------------------------------------------------------------------- 1 | ; Contributions from pestis, TomCat and exoticorn 2 | ; 3 | ; This is the 16-bit DOS x86 decompression stub for upkr, which decompresses the 4 | ; code starting at address 0x3FFE (or whatever is defined by the entrypoint 5 | ; below). Thus, the packed code needs to be assembled with org 0x3FFE to work. 6 | ; 7 | ; How to use: 8 | ; 1) Put POPA as the first instruction of your compiled code and use org 9 | ; 0x3FFE 10 | ; 2) Pack your intro using upkr into data.bin with the --x86 command line 11 | ; argument: 12 | ; 13 | ; $ upkr --x86 intro.com data.bin 14 | ; 15 | ; 2) Compile this .asm file using nasm (or any compatible assembler) e.g. 16 | ; 17 | ; $ nasm unpack_x86_16_DOS_no_relocation.asm -fbin -o intropck.com 18 | ; 19 | ; In specific cases, the unpacker stub can be further optimized to save a byte 20 | ; or two: 21 | ; 1) If your stub+compressed code is 2k or smaller, you can save 1 byte by 22 | ; putting probs at 0x900 and initializing DI with SALC; XCHG AX, DI instead 23 | ; of MOV DI, probs 24 | ; 2) If you remove the PUSHA (and POPA in the compressed code), then you can 25 | ; assume the registers as follows: AX = 0x00XX, BX = probs + 0x1XX, CX = 0 26 | ; DX = (trash), SI = DI = right after your program, SP = as it was when the 27 | ; program started, flags = carry set 28 | ; 29 | ; Note that even with the PUSHA / POPA, carry will be set (!) unlike normal dos 30 | ; program. 31 | entry equ 0x3FFE 32 | probs equ entry - 0x1FE ; must be aligned to 256 33 | 34 | org 0x100 35 | 36 | 37 | ; This is will be loaded at 0x100, but relocates the code and data to prog_start 38 | upkr_unpack: 39 | pusha 40 | xchg ax, bp ; position in bitstream = 0 41 | cwd ; upkr_state = 0; 42 | mov di, probs 43 | mov ax, 0x8080 ; for(int i = 0; i < sizeof(upkr_probs); ++i) upkr_probs[i] = 128; 44 | rep stosw 45 | push di 46 | .mainloop: 47 | mov bx, probs 48 | call upkr_decode_bit 49 | jc .else ; if(upkr_decode_bit(0)) { 50 | mov bh, (probs+256)/256 51 | jcxz .skip_call ; if(prev_was_match || upkr_decode_bit(257)) { 52 | call upkr_decode_bit 53 | jc .skipoffset 54 | .skip_call: 55 | stc 56 | call upkr_decode_number ; offset = upkr_decode_number(258) - 1; 57 | mov si, di 58 | loop .sub ; if(offset == 0) 59 | ret 60 | .sub: 61 | dec si 62 | loop .sub 63 | .skipoffset: 64 | mov bl, 128 ; int length = upkr_decode_number(384); 65 | call upkr_decode_number 66 | rep movsb ; *write_ptr = write_ptr[-offset]; 67 | jmp .mainloop 68 | .byteloop: 69 | call upkr_decode_bit ; int bit = upkr_decode_bit(byte); 70 | .else: 71 | adc bl, bl ; byte = (byte << 1) + bit; 72 | jnc .byteloop 73 | xchg ax, bx 74 | stosb 75 | inc si 76 | mov cl, 1 77 | jmp .mainloop ; prev_was_match = 0; 78 | 79 | 80 | ; upkr_decode_bit decodes one bit from the rANS entropy encoded bit stream. 81 | ; parameters: 82 | ; bx = memory address of the context probability 83 | ; dx = decoder state 84 | ; bp = bit position in input stream 85 | ; returns: 86 | ; dx = new decoder state 87 | ; bp = new bit position in input stream 88 | ; carry = bit 89 | ; trashes ax 90 | upkr_load_bit: 91 | bt [compressed_data], bp 92 | inc bp 93 | adc dx, dx 94 | upkr_decode_bit: 95 | inc dx 96 | dec dx ; inc dx, dec dx is used to test the top (sign) bit of dx 97 | jns upkr_load_bit 98 | movzx ax, byte [bx] ; u16 prob = upkr_probs[context_index] 99 | neg byte [bx] 100 | push ax ; save prob, tmp = prob 101 | cmp dl, al ; int bit = (upkr_state & 255) < prob ? 1 : 0; (carry = bit) 102 | pushf ; save bit flags 103 | jc .bit ; (skip if bit) 104 | xchg [bx], al ; tmp = 256 - tmp; 105 | .bit: 106 | shr byte [bx], 4 ; upkr_probs[context_index] = tmp + (256 - tmp + 8) >> 4; 107 | adc [bx], al 108 | mul dh ; upkr_state = tmp * (upkr_state >> 8) + (upkr_state & 255); 109 | mov dh, 0 110 | add dx, ax 111 | popf 112 | pop ax 113 | jc .bit2 ; (skip if bit) 114 | neg byte [bx] ; tmp = 256 - tmp; 115 | sub dx, ax ; upkr_state -= prob; note that this will also leave carry always unset, which is what we want 116 | .bit2: 117 | ret ; flags = bit 118 | 119 | 120 | ; upkr_decode_number loads a variable length encoded number (up to 16 bits) from 121 | ; the compressed stream. Only numbers 1..65535 can be encoded. If the encoded 122 | ; number has 4 bits and is 1ABC, it is encoded using a kind of an "interleaved 123 | ; elias code": 0A0B0C1. The 1 in the end implies that no more bits are coming. 124 | ; parameters: 125 | ; cx = must be 0 126 | ; bx = memory address of the context probability 127 | ; dx = decoder state 128 | ; bp = bit position in input stream 129 | ; carry = must be 1 130 | ; returns: 131 | ; cx = length 132 | ; dx = new decoder state 133 | ; bp = new bit position in input stream 134 | ; carry = 1 135 | ; trashes bl, ax 136 | upkr_decode_number_loop: 137 | inc bx 138 | call upkr_decode_bit 139 | upkr_decode_number: 140 | rcr cx, 1 141 | inc bx 142 | call upkr_decode_bit 143 | jnc upkr_decode_number_loop ; while(upkr_decode_bit(context_index)) { 144 | .loop2: 145 | rcr cx, 1 146 | jnc .loop2 147 | ret 148 | 149 | 150 | compressed_data: 151 | incbin "data.bin" 152 | -------------------------------------------------------------------------------- /dos_unpacker/unpack_x86_16_DOS_no_repeated_offset.asm: -------------------------------------------------------------------------------- 1 | ; Contributions from pestis, TomCat and exoticorn 2 | ; 3 | ; This is the 16-bit DOS x86 decompression stub for upkr, which is designed for 4 | ; the --no-repeated-offsets option of upkr. The decompression stub is slightly 5 | ; smaller, but the compressed data might be bigger, so you have to test if 6 | ; --no-repeated-offsets pays off in the end. This stub relocates the compressed 7 | ; data so it can be decompressed starting at the normal .COM starting address. 8 | ; 9 | ; How to use: 10 | ; 1) Pack your intro using upkr into data.bin with the --x86b command line 11 | ; argument: (notice the --x86b, not --x86!) 12 | ; 13 | ; $ upkr --x86b intro.com data.bin 14 | ; 15 | ; 2) Compile this .asm file using nasm (or any compatible assembler): 16 | ; 17 | ; $ nasm unpack_x86_16_DOS_no_repeated_offsets.asm -fbin -o intropck.com 18 | ; 19 | ; The packed size of the intro+stub is limited by max_len (see below) bytes. 20 | ; 21 | ; In specific cases, the unpacker stub can be further optimized to save a byte 22 | ; or two: 23 | ; 1) You can remove CLC before RET, if you don't mind carry being set upon 24 | ; program entry 25 | ; 2) You can also move PUSHA before PUSH SI and put POPA as the first 26 | ; operation of the compressed code. 27 | max_len equ 16384 28 | prog_start equ (0x100+max_len+510+relocation-upkr_unpack) 29 | probs equ (((prog_start+max_len+510)+255)/256)*256 30 | 31 | org 0x100 32 | 33 | 34 | ; This is will be loaded at 0x100, but relocates the code and data to prog_start 35 | relocation: 36 | push si ; si = 0x100 at DOS start, so save it for later ret 37 | pusha ; pusha to recall all registers before starting intro 38 | push si ; for pop di to start writing the output 39 | mov di, prog_start ; the depacker & data are relocated from 0x100 to prog_start 40 | mov ch, max_len/512 41 | rep movsw 42 | jmp si ; jump to relocated upkr_unpack 43 | 44 | 45 | ; upkr_unpack unpacks the code to 0x100 and runs it when done. 46 | upkr_unpack: 47 | xchg ax, bp ; position in bitstream = 0 48 | cwd ; upkr_state = 0; 49 | xchg cx, ax ; cx > 0x0200 50 | mov al, 128 ; for(int i = 0; i < sizeof(upkr_probs); ++i) upkr_probs[i] = 128; 51 | rep stosb 52 | pop di ; u8* write_ptr = (u8*)destination; 53 | .mainloop: 54 | mov bx, probs 55 | call upkr_decode_bit 56 | jnc .else ; if(upkr_decode_bit(0)) { 57 | inc bh 58 | call upkr_decode_number ; offset = upkr_decode_number(258) - 1; 59 | loop .notdone ; if(offset == 0) 60 | popa 61 | clc 62 | ret 63 | .notdone: 64 | mov si, di 65 | .sub: 66 | dec si 67 | loop .sub 68 | mov bl, 128 ; int length = upkr_decode_number(384); 69 | call upkr_decode_number 70 | rep movsb ; *write_ptr = write_ptr[-offset]; 71 | jmp .mainloop 72 | .else: 73 | inc bx 74 | .byteloop: 75 | call upkr_decode_bit ; int bit = upkr_decode_bit(byte); 76 | adc bl, bl ; byte = (byte << 1) + bit; 77 | jnc .byteloop 78 | xchg ax, bx 79 | stosb 80 | jmp .mainloop ; prev_was_match = 0; 81 | 82 | 83 | ; upkr_decode_bit decodes one bit from the rANS entropy encoded bit stream. 84 | ; parameters: 85 | ; bx = memory address of the context probability 86 | ; dx = decoder state 87 | ; bp = bit position in input stream 88 | ; returns: 89 | ; dx = new decoder state 90 | ; bp = new bit position in input stream 91 | ; carry = bit 92 | ; trashes ax 93 | upkr_load_bit: 94 | bt [compressed_data-relocation+prog_start], bp 95 | inc bp 96 | adc dx, dx 97 | upkr_decode_bit: 98 | inc dx 99 | dec dx ; or whatever other test for the top bit there is 100 | jns upkr_load_bit 101 | movzx ax, byte [bx] ; u16 prob = upkr_probs[context_index] 102 | neg byte [bx] 103 | push ax ; save prob, tmp = prob 104 | cmp dl, al ; int bit = (upkr_state & 255) < prob ? 1 : 0; (carry = bit) 105 | pushf ; save bit flags 106 | jc .bit ; (skip if bit) 107 | xchg [bx], al ; tmp = 256 - tmp; 108 | .bit: 109 | shr byte [bx], 4 ; upkr_probs[context_index] = tmp + (256 - tmp + 8) >> 4; 110 | adc [bx], al ; upkr_probs[context_index] = tmp; 111 | mul dh ; upkr_state = tmp * (upkr_state >> 8) + (upkr_state & 255); 112 | mov dh, 0 113 | add dx, ax 114 | popf 115 | pop ax 116 | jc .bit2 ; (skip if bit) 117 | neg byte [bx] ; tmp = 256 - tmp; 118 | sub dx, ax ; upkr_state -= prob; note that this will also leave carry always unset, which is what we want 119 | .bit2: 120 | ret ; flags = bit 121 | 122 | 123 | ; upkr_decode_number loads a variable length encoded number (up to 16 bits) from 124 | ; the compressed stream. Only numbers 1..65535 can be encoded. If the encoded 125 | ; number has 4 bits and is 1ABC, it is encoded using a kind of an "interleaved 126 | ; elias code": 0A0B0C1. The 1 in the end implies that no more bits are coming. 127 | ; parameters: 128 | ; cx = must be 0 129 | ; bx = memory address of the context probability 130 | ; dx = decoder state 131 | ; bp = bit position in input stream 132 | ; carry = must be 1 133 | ; returns: 134 | ; cx = length 135 | ; dx = new decoder state 136 | ; bp = new bit position in input stream 137 | ; carry = 1 138 | ; trashes bl, ax 139 | upkr_decode_number_loop: 140 | inc bx 141 | call upkr_decode_bit 142 | upkr_decode_number: 143 | rcr cx, 1 144 | inc bx 145 | call upkr_decode_bit 146 | jnc upkr_decode_number_loop ; 0 = there's more bits coming, 1 = no more bits 147 | .loop2: 148 | rcr cx, 1 149 | jnc .loop2 150 | ret 151 | 152 | 153 | compressed_data: 154 | incbin "data.bin" 155 | -------------------------------------------------------------------------------- /fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | corpus 3 | artifacts 4 | -------------------------------------------------------------------------------- /fuzz/Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "anyhow" 7 | version = "1.0.65" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "98161a4e3e2184da77bb14f02184cdd111e83bbbcc9979dfee3c44b9a85f5602" 10 | 11 | [[package]] 12 | name = "arbitrary" 13 | version = "1.1.6" 14 | source = "registry+https://github.com/rust-lang/crates.io-index" 15 | checksum = "f44124848854b941eafdb34f05b3bcf59472f643c7e151eba7c2b69daa469ed5" 16 | 17 | [[package]] 18 | name = "autocfg" 19 | version = "1.1.0" 20 | source = "registry+https://github.com/rust-lang/crates.io-index" 21 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 22 | 23 | [[package]] 24 | name = "cc" 25 | version = "1.0.73" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" 28 | dependencies = [ 29 | "jobserver", 30 | ] 31 | 32 | [[package]] 33 | name = "cdivsufsort" 34 | version = "2.0.0" 35 | source = "registry+https://github.com/rust-lang/crates.io-index" 36 | checksum = "edefce019197609da416762da75bb000bbd2224b2d89a7e722c2296cbff79b8c" 37 | dependencies = [ 38 | "cc", 39 | "sacabase", 40 | ] 41 | 42 | [[package]] 43 | name = "cfg-if" 44 | version = "1.0.0" 45 | source = "registry+https://github.com/rust-lang/crates.io-index" 46 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 47 | 48 | [[package]] 49 | name = "crossbeam-channel" 50 | version = "0.5.6" 51 | source = "registry+https://github.com/rust-lang/crates.io-index" 52 | checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" 53 | dependencies = [ 54 | "cfg-if", 55 | "crossbeam-utils", 56 | ] 57 | 58 | [[package]] 59 | name = "crossbeam-utils" 60 | version = "0.8.11" 61 | source = "registry+https://github.com/rust-lang/crates.io-index" 62 | checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" 63 | dependencies = [ 64 | "cfg-if", 65 | "once_cell", 66 | ] 67 | 68 | [[package]] 69 | name = "jobserver" 70 | version = "0.1.25" 71 | source = "registry+https://github.com/rust-lang/crates.io-index" 72 | checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" 73 | dependencies = [ 74 | "libc", 75 | ] 76 | 77 | [[package]] 78 | name = "lexopt" 79 | version = "0.2.1" 80 | source = "registry+https://github.com/rust-lang/crates.io-index" 81 | checksum = "478ee9e62aaeaf5b140bd4138753d1f109765488581444218d3ddda43234f3e8" 82 | 83 | [[package]] 84 | name = "libc" 85 | version = "0.2.133" 86 | source = "registry+https://github.com/rust-lang/crates.io-index" 87 | checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966" 88 | 89 | [[package]] 90 | name = "libfuzzer-sys" 91 | version = "0.4.4" 92 | source = "registry+https://github.com/rust-lang/crates.io-index" 93 | checksum = "ae185684fe19814afd066da15a7cc41e126886c21282934225d9fc847582da58" 94 | dependencies = [ 95 | "arbitrary", 96 | "cc", 97 | "once_cell", 98 | ] 99 | 100 | [[package]] 101 | name = "num-traits" 102 | version = "0.2.15" 103 | source = "registry+https://github.com/rust-lang/crates.io-index" 104 | checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" 105 | dependencies = [ 106 | "autocfg", 107 | ] 108 | 109 | [[package]] 110 | name = "once_cell" 111 | version = "1.15.0" 112 | source = "registry+https://github.com/rust-lang/crates.io-index" 113 | checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" 114 | 115 | [[package]] 116 | name = "pbr" 117 | version = "1.0.4" 118 | source = "registry+https://github.com/rust-lang/crates.io-index" 119 | checksum = "ff5751d87f7c00ae6403eb1fcbba229b9c76c9a30de8c1cf87182177b168cea2" 120 | dependencies = [ 121 | "crossbeam-channel", 122 | "libc", 123 | "time", 124 | "winapi", 125 | ] 126 | 127 | [[package]] 128 | name = "proc-macro2" 129 | version = "1.0.44" 130 | source = "registry+https://github.com/rust-lang/crates.io-index" 131 | checksum = "7bd7356a8122b6c4a24a82b278680c73357984ca2fc79a0f9fa6dea7dced7c58" 132 | dependencies = [ 133 | "unicode-ident", 134 | ] 135 | 136 | [[package]] 137 | name = "quote" 138 | version = "1.0.21" 139 | source = "registry+https://github.com/rust-lang/crates.io-index" 140 | checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" 141 | dependencies = [ 142 | "proc-macro2", 143 | ] 144 | 145 | [[package]] 146 | name = "sacabase" 147 | version = "2.0.0" 148 | source = "registry+https://github.com/rust-lang/crates.io-index" 149 | checksum = "9883fc3d6ce3d78bb54d908602f8bc1f7b5f983afe601dabe083009d86267a84" 150 | dependencies = [ 151 | "num-traits", 152 | ] 153 | 154 | [[package]] 155 | name = "syn" 156 | version = "1.0.101" 157 | source = "registry+https://github.com/rust-lang/crates.io-index" 158 | checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2" 159 | dependencies = [ 160 | "proc-macro2", 161 | "quote", 162 | "unicode-ident", 163 | ] 164 | 165 | [[package]] 166 | name = "thiserror" 167 | version = "1.0.36" 168 | source = "registry+https://github.com/rust-lang/crates.io-index" 169 | checksum = "0a99cb8c4b9a8ef0e7907cd3b617cc8dc04d571c4e73c8ae403d80ac160bb122" 170 | dependencies = [ 171 | "thiserror-impl", 172 | ] 173 | 174 | [[package]] 175 | name = "thiserror-impl" 176 | version = "1.0.36" 177 | source = "registry+https://github.com/rust-lang/crates.io-index" 178 | checksum = "3a891860d3c8d66fec8e73ddb3765f90082374dbaaa833407b904a94f1a7eb43" 179 | dependencies = [ 180 | "proc-macro2", 181 | "quote", 182 | "syn", 183 | ] 184 | 185 | [[package]] 186 | name = "time" 187 | version = "0.1.44" 188 | source = "registry+https://github.com/rust-lang/crates.io-index" 189 | checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" 190 | dependencies = [ 191 | "libc", 192 | "wasi", 193 | "winapi", 194 | ] 195 | 196 | [[package]] 197 | name = "unicode-ident" 198 | version = "1.0.4" 199 | source = "registry+https://github.com/rust-lang/crates.io-index" 200 | checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" 201 | 202 | [[package]] 203 | name = "upkr" 204 | version = "0.2.0-pre3" 205 | dependencies = [ 206 | "anyhow", 207 | "cdivsufsort", 208 | "lexopt", 209 | "pbr", 210 | "thiserror", 211 | ] 212 | 213 | [[package]] 214 | name = "upkr-fuzz" 215 | version = "0.0.0" 216 | dependencies = [ 217 | "libfuzzer-sys", 218 | "upkr", 219 | ] 220 | 221 | [[package]] 222 | name = "wasi" 223 | version = "0.10.0+wasi-snapshot-preview1" 224 | source = "registry+https://github.com/rust-lang/crates.io-index" 225 | checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" 226 | 227 | [[package]] 228 | name = "winapi" 229 | version = "0.3.9" 230 | source = "registry+https://github.com/rust-lang/crates.io-index" 231 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 232 | dependencies = [ 233 | "winapi-i686-pc-windows-gnu", 234 | "winapi-x86_64-pc-windows-gnu", 235 | ] 236 | 237 | [[package]] 238 | name = "winapi-i686-pc-windows-gnu" 239 | version = "0.4.0" 240 | source = "registry+https://github.com/rust-lang/crates.io-index" 241 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 242 | 243 | [[package]] 244 | name = "winapi-x86_64-pc-windows-gnu" 245 | version = "0.4.0" 246 | source = "registry+https://github.com/rust-lang/crates.io-index" 247 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 248 | -------------------------------------------------------------------------------- /fuzz/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "upkr-fuzz" 3 | version = "0.0.0" 4 | authors = ["Automatically generated"] 5 | publish = false 6 | edition = "2018" 7 | 8 | [package.metadata] 9 | cargo-fuzz = true 10 | 11 | [dependencies] 12 | libfuzzer-sys = "0.4" 13 | 14 | [dependencies.upkr] 15 | path = ".." 16 | 17 | # Prevent this from interfering with workspaces 18 | [workspace] 19 | members = ["."] 20 | 21 | [[bin]] 22 | name = "all_configs" 23 | path = "fuzz_targets/all_configs.rs" 24 | test = false 25 | doc = false 26 | 27 | [[bin]] 28 | name = "unpack" 29 | path = "fuzz_targets/unpack.rs" 30 | test = false 31 | doc = false 32 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/all_configs.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | use libfuzzer_sys::fuzz_target; 3 | 4 | fuzz_target!(|data: &[u8]| { 5 | let mut config = upkr::Config::default(); 6 | let mut level = 1; 7 | let mut data = data; 8 | if data.len() > 2 { 9 | let flags1 = data[0]; 10 | let flags2 = data[1]; 11 | data = &data[2..]; 12 | config.use_bitstream = (flags1 & 1) != 0; 13 | config.parity_contexts = if (flags1 & 2) == 0 { 1 } else { 2 }; 14 | config.invert_bit_encoding = (flags1 & 4) != 0; 15 | config.is_match_bit = (flags1 & 8) != 0; 16 | config.new_offset_bit = (flags1 & 16) != 0; 17 | config.continue_value_bit = (flags1 & 32) != 0; 18 | config.bitstream_is_big_endian = (flags1 & 64) != 0; 19 | config.simplified_prob_update = (flags1 & 128) != 0; 20 | config.no_repeated_offsets = (flags2 & 32) != 0; 21 | config.eof_in_length = (flags2 & 1) != 0; 22 | config.max_offset = if (flags2 & 2) == 0 { usize::MAX } else { 32 }; 23 | config.max_length = if (flags2 & 4) == 0 { usize::MAX } else { 5 }; 24 | level = (flags2 >> 3) & 3; 25 | } 26 | let packed = upkr::pack(data, level, &config, None); 27 | let unpacked = upkr::unpack(&packed, &config, 1024 * 1024).unwrap(); 28 | assert!(unpacked == data); 29 | }); 30 | -------------------------------------------------------------------------------- /fuzz/fuzz_targets/unpack.rs: -------------------------------------------------------------------------------- 1 | #![no_main] 2 | use libfuzzer_sys::fuzz_target; 3 | 4 | fuzz_target!(|data: &[u8]| { 5 | let _ = upkr::unpack(data, &upkr::Config::default(), 64 * 1024); 6 | }); 7 | -------------------------------------------------------------------------------- /release/.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | *.tgz 3 | upkr-linux/ 4 | upkr-windows/ 5 | upkr-windows-32/ 6 | -------------------------------------------------------------------------------- /release/Makefile: -------------------------------------------------------------------------------- 1 | VERSION := $(shell cargo run --release -- --version) 2 | 3 | all: clean upkr-linux-$(VERSION).tgz upkr-windows-$(VERSION).zip upkr-windows-32-$(VERSION).zip 4 | 5 | clean: 6 | rm -rf upkr-linux 7 | rm -f upkr-linux*.tgz 8 | rm -rf upkr-windows 9 | rm -rf upkr-windows-32 10 | rm -f upkr-windows*.zip 11 | 12 | upkr-linux-$(VERSION).tgz: upkr-linux/upkr PHONY 13 | cp ../README.md upkr-linux 14 | cd .. && git archive HEAD c_unpacker | tar -xC release/upkr-linux 15 | cd .. && git archive HEAD z80_unpacker | tar -xC release/upkr-linux 16 | cd .. && git archive HEAD asm_unpackers | tar -xC release/upkr-linux 17 | tar czf $@ upkr-linux 18 | 19 | upkr-windows-$(VERSION).zip: upkr-windows/upkr.exe PHONY 20 | cp ../README.md upkr-windows/ 21 | cd .. && git archive HEAD c_unpacker | tar -xC release/upkr-windows 22 | cd .. && git archive HEAD z80_unpacker | tar -xC release/upkr-windows 23 | cd .. && git archive HEAD asm_unpackers | tar -xC release/upkr-windows 24 | zip -r -9 $@ upkr-windows 25 | 26 | upkr-windows-32-$(VERSION).zip: upkr-windows-32/upkr.exe PHONY 27 | cp ../README.md upkr-windows-32/ 28 | cd .. && git archive HEAD c_unpacker | tar -xC release/upkr-windows-32 29 | cd .. && git archive HEAD z80_unpacker | tar -xC release/upkr-windows-32 30 | cd .. && git archive HEAD asm_unpackers | tar -xC release/upkr-windows-32 31 | zip -r -9 $@ upkr-windows-32 32 | 33 | upkr-linux/upkr: 34 | cargo build --target x86_64-unknown-linux-musl --release -F terminal 35 | mkdir -p upkr-linux 36 | cp ../target/x86_64-unknown-linux-musl/release/upkr upkr-linux/ 37 | strip upkr-linux/upkr 38 | 39 | upkr-windows/upkr.exe: 40 | cargo build --target x86_64-pc-windows-gnu --release -F terminal 41 | mkdir -p upkr-windows 42 | cp ../target/x86_64-pc-windows-gnu/release/upkr.exe upkr-windows/ 43 | x86_64-w64-mingw32-strip upkr-windows/upkr.exe 44 | 45 | upkr-windows-32/upkr.exe: 46 | cargo build --target i686-pc-windows-gnu --release -F terminal 47 | mkdir -p upkr-windows-32 48 | cp ../target/i686-pc-windows-gnu/release/upkr.exe upkr-windows-32/ 49 | i686-w64-mingw32-strip upkr-windows-32/upkr.exe 50 | 51 | PHONY: 52 | -------------------------------------------------------------------------------- /src/context_state.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | Config, 3 | rans::{ONE_PROB, PROB_BITS}, 4 | }; 5 | 6 | const INIT_PROB: u16 = 1 << (PROB_BITS - 1); 7 | const UPDATE_RATE: u32 = 4; 8 | const UPDATE_ADD: u32 = 8; 9 | 10 | #[derive(Clone)] 11 | pub struct ContextState { 12 | contexts: Vec, 13 | invert_bit_encoding: bool, 14 | simplified_prob_update: bool, 15 | } 16 | 17 | pub struct Context<'a> { 18 | state: &'a mut ContextState, 19 | index: usize, 20 | } 21 | 22 | impl ContextState { 23 | pub fn new(size: usize, config: &Config) -> ContextState { 24 | ContextState { 25 | contexts: vec![INIT_PROB as u8; size], 26 | invert_bit_encoding: config.invert_bit_encoding, 27 | simplified_prob_update: config.simplified_prob_update, 28 | } 29 | } 30 | 31 | pub fn context_mut(&mut self, index: usize) -> Context { 32 | Context { state: self, index } 33 | } 34 | } 35 | 36 | impl<'a> Context<'a> { 37 | pub fn prob(&self) -> u16 { 38 | self.state.contexts[self.index] as u16 39 | } 40 | 41 | pub fn update(&mut self, bit: bool) { 42 | let old = self.state.contexts[self.index]; 43 | 44 | self.state.contexts[self.index] = if self.state.simplified_prob_update { 45 | let offset = if bit ^ self.state.invert_bit_encoding { 46 | ONE_PROB as i32 >> UPDATE_RATE 47 | } else { 48 | 0 49 | }; 50 | 51 | (offset + old as i32 - ((old as i32 + UPDATE_ADD as i32) >> UPDATE_RATE)) as u8 52 | } else if bit ^ self.state.invert_bit_encoding { 53 | old + ((ONE_PROB - old as u32 + UPDATE_ADD) >> UPDATE_RATE) as u8 54 | } else { 55 | old - ((old as u32 + UPDATE_ADD) >> UPDATE_RATE) as u8 56 | }; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/greedy_packer.rs: -------------------------------------------------------------------------------- 1 | use crate::match_finder::MatchFinder; 2 | use crate::rans::RansCoder; 3 | use crate::ProgressCallback; 4 | use crate::{lz, Config}; 5 | 6 | pub fn pack( 7 | data: &[u8], 8 | config: &Config, 9 | mut progress_callback: Option, 10 | ) -> Vec { 11 | let mut match_finder = MatchFinder::new(data); 12 | let mut rans_coder = RansCoder::new(config); 13 | let mut state = lz::CoderState::new(config); 14 | 15 | let mut pos = 0; 16 | while pos < data.len() { 17 | if let Some(ref mut cb) = progress_callback { 18 | cb(pos); 19 | } 20 | let mut encoded_match = false; 21 | if let Some(m) = match_finder.matches(pos).next() { 22 | let max_offset = config.max_offset.min(1 << (m.length * 3 - 1).min(31)); 23 | let offset = pos - m.pos; 24 | if offset < max_offset && m.length >= config.min_length() { 25 | let length = m.length.min(config.max_length); 26 | lz::Op::Match { 27 | offset: offset as u32, 28 | len: length as u32, 29 | } 30 | .encode(&mut rans_coder, &mut state, config); 31 | pos += length; 32 | encoded_match = true; 33 | } 34 | } 35 | 36 | if !encoded_match { 37 | let offset = state.last_offset() as usize; 38 | if offset != 0 { 39 | let length = data[pos..] 40 | .iter() 41 | .zip(data[(pos - offset)..].iter()) 42 | .take_while(|(a, b)| a == b) 43 | .count() 44 | .min(config.max_length); 45 | if length >= config.min_length() { 46 | lz::Op::Match { 47 | offset: offset as u32, 48 | len: length as u32, 49 | } 50 | .encode(&mut rans_coder, &mut state, config); 51 | pos += length; 52 | encoded_match = true; 53 | } 54 | } 55 | } 56 | 57 | if !encoded_match { 58 | lz::Op::Literal(data[pos]).encode(&mut rans_coder, &mut state, config); 59 | pos += 1; 60 | } 61 | } 62 | 63 | lz::encode_eof(&mut rans_coder, &mut state, config); 64 | rans_coder.finish() 65 | } 66 | -------------------------------------------------------------------------------- /src/heatmap.rs: -------------------------------------------------------------------------------- 1 | /// Heatmap information about a compressed block of data. 2 | /// 3 | /// For each byte in the uncompressed data, the heatmap provides two pieces of intormation: 4 | /// 1. whether this byte was encoded as a literal or as part of a match 5 | /// 2. how many (fractional) bits where spend on encoding this byte 6 | /// 7 | /// For the sake of the heatmap, the cost of literals are spread out across all matches 8 | /// that reference the literal. 9 | /// 10 | /// If the `terminal` feature is enabled, there is a function to write out the 11 | /// heatmap as a colored hexdump. 12 | pub struct Heatmap { 13 | data: Vec, 14 | cost: Vec, 15 | raw_cost: Vec, 16 | literal_index: Vec, 17 | } 18 | 19 | impl Heatmap { 20 | pub(crate) fn new() -> Heatmap { 21 | Heatmap { 22 | data: Vec::new(), 23 | cost: Vec::new(), 24 | raw_cost: Vec::new(), 25 | literal_index: Vec::new(), 26 | } 27 | } 28 | 29 | pub(crate) fn add_literal(&mut self, byte: u8, cost: f32) { 30 | self.data.push(byte); 31 | self.cost.push(cost); 32 | self.literal_index.push(self.literal_index.len()); 33 | } 34 | 35 | pub(crate) fn add_match(&mut self, offset: usize, length: usize, mut cost: f32) { 36 | cost /= length as f32; 37 | for _ in 0..length { 38 | self.data.push(self.data[self.data.len() - offset]); 39 | self.literal_index 40 | .push(self.literal_index[self.literal_index.len() - offset]); 41 | self.cost.push(cost); 42 | } 43 | } 44 | 45 | pub(crate) fn finish(&mut self) { 46 | self.raw_cost = self.cost.clone(); 47 | 48 | let mut ref_count = vec![0usize; self.literal_index.len()]; 49 | for &index in &self.literal_index { 50 | ref_count[index] += 1; 51 | } 52 | 53 | let mut shifted = vec![]; 54 | for (&index, &cost) in self.literal_index.iter().zip(self.cost.iter()) { 55 | let delta = (self.cost[index] - cost) / ref_count[index] as f32; 56 | shifted.push(delta); 57 | shifted[index] -= delta; 58 | } 59 | 60 | for (cost, delta) in self.cost.iter_mut().zip(shifted.into_iter()) { 61 | *cost += delta; 62 | } 63 | } 64 | 65 | /// Reverses the heatmap 66 | pub fn reverse(&mut self) { 67 | self.data.reverse(); 68 | self.cost.reverse(); 69 | self.literal_index.reverse(); 70 | for index in self.literal_index.iter_mut() { 71 | *index = self.data.len() - *index; 72 | } 73 | } 74 | 75 | /// The number of (uncompressed) bytes of data in this heatmap 76 | pub fn len(&self) -> usize { 77 | self.cost.len() 78 | } 79 | 80 | /// Returns whether the heatmap data is empty 81 | pub fn is_empty(&self) -> bool { 82 | self.cost.is_empty() 83 | } 84 | 85 | /// Returns whether the byte at `index` was encoded as a literal 86 | pub fn is_literal(&self, index: usize) -> bool { 87 | self.literal_index[index] == index 88 | } 89 | 90 | /// Returns the cost of encoding the byte at `index` in (fractional) bits. 91 | /// The cost of literal bytes is spread across the matches that reference it. 92 | /// See `raw_cost` for the raw encoding cost of each byte. 93 | pub fn cost(&self, index: usize) -> f32 { 94 | self.cost[index] 95 | } 96 | 97 | /// Returns the raw cost of encoding the byte at `index` in (fractional) bits 98 | pub fn raw_cost(&self, index: usize) -> f32 { 99 | self.raw_cost[index] 100 | } 101 | 102 | /// Returns the uncompressed data byte at `index` 103 | pub fn byte(&self, index: usize) -> u8 { 104 | self.data[index] 105 | } 106 | 107 | #[cfg(feature = "crossterm")] 108 | /// Print the heatmap as a colored hexdump 109 | pub fn print_as_hex(&self) -> std::io::Result<()> { 110 | self.print_as_hex_internal(false) 111 | } 112 | 113 | #[cfg(feature = "crossterm")] 114 | /// Print the heatmap as a colored hexdump, based on `raw_cost`. 115 | pub fn print_as_hex_raw_cost(&self) -> std::io::Result<()> { 116 | self.print_as_hex_internal(true) 117 | } 118 | 119 | #[cfg(feature = "crossterm")] 120 | fn print_as_hex_internal(&self, report_raw_cost: bool) -> std::io::Result<()> { 121 | use crossterm::{ 122 | QueueableCommand, 123 | style::{Attribute, Color, Print, SetAttribute, SetBackgroundColor}, 124 | }; 125 | use std::io::{Write, stdout}; 126 | 127 | fn set_color( 128 | mut out: impl QueueableCommand, 129 | heatmap: &Heatmap, 130 | index: usize, 131 | num_colors: u16, 132 | report_raw_cost: bool, 133 | ) -> std::io::Result<()> { 134 | let cost = if report_raw_cost { 135 | heatmap.raw_cost(index) 136 | } else { 137 | heatmap.cost(index) 138 | }; 139 | if num_colors < 256 { 140 | let colors = [ 141 | Color::Red, 142 | Color::Yellow, 143 | Color::Green, 144 | Color::Cyan, 145 | Color::Blue, 146 | Color::DarkBlue, 147 | Color::Black, 148 | ]; 149 | let color_index = (3. - cost.log2()) 150 | .round() 151 | .max(0.) 152 | .min((colors.len() - 1) as f32) as usize; 153 | out.queue(SetBackgroundColor(colors[color_index]))?; 154 | } else { 155 | let colors = [ 156 | 196, 166, 136, 106, 76, 46, 41, 36, 31, 26, 21, 20, 19, 18, 17, 16, 157 | ]; 158 | let color_index = ((3. - cost.log2()) * 2.5) 159 | .round() 160 | .max(0.) 161 | .min((colors.len() - 1) as f32) as usize; 162 | out.queue(SetBackgroundColor(Color::AnsiValue(colors[color_index])))?; 163 | } 164 | out.queue(SetAttribute(if heatmap.is_literal(index) { 165 | Attribute::Underlined 166 | } else { 167 | Attribute::NoUnderline 168 | }))?; 169 | Ok(()) 170 | } 171 | 172 | let num_colors = crossterm::style::available_color_count(); 173 | 174 | let term_width = crossterm::terminal::size()?.0.min(120) as usize; 175 | let bytes_per_row = (term_width - 8) / 4; 176 | 177 | for row_start in (0..self.data.len()).step_by(bytes_per_row) { 178 | let row_range = row_start..self.data.len().min(row_start + bytes_per_row); 179 | let mut stdout = stdout(); 180 | 181 | stdout.queue(Print(&format!("{:04x} ", row_start)))?; 182 | 183 | for i in row_range.clone() { 184 | set_color(&mut stdout, self, i, num_colors, report_raw_cost)?; 185 | stdout.queue(Print(&format!("{:02x} ", self.data[i])))?; 186 | } 187 | 188 | let num_spaces = 1 + (bytes_per_row - (row_range.end - row_range.start)) * 3; 189 | let gap: String = std::iter::repeat(' ').take(num_spaces).collect(); 190 | stdout 191 | .queue(SetAttribute(Attribute::Reset))? 192 | .queue(Print(&gap))?; 193 | 194 | for i in row_range.clone() { 195 | set_color(&mut stdout, self, i, num_colors, report_raw_cost)?; 196 | let byte = self.data[i]; 197 | if byte >= 32 && byte < 127 { 198 | stdout.queue(Print(format!("{}", byte as char)))?; 199 | } else { 200 | stdout.queue(Print("."))?; 201 | } 202 | } 203 | 204 | stdout 205 | .queue(SetAttribute(Attribute::Reset))? 206 | .queue(Print("\n"))?; 207 | 208 | stdout.flush()?; 209 | } 210 | 211 | Ok(()) 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![deny(missing_docs)] 2 | 3 | //! Compression and decompression of the upkr format and variants. 4 | //! 5 | //! Upkr is a compression format initially designed for the MicroW8 fantasy console, 6 | //! with design goals being a competitive compression ratio, reasonable fast 7 | //! decompression, low memory overhead and very small decompression code 8 | //! when handoptimized in assembler. (An optimized DOS execuable decompressor is <140 bytes.) 9 | 10 | mod context_state; 11 | mod greedy_packer; 12 | mod heatmap; 13 | mod lz; 14 | mod match_finder; 15 | mod parsing_packer; 16 | mod rans; 17 | 18 | pub use heatmap::Heatmap; 19 | pub use lz::{calculate_margin, create_heatmap, unpack, UnpackError}; 20 | 21 | /// The type of a callback function to be given to the `pack` function. 22 | /// 23 | /// It will be periodically called with the number of bytes of the input already processed. 24 | pub type ProgressCallback<'a> = &'a mut dyn FnMut(usize); 25 | 26 | /// A configuration of which compression format variation to use. 27 | /// 28 | /// Use `Config::default()` for the standard upkr format. 29 | /// 30 | /// Compression format variants exist to help with micro-optimizations in uncompression 31 | /// code on specific platforms. 32 | 33 | #[derive(Debug)] 34 | pub struct Config { 35 | /// Shift in bits from a bitstream into the rANS state, rather than whole bytes. 36 | /// This decreases the size of the rNAS state to 16 bits which is very useful on 37 | /// 8 bit platforms. 38 | pub use_bitstream: bool, 39 | /// The number of parity contexts (usually 1, 2 or 4). This can improve compression 40 | /// on data that consists of regular groups of 2 or 4 bytes. One example is 32bit ARM 41 | /// code, where each instruction is 4 bytes, so `parity_contexts = 4` improves compression 42 | /// quite a bit. Defaults to `1`. 43 | pub parity_contexts: usize, 44 | 45 | /// Invert the encoding of bits in the rANS coder. `bit = state_lo >= prob` instead of 46 | /// `bit = state_lo < prob`. 47 | pub invert_bit_encoding: bool, 48 | /// The boolean value which encodes a match. Defaults to `true`. 49 | pub is_match_bit: bool, 50 | /// The boolean value which encodes a new offset (rather than re-using the previous offset). 51 | /// Defaults to `true`. 52 | pub new_offset_bit: bool, 53 | /// The boolean value which encodes that there are more bits comming for length/offset values. 54 | /// Defaults to `true`. 55 | pub continue_value_bit: bool, 56 | 57 | /// Reverses the bits in the bitstream. 58 | pub bitstream_is_big_endian: bool, 59 | /// A slightly less accurate, but slightly simpler variation of the prob update in the 60 | /// rANS coder, Used for the z80 uncompressor. 61 | pub simplified_prob_update: bool, 62 | 63 | /// Disables support for re-using the last offset in the compression format. 64 | /// This might save a few bytes when working with very small data. 65 | pub no_repeated_offsets: bool, 66 | /// Standard upkr encodes the EOF marker in the offset. This encodes it in the match length 67 | /// instead. 68 | pub eof_in_length: bool, 69 | 70 | /// The maximum match offset value to encode when compressing. 71 | pub max_offset: usize, 72 | /// The maximum match length value to encode when compressing. 73 | pub max_length: usize, 74 | } 75 | 76 | impl Default for Config { 77 | fn default() -> Config { 78 | Config { 79 | use_bitstream: false, 80 | parity_contexts: 1, 81 | 82 | invert_bit_encoding: false, 83 | is_match_bit: true, 84 | new_offset_bit: true, 85 | continue_value_bit: true, 86 | 87 | bitstream_is_big_endian: false, 88 | simplified_prob_update: false, 89 | 90 | no_repeated_offsets: false, 91 | eof_in_length: false, 92 | 93 | max_offset: usize::MAX, 94 | max_length: usize::MAX, 95 | } 96 | } 97 | } 98 | 99 | impl Config { 100 | fn min_length(&self) -> usize { 101 | if self.eof_in_length { 102 | 2 103 | } else { 104 | 1 105 | } 106 | } 107 | } 108 | 109 | /// Compresses the given data. 110 | /// 111 | /// # Arguments 112 | /// - `data`: The data to compress 113 | /// - `level`: The compression level (0-9). Increasing the level by one roughly halves the 114 | /// compression speed. 115 | /// - `config`: The compression format variant to use. 116 | /// - `progress_callback`: An optional callback which will periodically be called with 117 | /// the number of bytes already processed. 118 | /// 119 | /// # Example 120 | /// ```rust 121 | /// let compressed_data = upkr::pack(b"Hello, World! Yellow world!", 0, &upkr::Config::default(), None); 122 | /// assert!(compressed_data.len() < 27); 123 | /// ``` 124 | pub fn pack( 125 | data: &[u8], 126 | level: u8, 127 | config: &Config, 128 | progress_callback: Option, 129 | ) -> Vec { 130 | if level == 0 { 131 | greedy_packer::pack(data, config, progress_callback) 132 | } else { 133 | parsing_packer::pack(data, level, config, progress_callback) 134 | } 135 | } 136 | 137 | /// Estimate the exact (fractional) size of upkr compressed data. 138 | /// 139 | /// Note that this currently does NOT work for the bitstream variant. 140 | pub fn compressed_size(mut data: &[u8]) -> f32 { 141 | let mut state = 0; 142 | while state < 4096 { 143 | state = (state << 8) | data[0] as u32; 144 | data = &data[1..]; 145 | } 146 | data.len() as f32 + (state as f32).log2() / 8. 147 | } 148 | -------------------------------------------------------------------------------- /src/lz.rs: -------------------------------------------------------------------------------- 1 | use crate::Config; 2 | use crate::context_state::ContextState; 3 | use crate::heatmap::Heatmap; 4 | use crate::rans::{EntropyCoder, RansDecoder}; 5 | use thiserror::Error; 6 | 7 | #[derive(Copy, Clone, Debug)] 8 | pub enum Op { 9 | Literal(u8), 10 | Match { offset: u32, len: u32 }, 11 | } 12 | 13 | impl Op { 14 | pub fn encode(&self, coder: &mut dyn EntropyCoder, state: &mut CoderState, config: &Config) { 15 | let literal_base = state.pos % state.parity_contexts * 256; 16 | match *self { 17 | Op::Literal(lit) => { 18 | encode_bit(coder, state, literal_base, !config.is_match_bit); 19 | let mut context_index = 1; 20 | for i in (0..8).rev() { 21 | let bit = (lit >> i) & 1 != 0; 22 | encode_bit(coder, state, literal_base + context_index, bit); 23 | context_index = (context_index << 1) | bit as usize; 24 | } 25 | state.prev_was_match = false; 26 | state.pos += 1; 27 | } 28 | Op::Match { offset, len } => { 29 | encode_bit(coder, state, literal_base, config.is_match_bit); 30 | let mut new_offset = true; 31 | if !state.prev_was_match && !config.no_repeated_offsets { 32 | new_offset = offset != state.last_offset; 33 | encode_bit( 34 | coder, 35 | state, 36 | 256 * state.parity_contexts, 37 | new_offset == config.new_offset_bit, 38 | ); 39 | } 40 | assert!(offset as usize <= config.max_offset); 41 | if new_offset { 42 | encode_length( 43 | coder, 44 | state, 45 | 256 * state.parity_contexts + 1, 46 | offset + if config.eof_in_length { 0 } else { 1 }, 47 | config, 48 | ); 49 | state.last_offset = offset; 50 | } 51 | assert!(len as usize >= config.min_length() && len as usize <= config.max_length); 52 | encode_length(coder, state, 256 * state.parity_contexts + 65, len, config); 53 | state.prev_was_match = true; 54 | state.pos += len as usize; 55 | } 56 | } 57 | } 58 | } 59 | 60 | pub fn encode_eof(coder: &mut dyn EntropyCoder, state: &mut CoderState, config: &Config) { 61 | encode_bit( 62 | coder, 63 | state, 64 | state.pos % state.parity_contexts * 256, 65 | config.is_match_bit, 66 | ); 67 | if !state.prev_was_match && !config.no_repeated_offsets { 68 | encode_bit( 69 | coder, 70 | state, 71 | 256 * state.parity_contexts, 72 | config.new_offset_bit ^ config.eof_in_length, 73 | ); 74 | } 75 | if !config.eof_in_length || state.prev_was_match || config.no_repeated_offsets { 76 | encode_length(coder, state, 256 * state.parity_contexts + 1, 1, config); 77 | } 78 | if config.eof_in_length { 79 | encode_length(coder, state, 256 * state.parity_contexts + 65, 1, config); 80 | } 81 | } 82 | 83 | fn encode_bit( 84 | coder: &mut dyn EntropyCoder, 85 | state: &mut CoderState, 86 | context_index: usize, 87 | bit: bool, 88 | ) { 89 | coder.encode_with_context(bit, &mut state.contexts.context_mut(context_index)); 90 | } 91 | 92 | fn encode_length( 93 | coder: &mut dyn EntropyCoder, 94 | state: &mut CoderState, 95 | context_start: usize, 96 | mut value: u32, 97 | config: &Config, 98 | ) { 99 | assert!(value >= 1); 100 | 101 | let mut context_index = context_start; 102 | while value >= 2 { 103 | encode_bit(coder, state, context_index, config.continue_value_bit); 104 | encode_bit(coder, state, context_index + 1, value & 1 != 0); 105 | context_index += 2; 106 | value >>= 1; 107 | } 108 | encode_bit(coder, state, context_index, !config.continue_value_bit); 109 | } 110 | 111 | #[derive(Clone)] 112 | pub struct CoderState { 113 | contexts: ContextState, 114 | last_offset: u32, 115 | prev_was_match: bool, 116 | pos: usize, 117 | parity_contexts: usize, 118 | } 119 | 120 | impl CoderState { 121 | pub fn new(config: &Config) -> CoderState { 122 | CoderState { 123 | contexts: ContextState::new((1 + 255) * config.parity_contexts + 1 + 64 + 64, config), 124 | last_offset: 0, 125 | prev_was_match: false, 126 | pos: 0, 127 | parity_contexts: config.parity_contexts, 128 | } 129 | } 130 | 131 | pub fn last_offset(&self) -> u32 { 132 | self.last_offset 133 | } 134 | } 135 | 136 | /// The error type for the uncompressing related functions 137 | #[derive(Error, Debug)] 138 | pub enum UnpackError { 139 | /// a match offset pointing beyond the start of the unpacked data was encountered 140 | #[error("match offset out of range: {offset} > {position}")] 141 | OffsetOutOfRange { 142 | /// the match offset 143 | offset: usize, 144 | /// the current position in the uncompressed stream 145 | position: usize, 146 | }, 147 | /// The passed size limit was exceeded 148 | #[error("Unpacked data over size limit: {size} > {limit}")] 149 | OverSize { 150 | /// the size of the uncompressed data 151 | size: usize, 152 | /// the size limit passed into the function 153 | limit: usize, 154 | }, 155 | /// The end of the packed data was reached without an encoded EOF marker 156 | #[error("Unexpected end of input data")] 157 | UnexpectedEOF { 158 | #[from] 159 | /// the underlying EOF error in the rANS decoder 160 | source: crate::rans::UnexpectedEOF, 161 | }, 162 | /// An offset or length value was found that exceeded 32bit 163 | #[error("Overflow while reading value")] 164 | ValueOverflow, 165 | } 166 | 167 | /// Uncompress a piece of compressed data 168 | /// 169 | /// Returns either the uncompressed data, or an `UnpackError` 170 | /// 171 | /// # Parameters 172 | /// 173 | /// - `packed_data`: the compressed data 174 | /// - `config`: the exact compression format config used to compress the data 175 | /// - `max_size`: the maximum size of uncompressed data to return. When this is exceeded, 176 | /// `UnpackError::OverSize` is returned 177 | pub fn unpack( 178 | packed_data: &[u8], 179 | config: &Config, 180 | max_size: usize, 181 | ) -> Result, UnpackError> { 182 | let mut result = vec![]; 183 | let _ = unpack_internal(Some(&mut result), None, packed_data, config, max_size)?; 184 | Ok(result) 185 | } 186 | 187 | /// Calculates the minimum margin when overlapping buffers. 188 | /// 189 | /// Returns the minimum margin needed between the end of the compressed data and the 190 | /// end of the uncompressed data when overlapping the two buffers to save on RAM. 191 | pub fn calculate_margin(packed_data: &[u8], config: &Config) -> Result { 192 | unpack_internal(None, None, packed_data, config, usize::MAX) 193 | } 194 | 195 | /// Calculates a `Heatmap` from compressed data. 196 | /// 197 | /// # Parameters 198 | /// 199 | /// - `packed_data`: the compressed data 200 | /// - `config`: the exact compression format config used to compress the data 201 | /// - `max_size`: the maximum size of the heatmap to return. When this is exceeded, 202 | /// `UnpackError::OverSize` is returned 203 | pub fn create_heatmap( 204 | packed_data: &[u8], 205 | config: &Config, 206 | max_size: usize, 207 | ) -> Result { 208 | let mut heatmap = Heatmap::new(); 209 | let _ = unpack_internal(None, Some(&mut heatmap), packed_data, config, max_size)?; 210 | Ok(heatmap) 211 | } 212 | 213 | fn unpack_internal( 214 | mut result: Option<&mut Vec>, 215 | mut heatmap: Option<&mut Heatmap>, 216 | packed_data: &[u8], 217 | config: &Config, 218 | max_size: usize, 219 | ) -> Result { 220 | let mut decoder = RansDecoder::new(packed_data, config)?; 221 | let mut contexts = ContextState::new((1 + 255) * config.parity_contexts + 1 + 64 + 64, config); 222 | let mut offset = usize::MAX; 223 | let mut position = 0usize; 224 | let mut prev_was_match = false; 225 | let mut margin = 0isize; 226 | 227 | fn decode_length( 228 | decoder: &mut RansDecoder, 229 | contexts: &mut ContextState, 230 | mut context_index: usize, 231 | config: &Config, 232 | ) -> Result { 233 | let mut length = 0; 234 | let mut bit_pos = 0; 235 | while decoder.decode_with_context(&mut contexts.context_mut(context_index))? 236 | == config.continue_value_bit 237 | { 238 | length |= (decoder.decode_with_context(&mut contexts.context_mut(context_index + 1))? 239 | as usize) 240 | << bit_pos; 241 | bit_pos += 1; 242 | if bit_pos >= 32 { 243 | return Err(UnpackError::ValueOverflow); 244 | } 245 | context_index += 2; 246 | } 247 | Ok(length | (1 << bit_pos)) 248 | } 249 | 250 | loop { 251 | let prev_decoder = decoder.clone(); 252 | margin = margin.max(position as isize - decoder.pos() as isize); 253 | let literal_base = position % config.parity_contexts * 256; 254 | if decoder.decode_with_context(&mut contexts.context_mut(literal_base))? 255 | == config.is_match_bit 256 | { 257 | if config.no_repeated_offsets 258 | || prev_was_match 259 | || decoder 260 | .decode_with_context(&mut contexts.context_mut(256 * config.parity_contexts))? 261 | == config.new_offset_bit 262 | { 263 | offset = decode_length( 264 | &mut decoder, 265 | &mut contexts, 266 | 256 * config.parity_contexts + 1, 267 | config, 268 | )? - if config.eof_in_length { 0 } else { 1 }; 269 | if offset == 0 { 270 | break; 271 | } 272 | } 273 | let length = decode_length( 274 | &mut decoder, 275 | &mut contexts, 276 | 256 * config.parity_contexts + 65, 277 | config, 278 | )?; 279 | if config.eof_in_length && length == 1 { 280 | break; 281 | } 282 | if offset > position { 283 | return Err(UnpackError::OffsetOutOfRange { offset, position }); 284 | } 285 | if let Some(ref mut heatmap) = heatmap { 286 | heatmap.add_match(offset, length, decoder.cost(&prev_decoder)); 287 | } 288 | if let Some(ref mut result) = result { 289 | for _ in 0..length { 290 | if result.len() < max_size { 291 | result.push(result[result.len() - offset]); 292 | } else { 293 | break; 294 | } 295 | } 296 | } 297 | position += length; 298 | prev_was_match = true; 299 | } else { 300 | let mut context_index = 1; 301 | let mut byte = 0; 302 | for i in (0..8).rev() { 303 | let bit = decoder 304 | .decode_with_context(&mut contexts.context_mut(literal_base + context_index))?; 305 | context_index = (context_index << 1) | bit as usize; 306 | byte |= (bit as u8) << i; 307 | } 308 | if let Some(ref mut heatmap) = heatmap { 309 | heatmap.add_literal(byte, decoder.cost(&prev_decoder)); 310 | } 311 | if let Some(ref mut result) = result { 312 | if result.len() < max_size { 313 | result.push(byte); 314 | } 315 | } 316 | position += 1; 317 | prev_was_match = false; 318 | } 319 | } 320 | 321 | if let Some(heatmap) = heatmap { 322 | heatmap.finish(); 323 | } 324 | 325 | if position > max_size { 326 | return Err(UnpackError::OverSize { 327 | size: position, 328 | limit: max_size, 329 | }); 330 | } 331 | 332 | Ok(margin + decoder.pos() as isize - position as isize) 333 | } 334 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use anyhow::Result; 2 | use std::ffi::OsStr; 3 | use std::io::prelude::*; 4 | use std::process; 5 | use std::{fs::File, path::PathBuf}; 6 | 7 | fn main() -> Result<()> { 8 | let mut config = upkr::Config::default(); 9 | let mut reverse = false; 10 | let mut unpack = false; 11 | let mut calculate_margin = false; 12 | let mut create_heatmap = false; 13 | let mut report_raw_cost = false; 14 | #[allow(unused_mut)] 15 | let mut do_hexdump = false; 16 | let mut level = 2; 17 | let mut infile: Option = None; 18 | let mut outfile: Option = None; 19 | let mut max_unpacked_size = 512 * 1024 * 1024; 20 | 21 | let mut parser = lexopt::Parser::from_env(); 22 | while let Some(arg) = parser.next()? { 23 | use lexopt::prelude::*; 24 | match arg { 25 | Short('b') | Long("bitstream") => config.use_bitstream = true, 26 | Short('p') | Long("parity") => config.parity_contexts = parser.value()?.parse()?, 27 | Short('r') | Long("reverse") => reverse = true, 28 | Long("invert-is-match-bit") => config.is_match_bit = false, 29 | Long("invert-new-offset-bit") => config.new_offset_bit = false, 30 | Long("invert-continue-value-bit") => config.continue_value_bit = false, 31 | Long("invert-bit-encoding") => config.invert_bit_encoding = true, 32 | Long("simplified-prob-update") => config.simplified_prob_update = true, 33 | Long("big-endian-bitstream") => { 34 | config.use_bitstream = true; 35 | config.bitstream_is_big_endian = true; 36 | } 37 | Long("no-repeated-offsets") => config.no_repeated_offsets = true, 38 | Long("eof-in-length") => config.eof_in_length = true, 39 | 40 | Long("max-offset") => config.max_offset = parser.value()?.parse()?, 41 | Long("max-length") => config.max_length = parser.value()?.parse()?, 42 | 43 | Long("z80") => { 44 | config.use_bitstream = true; 45 | config.bitstream_is_big_endian = true; 46 | config.invert_bit_encoding = true; 47 | config.simplified_prob_update = true; 48 | level = 9; 49 | } 50 | Long("x86") => { 51 | config.use_bitstream = true; 52 | config.continue_value_bit = false; 53 | config.is_match_bit = false; 54 | config.new_offset_bit = false; 55 | } 56 | Long("x86b") => { 57 | config.use_bitstream = true; 58 | config.continue_value_bit = false; 59 | config.no_repeated_offsets = true; 60 | level = 9; 61 | } 62 | 63 | Short('u') | Long("unpack") | Short('d') | Long("decompress") => unpack = true, 64 | Long("margin") => calculate_margin = true, 65 | Long("heatmap") => create_heatmap = true, 66 | Long("raw-cost") => report_raw_cost = true, 67 | #[cfg(feature = "crossterm")] 68 | Long("hexdump") => do_hexdump = true, 69 | Short('l') | Long("level") => level = parser.value()?.parse()?, 70 | Short(n) if n.is_ascii_digit() => level = n as u8 - b'0', 71 | Short('h') | Long("help") => print_help(0), 72 | Long("version") => { 73 | println!("{}", env!("CARGO_PKG_VERSION")); 74 | process::exit(0); 75 | } 76 | Long("max-unpacked-size") => max_unpacked_size = parser.value()?.parse()?, 77 | Value(val) if infile.is_none() => infile = Some(val.into()), 78 | Value(val) if outfile.is_none() => outfile = Some(val.into()), 79 | _ => return Err(arg.unexpected().into()), 80 | } 81 | } 82 | 83 | let infile = IoTarget::from_filename(infile); 84 | let outfile = |tpe: OutFileType| infile.output(tpe, &outfile); 85 | 86 | if config.parity_contexts != 1 && config.parity_contexts != 2 && config.parity_contexts != 4 { 87 | eprintln!("--parity has to be 1, 2, or 4"); 88 | process::exit(1); 89 | } 90 | 91 | if !unpack && !calculate_margin && !create_heatmap { 92 | let mut data = infile.read()?; 93 | if reverse { 94 | data.reverse(); 95 | } 96 | 97 | #[cfg(feature = "terminal")] 98 | let mut packed_data = { 99 | let mut pb = pbr::ProgressBar::on(std::io::stderr(), data.len() as u64); 100 | pb.set_units(pbr::Units::Bytes); 101 | let packed_data = upkr::pack( 102 | &data, 103 | level, 104 | &config, 105 | Some(&mut |pos| { 106 | pb.set(pos as u64); 107 | }), 108 | ); 109 | pb.finish(); 110 | eprintln!(); 111 | packed_data 112 | }; 113 | #[cfg(not(feature = "terminal"))] 114 | let mut packed_data = upkr::pack(&data, level, &config, None); 115 | 116 | if reverse { 117 | packed_data.reverse(); 118 | } 119 | 120 | eprintln!( 121 | "Compressed {} bytes to {} bytes ({}%)", 122 | data.len(), 123 | packed_data.len(), 124 | packed_data.len() as f32 * 100. / data.len() as f32 125 | ); 126 | outfile(OutFileType::Packed).write(&packed_data)?; 127 | } else { 128 | let mut data = infile.read()?; 129 | if reverse { 130 | data.reverse(); 131 | } 132 | if unpack { 133 | let mut unpacked_data = upkr::unpack(&data, &config, max_unpacked_size)?; 134 | if reverse { 135 | unpacked_data.reverse(); 136 | } 137 | outfile(OutFileType::Unpacked).write(&unpacked_data)?; 138 | } 139 | if create_heatmap { 140 | let mut heatmap = upkr::create_heatmap(&data, &config, max_unpacked_size)?; 141 | if reverse { 142 | heatmap.reverse(); 143 | } 144 | match do_hexdump { 145 | #[cfg(feature = "crossterm")] 146 | true => { 147 | if report_raw_cost { 148 | heatmap.print_as_hex_raw_cost()? 149 | } else { 150 | heatmap.print_as_hex()? 151 | } 152 | } 153 | _ => { 154 | let mut heatmap_bin = Vec::with_capacity(heatmap.len()); 155 | for i in 0..heatmap.len() { 156 | let cost = if report_raw_cost { 157 | heatmap.raw_cost(i) 158 | } else { 159 | heatmap.cost(i) 160 | }; 161 | let cost = (cost.log2() * 8. + 64.).round().clamp(0., 127.) as u8; 162 | heatmap_bin.push((cost << 1) | heatmap.is_literal(i) as u8); 163 | } 164 | outfile(OutFileType::Heatmap).write(&heatmap_bin)?; 165 | } 166 | } 167 | } 168 | if calculate_margin { 169 | println!("{}", upkr::calculate_margin(&data, &config)?); 170 | } 171 | } 172 | 173 | Ok(()) 174 | } 175 | 176 | enum OutFileType { 177 | Packed, 178 | Unpacked, 179 | Heatmap, 180 | } 181 | 182 | enum IoTarget { 183 | StdInOut, 184 | File(PathBuf), 185 | } 186 | 187 | impl IoTarget { 188 | fn from_filename(filename: Option) -> IoTarget { 189 | if let Some(path) = filename { 190 | if path.as_os_str() == "-" { 191 | IoTarget::StdInOut 192 | } else { 193 | IoTarget::File(path) 194 | } 195 | } else { 196 | IoTarget::StdInOut 197 | } 198 | } 199 | 200 | fn read(&self) -> Result> { 201 | let mut buffer = vec![]; 202 | match *self { 203 | IoTarget::StdInOut => std::io::stdin().read_to_end(&mut buffer)?, 204 | IoTarget::File(ref path) => File::open(path)?.read_to_end(&mut buffer)?, 205 | }; 206 | Ok(buffer) 207 | } 208 | 209 | fn write(&self, data: &[u8]) -> Result<()> { 210 | match *self { 211 | IoTarget::StdInOut => std::io::stdout().write_all(data)?, 212 | IoTarget::File(ref path) => File::create(path)?.write_all(data)?, 213 | }; 214 | Ok(()) 215 | } 216 | 217 | fn output(&self, tpe: OutFileType, outname: &Option) -> IoTarget { 218 | if outname.is_some() { 219 | return IoTarget::from_filename(outname.clone()); 220 | } 221 | match *self { 222 | IoTarget::StdInOut => IoTarget::StdInOut, 223 | IoTarget::File(ref path) => { 224 | let mut name = path.clone(); 225 | match tpe { 226 | OutFileType::Packed => { 227 | let mut filename = name 228 | .file_name() 229 | .unwrap_or_else(|| OsStr::new("")) 230 | .to_os_string(); 231 | filename.push(".upk"); 232 | name.set_file_name(filename); 233 | } 234 | OutFileType::Unpacked => { 235 | if name.extension().filter(|&e| e == "upk").is_some() { 236 | name.set_extension(""); 237 | } else { 238 | name.set_extension("bin"); 239 | } 240 | } 241 | OutFileType::Heatmap => { 242 | name.set_extension("heatmap"); 243 | } 244 | } 245 | IoTarget::File(name) 246 | } 247 | } 248 | } 249 | } 250 | 251 | fn print_help(exit_code: i32) -> ! { 252 | eprintln!("Usage:"); 253 | eprintln!(" upkr [-l level(0-9)] [config options] []"); 254 | eprintln!(" upkr -u [config options] []"); 255 | eprintln!(" upkr --heatmap [config options] []"); 256 | eprintln!(" upkr --margin [config options] "); 257 | eprintln!(); 258 | eprintln!(" -l, --level N compression level 0-9"); 259 | eprintln!(" -0, ..., -9 short form for setting compression level"); 260 | eprintln!(" -d, --decompress decompress infile"); 261 | eprintln!(" --heatmap calculate heatmap from compressed file"); 262 | eprintln!(" --raw-cost report raw cost of literals in heatmap"); 263 | #[cfg(feature = "crossterm")] 264 | eprintln!(" --hexdump print heatmap as colored hexdump"); 265 | eprintln!(" --margin calculate margin for overlapped unpacking of a packed file"); 266 | eprintln!(); 267 | eprintln!("When no infile is given, or the infile is '-', read from stdin."); 268 | eprintln!( 269 | "When no outfile is given and reading from stdin, or when outfile is '-', write to stdout." 270 | ); 271 | eprintln!(); 272 | eprintln!("Version: {}", env!("CARGO_PKG_VERSION")); 273 | eprintln!(); 274 | eprintln!("Config presets for specific unpackers:"); 275 | eprintln!( 276 | " --z80 --big-endian-bitstream --invert-bit-encoding --simplified-prob-update -9" 277 | ); 278 | eprintln!( 279 | " --x86 --bitstream --invert-is-match-bit --invert-continue-value-bit --invert-new-offset-bit" 280 | ); 281 | eprintln!( 282 | " --x86b --bitstream --invert-continue-value-bit --no-repeated-offsets -9" 283 | ); 284 | eprintln!(); 285 | eprintln!("Config options (need to match when packing/unpacking):"); 286 | eprintln!(" -b, --bitstream bitstream mode"); 287 | eprintln!(" -p, --parity N use N (2/4) parity contexts"); 288 | eprintln!(" -r, --reverse reverse input & output"); 289 | eprintln!(); 290 | eprintln!("Config options to tailor output to specific optimized unpackers:"); 291 | eprintln!(" --invert-is-match-bit"); 292 | eprintln!(" --invert-new-offset-bit"); 293 | eprintln!(" --invert-continue-value-bit"); 294 | eprintln!(" --invert-bit-encoding"); 295 | eprintln!(" --simplified-prob-update"); 296 | eprintln!(" --big-endian-bitstream (implies --bitstream)"); 297 | eprintln!(" --no-repeated-offsets"); 298 | eprintln!(" --eof-in-length"); 299 | eprintln!(" --max-offset N"); 300 | eprintln!(" --max-length N"); 301 | process::exit(exit_code); 302 | } 303 | -------------------------------------------------------------------------------- /src/match_finder.rs: -------------------------------------------------------------------------------- 1 | use std::collections::BinaryHeap; 2 | use std::ops::Range; 3 | 4 | pub struct MatchFinder { 5 | suffixes: Vec, 6 | rev_suffixes: Vec, 7 | lcp: Vec, 8 | 9 | max_queue_size: usize, 10 | max_matches_per_length: usize, 11 | patience: usize, 12 | max_length_diff: usize, 13 | 14 | queue: BinaryHeap 15 | } 16 | 17 | impl MatchFinder { 18 | pub fn new(data: &[u8]) -> MatchFinder { 19 | let mut suffixes = vec![0i32; data.len()]; 20 | cdivsufsort::sort_in_place(data, &mut suffixes); 21 | 22 | let mut rev_suffixes = vec![0u32; data.len()]; 23 | for (suffix_index, index) in suffixes.iter().enumerate() { 24 | rev_suffixes[*index as usize] = suffix_index as u32; 25 | } 26 | 27 | let mut lcp = vec![0u32; data.len()]; 28 | let mut length = 0usize; 29 | for suffix_index in &rev_suffixes { 30 | if *suffix_index as usize + 1 < suffixes.len() { 31 | let i = suffixes[*suffix_index as usize] as usize; 32 | let j = suffixes[*suffix_index as usize + 1] as usize; 33 | while i + length < data.len() 34 | && j + length < data.len() 35 | && data[i + length] == data[j + length] 36 | { 37 | length += 1; 38 | } 39 | lcp[*suffix_index as usize] = length as u32; 40 | } 41 | length = length.saturating_sub(1); 42 | } 43 | 44 | MatchFinder { 45 | suffixes, 46 | rev_suffixes, 47 | lcp, 48 | max_queue_size: 100, 49 | max_matches_per_length: 5, 50 | patience: 100, 51 | max_length_diff: 2, 52 | queue: BinaryHeap::new() 53 | } 54 | } 55 | 56 | pub fn with_max_queue_size(mut self, v: usize) -> MatchFinder { 57 | self.max_queue_size = v; 58 | self 59 | } 60 | 61 | pub fn with_patience(mut self, v: usize) -> MatchFinder { 62 | self.patience = v; 63 | self 64 | } 65 | 66 | pub fn with_max_matches_per_length(mut self, v: usize) -> MatchFinder { 67 | self.max_matches_per_length = v; 68 | self 69 | } 70 | 71 | pub fn with_max_length_diff(mut self, v: usize) -> MatchFinder { 72 | self.max_length_diff = v; 73 | self 74 | } 75 | 76 | pub fn matches(&mut self, pos: usize) -> Matches { 77 | let index = self.rev_suffixes[pos] as usize; 78 | self.queue.clear(); 79 | let mut matches = Matches { 80 | finder: self, 81 | pos_range: 0..pos, 82 | left_index: index, 83 | left_length: usize::MAX, 84 | right_index: index, 85 | right_length: usize::MAX, 86 | current_length: usize::MAX, 87 | matches_left: 0, 88 | max_length: 0, 89 | }; 90 | 91 | matches.move_left(); 92 | matches.move_right(); 93 | 94 | matches 95 | } 96 | } 97 | 98 | pub struct Matches<'a> { 99 | finder: &'a mut MatchFinder, 100 | pos_range: Range, 101 | left_index: usize, 102 | left_length: usize, 103 | right_index: usize, 104 | right_length: usize, 105 | current_length: usize, 106 | matches_left: usize, 107 | max_length: usize, 108 | } 109 | 110 | #[derive(Debug)] 111 | pub struct Match { 112 | pub pos: usize, 113 | pub length: usize, 114 | } 115 | 116 | impl<'a> Iterator for Matches<'a> { 117 | type Item = Match; 118 | 119 | fn next(&mut self) -> Option { 120 | if self.finder.queue.is_empty() || self.matches_left == 0 { 121 | self.finder.queue.clear(); 122 | self.current_length = self.current_length.saturating_sub(1).min(self.left_length.max(self.right_length)); 123 | self.max_length = self.max_length.max(self.current_length); 124 | if self.current_length < 2 125 | || self.current_length + self.finder.max_length_diff < self.max_length 126 | { 127 | return None; 128 | } 129 | while self.finder.queue.len() < self.finder.max_queue_size 130 | && (self.left_length == self.current_length 131 | || self.right_length == self.current_length) 132 | { 133 | if self.left_length == self.current_length { 134 | self.finder.queue.push(self.finder.suffixes[self.left_index] as usize); 135 | self.move_left(); 136 | } 137 | if self.right_length == self.current_length { 138 | self.finder.queue.push(self.finder.suffixes[self.right_index] as usize); 139 | self.move_right(); 140 | } 141 | } 142 | self.matches_left = self.finder.max_matches_per_length; 143 | } 144 | 145 | self.matches_left = self.matches_left.saturating_sub(1); 146 | 147 | self.finder.queue.pop().map(|pos| Match { 148 | pos, 149 | length: self.current_length, 150 | }) 151 | } 152 | } 153 | 154 | impl<'a> Matches<'a> { 155 | fn move_left(&mut self) { 156 | let mut patience = self.finder.patience; 157 | while self.left_length > 0 && patience > 0 && self.left_index > 0 { 158 | self.left_index -= 1; 159 | self.left_length = self 160 | .left_length 161 | .min(self.finder.lcp[self.left_index] as usize); 162 | if self 163 | .pos_range 164 | .contains(&(self.finder.suffixes[self.left_index] as usize)) 165 | { 166 | return; 167 | } 168 | patience -= 1; 169 | } 170 | self.left_length = 0; 171 | } 172 | 173 | fn move_right(&mut self) { 174 | let mut patience = self.finder.patience; 175 | while self.right_length > 0 176 | && patience > 0 177 | && self.right_index + 1 < self.finder.suffixes.len() 178 | { 179 | self.right_index += 1; 180 | self.right_length = self 181 | .right_length 182 | .min(self.finder.lcp[self.right_index - 1] as usize); 183 | if self 184 | .pos_range 185 | .contains(&(self.finder.suffixes[self.right_index] as usize)) 186 | { 187 | return; 188 | } 189 | patience -= 1; 190 | } 191 | self.right_length = 0; 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /src/parsing_packer.rs: -------------------------------------------------------------------------------- 1 | use std::collections::{HashMap, HashSet}; 2 | use std::mem; 3 | use std::rc::Rc; 4 | 5 | use crate::match_finder::MatchFinder; 6 | use crate::rans::{CostCounter, RansCoder}; 7 | use crate::{ProgressCallback, lz}; 8 | 9 | pub fn pack( 10 | data: &[u8], 11 | level: u8, 12 | config: &crate::Config, 13 | progress_cb: Option, 14 | ) -> Vec { 15 | let mut parse = parse(data, Config::from_level(level), config, progress_cb); 16 | let mut ops = vec![]; 17 | while let Some(link) = parse { 18 | ops.push(link.op); 19 | parse = link.prev.clone(); 20 | } 21 | let mut state = lz::CoderState::new(config); 22 | let mut coder = RansCoder::new(config); 23 | for op in ops.into_iter().rev() { 24 | op.encode(&mut coder, &mut state, config); 25 | } 26 | lz::encode_eof(&mut coder, &mut state, config); 27 | coder.finish() 28 | } 29 | 30 | struct Parse { 31 | prev: Option>, 32 | op: lz::Op, 33 | } 34 | 35 | struct Arrival { 36 | parse: Option>, 37 | state: lz::CoderState, 38 | cost: f64, 39 | } 40 | 41 | type Arrivals = HashMap>; 42 | 43 | fn parse( 44 | data: &[u8], 45 | config: Config, 46 | encoding_config: &crate::Config, 47 | mut progress_cb: Option, 48 | ) -> Option> { 49 | let mut match_finder = MatchFinder::new(data) 50 | .with_max_queue_size(config.max_queue_size) 51 | .with_patience(config.patience) 52 | .with_max_matches_per_length(config.max_matches_per_length) 53 | .with_max_length_diff(config.max_length_diff); 54 | let mut near_matches = [usize::MAX; 1024]; 55 | let mut last_seen = [usize::MAX; 256]; 56 | 57 | let max_arrivals = config.max_arrivals; 58 | 59 | let mut arrivals: Arrivals = HashMap::new(); 60 | fn sort_arrivals(vec: &mut Vec, max_arrivals: usize) { 61 | if max_arrivals == 0 { 62 | return; 63 | } 64 | vec.sort_by(|a, b| { 65 | a.cost 66 | .partial_cmp(&b.cost) 67 | .unwrap_or(std::cmp::Ordering::Equal) 68 | }); 69 | let mut seen_offsets = HashSet::new(); 70 | let mut remaining = Vec::new(); 71 | for arr in mem::take(vec) { 72 | if seen_offsets.insert(arr.state.last_offset()) { 73 | if vec.len() < max_arrivals { 74 | vec.push(arr); 75 | } 76 | } else { 77 | remaining.push(arr); 78 | } 79 | } 80 | for arr in remaining { 81 | if vec.len() >= max_arrivals { 82 | break; 83 | } 84 | vec.push(arr); 85 | } 86 | } 87 | 88 | fn add_arrival(arrivals: &mut Arrivals, pos: usize, arrival: Arrival, max_arrivals: usize) { 89 | let vec = arrivals.entry(pos).or_default(); 90 | if max_arrivals == 0 { 91 | if vec.is_empty() { 92 | vec.push(arrival); 93 | } else if vec[0].cost > arrival.cost { 94 | vec[0] = arrival; 95 | } 96 | return; 97 | } 98 | vec.push(arrival); 99 | if vec.len() > max_arrivals * 2 { 100 | sort_arrivals(vec, max_arrivals); 101 | } 102 | } 103 | fn add_match( 104 | arrivals: &mut Arrivals, 105 | cost_counter: &mut CostCounter, 106 | pos: usize, 107 | offset: usize, 108 | mut length: usize, 109 | arrival: &Arrival, 110 | max_arrivals: usize, 111 | config: &crate::Config, 112 | ) { 113 | if length < config.min_length() { 114 | return; 115 | } 116 | length = length.min(config.max_length); 117 | cost_counter.reset(); 118 | let mut state = arrival.state.clone(); 119 | let op = lz::Op::Match { 120 | offset: offset as u32, 121 | len: length as u32, 122 | }; 123 | op.encode(cost_counter, &mut state, config); 124 | add_arrival( 125 | arrivals, 126 | pos + length, 127 | Arrival { 128 | parse: Some(Rc::new(Parse { 129 | prev: arrival.parse.clone(), 130 | op, 131 | })), 132 | state, 133 | cost: arrival.cost + cost_counter.cost(), 134 | }, 135 | max_arrivals, 136 | ); 137 | } 138 | add_arrival( 139 | &mut arrivals, 140 | 0, 141 | Arrival { 142 | parse: None, 143 | state: lz::CoderState::new(encoding_config), 144 | cost: 0.0, 145 | }, 146 | max_arrivals, 147 | ); 148 | 149 | let cost_counter = &mut CostCounter::new(encoding_config); 150 | let mut best_per_offset = HashMap::new(); 151 | for pos in 0..data.len() { 152 | let match_length = |offset: usize| { 153 | data[pos..] 154 | .iter() 155 | .zip(data[(pos - offset)..].iter()) 156 | .take_while(|(a, b)| a == b) 157 | .count() 158 | }; 159 | 160 | let here_arrivals = if let Some(mut arr) = arrivals.remove(&pos) { 161 | sort_arrivals(&mut arr, max_arrivals); 162 | arr 163 | } else { 164 | continue; 165 | }; 166 | best_per_offset.clear(); 167 | let mut best_cost = f64::MAX; 168 | for arrival in &here_arrivals { 169 | best_cost = best_cost.min(arrival.cost); 170 | let per_offset = best_per_offset 171 | .entry(arrival.state.last_offset()) 172 | .or_insert(f64::MAX); 173 | *per_offset = per_offset.min(arrival.cost); 174 | } 175 | 176 | 'arrival_loop: for arrival in here_arrivals { 177 | if arrival.cost 178 | > (best_cost + config.max_cost_delta).min( 179 | *best_per_offset.get(&arrival.state.last_offset()).unwrap() 180 | + config.max_offset_cost_delta, 181 | ) 182 | { 183 | continue; 184 | } 185 | let mut found_last_offset = false; 186 | let mut closest_match = None; 187 | for m in match_finder.matches(pos) { 188 | closest_match = Some(closest_match.unwrap_or(0).max(m.pos)); 189 | let offset = pos - m.pos; 190 | if offset <= encoding_config.max_offset { 191 | found_last_offset |= offset as u32 == arrival.state.last_offset(); 192 | add_match( 193 | &mut arrivals, 194 | cost_counter, 195 | pos, 196 | offset, 197 | m.length, 198 | &arrival, 199 | max_arrivals, 200 | encoding_config, 201 | ); 202 | if m.length >= config.greedy_size { 203 | break 'arrival_loop; 204 | } 205 | } 206 | } 207 | 208 | let mut near_matches_left = config.num_near_matches; 209 | let mut match_pos = last_seen[data[pos] as usize]; 210 | while near_matches_left > 0 211 | && match_pos != usize::MAX 212 | && closest_match.iter().all(|p| *p < match_pos) 213 | { 214 | let offset = pos - match_pos; 215 | if offset > encoding_config.max_offset { 216 | break; 217 | } 218 | let length = match_length(offset); 219 | assert!(length > 0); 220 | add_match( 221 | &mut arrivals, 222 | cost_counter, 223 | pos, 224 | offset, 225 | length, 226 | &arrival, 227 | max_arrivals, 228 | encoding_config, 229 | ); 230 | found_last_offset |= offset as u32 == arrival.state.last_offset(); 231 | if offset < near_matches.len() { 232 | match_pos = near_matches[match_pos % near_matches.len()]; 233 | } 234 | near_matches_left -= 1; 235 | } 236 | 237 | if !found_last_offset && arrival.state.last_offset() > 0 { 238 | let offset = arrival.state.last_offset() as usize; 239 | let length = match_length(offset); 240 | if length > 0 { 241 | add_match( 242 | &mut arrivals, 243 | cost_counter, 244 | pos, 245 | offset, 246 | length, 247 | &arrival, 248 | max_arrivals, 249 | encoding_config, 250 | ); 251 | } 252 | } 253 | 254 | cost_counter.reset(); 255 | let mut state = arrival.state; 256 | let op = lz::Op::Literal(data[pos]); 257 | op.encode(cost_counter, &mut state, encoding_config); 258 | add_arrival( 259 | &mut arrivals, 260 | pos + 1, 261 | Arrival { 262 | parse: Some(Rc::new(Parse { 263 | prev: arrival.parse, 264 | op, 265 | })), 266 | state, 267 | cost: arrival.cost + cost_counter.cost(), 268 | }, 269 | max_arrivals, 270 | ); 271 | } 272 | near_matches[pos % near_matches.len()] = last_seen[data[pos] as usize]; 273 | last_seen[data[pos] as usize] = pos; 274 | if let Some(ref mut cb) = progress_cb { 275 | cb(pos + 1); 276 | } 277 | } 278 | arrivals.remove(&data.len()).unwrap()[0].parse.clone() 279 | } 280 | 281 | struct Config { 282 | max_arrivals: usize, 283 | max_cost_delta: f64, 284 | max_offset_cost_delta: f64, 285 | num_near_matches: usize, 286 | greedy_size: usize, 287 | max_queue_size: usize, 288 | patience: usize, 289 | max_matches_per_length: usize, 290 | max_length_diff: usize, 291 | } 292 | 293 | impl Config { 294 | fn from_level(level: u8) -> Config { 295 | let max_arrivals = match level { 296 | 0..=1 => 0, 297 | 2 => 2, 298 | 3 => 4, 299 | 4 => 8, 300 | 5 => 16, 301 | 6 => 32, 302 | 7 => 64, 303 | 8 => 96, 304 | _ => 128, 305 | }; 306 | let (max_cost_delta, max_offset_cost_delta) = match level { 307 | 0..=4 => (16.0, 0.0), 308 | 5..=8 => (16.0, 4.0), 309 | _ => (16.0, 8.0), 310 | }; 311 | let num_near_matches = level.saturating_sub(1) as usize; 312 | let greedy_size = 4 + level as usize * level as usize * 3; 313 | let max_length_diff = match level { 314 | 0..=1 => 0, 315 | 2..=3 => 1, 316 | 4..=5 => 2, 317 | 6..=7 => 3, 318 | _ => 4, 319 | }; 320 | Config { 321 | max_arrivals, 322 | max_cost_delta, 323 | max_offset_cost_delta, 324 | num_near_matches, 325 | greedy_size, 326 | max_queue_size: level as usize * 100, 327 | patience: level as usize * 100, 328 | max_matches_per_length: level as usize, 329 | max_length_diff, 330 | } 331 | } 332 | } 333 | -------------------------------------------------------------------------------- /src/rans.rs: -------------------------------------------------------------------------------- 1 | use crate::{context_state::Context, Config}; 2 | use thiserror::Error; 3 | 4 | pub const PROB_BITS: u32 = 8; 5 | pub const ONE_PROB: u32 = 1 << PROB_BITS; 6 | 7 | pub trait EntropyCoder { 8 | fn encode_bit(&mut self, bit: bool, prob: u16); 9 | 10 | fn encode_with_context(&mut self, bit: bool, context: &mut Context) { 11 | self.encode_bit(bit, context.prob()); 12 | context.update(bit); 13 | } 14 | } 15 | 16 | pub struct RansCoder { 17 | bits: Vec, 18 | use_bitstream: bool, 19 | bitstream_is_big_endian: bool, 20 | invert_bit_encoding: bool, 21 | } 22 | 23 | impl EntropyCoder for RansCoder { 24 | fn encode_bit(&mut self, bit: bool, prob: u16) { 25 | assert!(prob < 32768); 26 | self.bits 27 | .push(prob | (((bit ^ self.invert_bit_encoding) as u16) << 15)); 28 | } 29 | } 30 | 31 | impl RansCoder { 32 | pub fn new(config: &Config) -> RansCoder { 33 | RansCoder { 34 | bits: Vec::new(), 35 | use_bitstream: config.use_bitstream, 36 | bitstream_is_big_endian: config.bitstream_is_big_endian, 37 | invert_bit_encoding: config.invert_bit_encoding, 38 | } 39 | } 40 | 41 | pub fn finish(self) -> Vec { 42 | let mut buffer = vec![]; 43 | let l_bits: u32 = if self.use_bitstream { 15 } else { 12 }; 44 | let mut state = 1 << l_bits; 45 | 46 | let mut byte = 0u8; 47 | let mut bit = if self.bitstream_is_big_endian { 0 } else { 8 }; 48 | let mut flush_state: Box = if self.use_bitstream { 49 | if self.bitstream_is_big_endian { 50 | Box::new(|state: &mut u32| { 51 | byte |= ((*state & 1) as u8) << bit; 52 | bit += 1; 53 | if bit == 8 { 54 | buffer.push(byte); 55 | byte = 0; 56 | bit = 0; 57 | } 58 | *state >>= 1; 59 | }) 60 | } else { 61 | Box::new(|state: &mut u32| { 62 | bit -= 1; 63 | byte |= ((*state & 1) as u8) << bit; 64 | if bit == 0 { 65 | buffer.push(byte); 66 | byte = 0; 67 | bit = 8; 68 | } 69 | *state >>= 1; 70 | }) 71 | } 72 | } else { 73 | Box::new(|state: &mut u32| { 74 | buffer.push(*state as u8); 75 | *state >>= 8; 76 | }) 77 | }; 78 | 79 | let num_flush_bits = if self.use_bitstream { 1 } else { 8 }; 80 | let max_state_factor: u32 = 1 << (l_bits + num_flush_bits - PROB_BITS); 81 | for step in self.bits.into_iter().rev() { 82 | let prob = step as u32 & 32767; 83 | let (start, prob) = if step & 32768 != 0 { 84 | (0, prob) 85 | } else { 86 | (prob, ONE_PROB - prob) 87 | }; 88 | let max_state = max_state_factor * prob; 89 | while state >= max_state { 90 | flush_state(&mut state); 91 | } 92 | state = ((state / prob) << PROB_BITS) + (state % prob) + start; 93 | } 94 | 95 | while state > 0 { 96 | flush_state(&mut state); 97 | } 98 | 99 | drop(flush_state); 100 | 101 | if self.use_bitstream && byte != 0 { 102 | buffer.push(byte); 103 | } 104 | 105 | buffer.reverse(); 106 | buffer 107 | } 108 | } 109 | 110 | pub struct CostCounter { 111 | cost: f64, 112 | log2_table: Vec, 113 | invert_bit_encoding: bool, 114 | } 115 | 116 | impl CostCounter { 117 | pub fn new(config: &Config) -> CostCounter { 118 | let log2_table = (0..ONE_PROB) 119 | .map(|prob| { 120 | let inv_prob = ONE_PROB as f64 / prob as f64; 121 | inv_prob.log2() 122 | }) 123 | .collect(); 124 | CostCounter { 125 | cost: 0.0, 126 | log2_table, 127 | invert_bit_encoding: config.invert_bit_encoding, 128 | } 129 | } 130 | 131 | pub fn cost(&self) -> f64 { 132 | self.cost 133 | } 134 | 135 | pub fn reset(&mut self) { 136 | self.cost = 0.0; 137 | } 138 | } 139 | 140 | impl EntropyCoder for CostCounter { 141 | fn encode_bit(&mut self, bit: bool, prob: u16) { 142 | let prob = if bit ^ self.invert_bit_encoding { 143 | prob as u32 144 | } else { 145 | ONE_PROB - prob as u32 146 | }; 147 | self.cost += self.log2_table[prob as usize]; 148 | } 149 | } 150 | 151 | #[derive(Clone)] 152 | pub struct RansDecoder<'a> { 153 | data: &'a [u8], 154 | pos: usize, 155 | state: u32, 156 | use_bitstream: bool, 157 | byte: u8, 158 | bits_left: u8, 159 | invert_bit_encoding: bool, 160 | bitstream_is_big_endian: bool, 161 | } 162 | 163 | const PROB_MASK: u32 = ONE_PROB - 1; 164 | 165 | #[derive(Debug, Error)] 166 | #[error("Unexpected end of input")] 167 | pub struct UnexpectedEOF; 168 | 169 | impl<'a> RansDecoder<'a> { 170 | pub fn new(data: &'a [u8], config: &Config) -> Result, UnexpectedEOF> { 171 | let mut decoder = RansDecoder { 172 | data, 173 | pos: 0, 174 | state: 0, 175 | use_bitstream: config.use_bitstream, 176 | byte: 0, 177 | bits_left: 0, 178 | invert_bit_encoding: config.invert_bit_encoding, 179 | bitstream_is_big_endian: config.bitstream_is_big_endian, 180 | }; 181 | decoder.refill()?; 182 | Ok(decoder) 183 | } 184 | 185 | pub fn pos(&self) -> usize { 186 | self.pos 187 | } 188 | 189 | pub fn decode_with_context(&mut self, context: &mut Context) -> Result { 190 | let bit = self.decode_bit(context.prob())?; 191 | context.update(bit); 192 | Ok(bit) 193 | } 194 | 195 | fn refill(&mut self) -> Result<(), UnexpectedEOF> { 196 | if self.use_bitstream { 197 | while self.state < 32768 { 198 | if self.bits_left == 0 { 199 | if self.pos >= self.data.len() { 200 | return Err(UnexpectedEOF); 201 | } 202 | self.byte = self.data[self.pos]; 203 | self.pos += 1; 204 | self.bits_left = 8; 205 | } 206 | if self.bitstream_is_big_endian { 207 | self.state = (self.state << 1) | (self.byte >> 7) as u32; 208 | self.byte <<= 1; 209 | } else { 210 | self.state = (self.state << 1) | (self.byte & 1) as u32; 211 | self.byte >>= 1; 212 | } 213 | self.bits_left -= 1; 214 | } 215 | } else { 216 | while self.state < 4096 { 217 | if self.pos >= self.data.len() { 218 | return Err(UnexpectedEOF); 219 | } 220 | self.state = (self.state << 8) | self.data[self.pos] as u32; 221 | self.pos += 1; 222 | } 223 | } 224 | Ok(()) 225 | } 226 | 227 | pub fn decode_bit(&mut self, prob: u16) -> Result { 228 | self.refill()?; 229 | 230 | let prob = prob as u32; 231 | 232 | let bit = (self.state & PROB_MASK) < prob; 233 | 234 | let (start, prob) = if bit { 235 | (0, prob) 236 | } else { 237 | (prob, ONE_PROB - prob) 238 | }; 239 | self.state = prob * (self.state >> PROB_BITS) + (self.state & PROB_MASK) - start; 240 | 241 | Ok(bit ^ self.invert_bit_encoding) 242 | } 243 | 244 | pub fn cost(&self, prev: &RansDecoder) -> f32 { 245 | f32::log2(prev.state as f32) - f32::log2(self.state as f32) 246 | + (self.pos - prev.pos) as f32 * 8. 247 | } 248 | } 249 | -------------------------------------------------------------------------------- /z80_unpacker/.gitignore: -------------------------------------------------------------------------------- 1 | *.bin 2 | *.tap 3 | *.lst 4 | -------------------------------------------------------------------------------- /z80_unpacker/Makefile: -------------------------------------------------------------------------------- 1 | all: unpack.bin example/example.sna 2 | 3 | # binary is positioned from ORG 0, not usable, just assembling to verify the syntax 4 | unpack.bin: unpack.asm 5 | sjasmplus --msg=war --lst --lstlab=sort --raw=unpack.bin unpack.asm 6 | 7 | example/example.sna: unpack.asm example/example.asm 8 | cd example && sjasmplus --msg=war --lst --lstlab=sort example.asm 9 | 10 | clean: 11 | $(RM) unpack.bin unpack.lst example/example.sna example/example.lst 12 | -------------------------------------------------------------------------------- /z80_unpacker/example/example.asm: -------------------------------------------------------------------------------- 1 | ;; Example using upkr depacker for screens slideshow 2 | OPT --syntax=abf 3 | DEVICE ZXSPECTRUM48,$8FFF 4 | 5 | ORG $9000 6 | ;; forward example data 7 | compressed_scr_files.fwd: ; border color byte + upkr-packed .scr file 8 | DB 1 9 | INCBIN "screens/Grongy - ZX Spectrum (2022).scr.upk" 10 | DB 7 11 | INCBIN "screens/Schafft - Poison (2017).scr.upk" 12 | DB 0 13 | INCBIN "screens/diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr.upk" 14 | DB 6 15 | INCBIN "screens/diver - Back to Bjork (2015).scr.upk" 16 | .e: 17 | ;; backward example data (unpacker goes from the end of the data!) 18 | compressed_scr_files.rwd.e: EQU $-1 ; the final IX will point one byte ahead of "$" here 19 | INCBIN "screens.reversed/diver - Back to Bjork (2015).scr.upk" 20 | DB 6 21 | INCBIN "screens.reversed/diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr.upk" 22 | DB 0 23 | INCBIN "screens.reversed/Schafft - Poison (2017).scr.upk" 24 | DB 7 25 | INCBIN "screens.reversed/Grongy - ZX Spectrum (2022).scr.upk" 26 | compressed_scr_files.rwd: ; border color byte + upkr-packed .scr file (backward) 27 | DB 1 28 | 29 | start: 30 | di 31 | ; OPT --zxnext 32 | ; nextreg 7,3 ; ZX Next: switch to 28Mhz 33 | 34 | ;;; FORWARD packed/unpacked data demo 35 | ld ix,compressed_scr_files.fwd 36 | .slideshow_loop.fwd: 37 | ; set BORDER for next image 38 | ld a,(ix) 39 | inc ix 40 | out (254),a 41 | ; call unpack of next image directly into VRAM 42 | ld de,$4000 ; target VRAM 43 | exx 44 | ; IX = packed data, DE' = destination ($4000) 45 | ; returned IX will point right after the packed data 46 | call fwd.upkr.unpack 47 | ; do some busy loop with CPU to delay between images 48 | call delay 49 | ; check if all images were displayed, loop around from first one then 50 | ld a,ixl 51 | cp low compressed_scr_files.fwd.e 52 | jr nz,.slideshow_loop.fwd 53 | 54 | ;;; BACKWARD packed/unpacked data demo 55 | ld ix,compressed_scr_files.rwd 56 | .slideshow_loop.rwd: 57 | ; set BORDER for next image 58 | ld a,(ix) 59 | dec ix 60 | out (254),a 61 | ; call unpack of next image directly into VRAM 62 | ld de,$5AFF ; target VRAM 63 | exx 64 | ; IX = packed data, DE' = destination 65 | ; returned IX will point right ahead of the packed data 66 | call rwd.upkr.unpack 67 | ; do some busy loop with CPU to delay between images 68 | call delay 69 | ; check if all images were displayed, loop around from first one then 70 | ld a,ixl 71 | cp low compressed_scr_files.rwd.e 72 | jr nz,.slideshow_loop.rwd 73 | 74 | jr start 75 | 76 | delay: 77 | ld bc,$AA00 78 | .delay: 79 | .8 ex (sp),ix 80 | dec c 81 | jr nz,.delay 82 | djnz .delay 83 | ret 84 | 85 | ; include the depacker library, optionally putting probs array buffer near end of RAM 86 | DEFINE UPKR_PROBS_ORIGIN $FA00 ; if not defined, array will be put after unpack code 87 | 88 | MODULE fwd 89 | INCLUDE "../unpack.asm" 90 | ENDMODULE 91 | 92 | MODULE rwd 93 | DEFINE BACKWARDS_UNPACK ; defined to build backwards unpack 94 | ; initial IX points at last byte of compressed data 95 | ; initial DE' points at last byte of unpacked data 96 | 97 | INCLUDE "../unpack.asm" 98 | ENDMODULE 99 | 100 | SAVESNA "example.sna",start 101 | -------------------------------------------------------------------------------- /z80_unpacker/example/example.sna: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/example.sna -------------------------------------------------------------------------------- /z80_unpacker/example/screens.reversed/Grongy - ZX Spectrum (2022).scr.upk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens.reversed/Grongy - ZX Spectrum (2022).scr.upk -------------------------------------------------------------------------------- /z80_unpacker/example/screens.reversed/Schafft - Poison (2017).scr.upk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens.reversed/Schafft - Poison (2017).scr.upk -------------------------------------------------------------------------------- /z80_unpacker/example/screens.reversed/diver - Back to Bjork (2015).scr.upk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens.reversed/diver - Back to Bjork (2015).scr.upk -------------------------------------------------------------------------------- /z80_unpacker/example/screens.reversed/diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr.upk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens.reversed/diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr.upk -------------------------------------------------------------------------------- /z80_unpacker/example/screens/Grongy - ZX Spectrum (2022).scr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens/Grongy - ZX Spectrum (2022).scr -------------------------------------------------------------------------------- /z80_unpacker/example/screens/Grongy - ZX Spectrum (2022).scr.upk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens/Grongy - ZX Spectrum (2022).scr.upk -------------------------------------------------------------------------------- /z80_unpacker/example/screens/Schafft - Poison (2017).scr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens/Schafft - Poison (2017).scr -------------------------------------------------------------------------------- /z80_unpacker/example/screens/Schafft - Poison (2017).scr.upk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens/Schafft - Poison (2017).scr.upk -------------------------------------------------------------------------------- /z80_unpacker/example/screens/diver - Back to Bjork (2015).scr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens/diver - Back to Bjork (2015).scr -------------------------------------------------------------------------------- /z80_unpacker/example/screens/diver - Back to Bjork (2015).scr.upk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens/diver - Back to Bjork (2015).scr.upk -------------------------------------------------------------------------------- /z80_unpacker/example/screens/diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens/diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr -------------------------------------------------------------------------------- /z80_unpacker/example/screens/diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr.upk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/exoticorn/upkr/a9e56d9d50d1f64cab7590f2c584a12b173e7715/z80_unpacker/example/screens/diver - Mercenary 4. The Heaven's Devil (2014) (Forever 2014 Olympic Edition, 1).scr.upk -------------------------------------------------------------------------------- /z80_unpacker/readme.txt: -------------------------------------------------------------------------------- 1 | Z80 asm implementation of C unpacker, code-size focused (not performance). 2 | 3 | **ONLY BITSTREAM** variant is currently supported, make sure to use "-b" in packer. 4 | 5 | The project is expected to further evolve, including possible changes to binary format, this is 6 | initial version of Z80 unpacker to explore if/how it works and how it can be improved further. 7 | 8 | (copy full packer+depacker source to your project if you plan to use it, as future revisions 9 | may be incompatible with files you will produce with current version) 10 | 11 | Asm syntax is z00m's sjasmplus: https://github.com/z00m128/sjasmplus 12 | 13 | Backward direction unpacker added as compile-time option, see example for both forward/backward 14 | depacker in action. 15 | 16 | The packed/unpacked data-overlap has to be tested per-case, in worst case the packed data 17 | may need even more than 7 bytes to unpack final byte, but usually 1-4 bytes may suffice. 18 | 19 | TODO: 20 | - build bigger corpus of test data to benchmark future changes in algorithm/format (example and zx48.rom was used to do initial tests) 21 | - maybe try to beat double-loop `decode_number` with different encoding format 22 | - (@ped7g) Z80N version of unpacker for ZX Next devs 23 | - (@exoticorn) add Z80 specific packer (to avoid confusion with original MicroW8 variant), and land it all to master branch, maybe in "z80" directory or something? (and overall decide how to organise+merge this upstream into main repo) 24 | - (@exoticorn) add to packer output with possible packed/unpacked region overlap 25 | 26 | DONE: 27 | * review non-bitstream variant, if it's feasible to try to implement it with Z80 28 | - Ped7g: IMHO nope, the 12b x 8b MUL code would probably quickly cancel any gains from the simpler state update 29 | * review first implementation to identify weak spots where the implementation can be shorter+faster 30 | with acceptable small changes to the format 31 | - Ped7g: the decode_bit settled down and now doesn't feel so confused and redundant, the code seems pretty on point to me, no obvious simplification from format change 32 | - Ped7g: the decode_number double-loop is surprisingly resilient, especially in terms of code size I failed to beat it, speed wise only negligible gains 33 | -------------------------------------------------------------------------------- /z80_unpacker/unpack.asm: -------------------------------------------------------------------------------- 1 | ;; https://github.com/exoticorn/upkr/blob/z80/c_unpacker/unpack.c - original C implementation 2 | ;; C source in comments ahead of asm - the C macros are removed to keep only bitstream variant 3 | ;; 4 | ;; initial version by Peter "Ped" Helcmanovsky (C) 2022, licensed same as upkr project ("unlicensed") 5 | ;; to assemble use z00m's sjasmplus: https://github.com/z00m128/sjasmplus 6 | ;; 7 | ;; you can define UPKR_PROBS_ORIGIN to specific 256 byte aligned address for probs array (320 bytes), 8 | ;; otherwise it will be positioned after the unpacker code (256 aligned) 9 | ;; 10 | ;; public API: 11 | ;; 12 | ;; upkr.unpack 13 | ;; IN: IX = packed data, DE' (shadow DE) = destination 14 | ;; OUT: IX = after packed data 15 | ;; modifies: all registers except IY, requires 10 bytes of stack space 16 | ;; 17 | 18 | ; DEFINE BACKWARDS_UNPACK ; uncomment to build backwards depacker (write_ptr--, upkr_data_ptr--) 19 | ; initial IX points at last byte of compressed data 20 | ; initial DE' points at last byte of unpacked data 21 | 22 | ; DEFINE UPKR_UNPACK_SPEED ; uncomment to get larger but faster unpack routine 23 | 24 | ; code size hint: if you put probs array just ahead of BASIC entry point, you will get BC 25 | ; initialised to probs.e by BASIC `USR` command and you can remove it from unpack init (-3B) 26 | 27 | OPT push reset --syntax=abf 28 | MODULE upkr 29 | 30 | NUMBER_BITS EQU 16+15 ; context-bits per offset/length (16+15 for 16bit offsets/pointers) 31 | ; numbers (offsets/lengths) are encoded like: 1a1b1c1d1e0 = 0000'0000'001e'dbca 32 | 33 | /* 34 | u8* upkr_data_ptr; 35 | u8 upkr_probs[1 + 255 + 1 + 2*32 + 2*32]; 36 | u16 upkr_state; 37 | u8 upkr_current_byte; 38 | int upkr_bits_left; 39 | 40 | int upkr_unpack(void* destination, void* compressed_data) { 41 | upkr_data_ptr = (u8*)compressed_data; 42 | upkr_state = 0; 43 | upkr_bits_left = 0; 44 | for(int i = 0; i < sizeof(upkr_probs); ++i) 45 | upkr_probs[i] = 128; 46 | 47 | u8* write_ptr = (u8*)destination; 48 | 49 | int prev_was_match = 0; 50 | int offset = 0; 51 | for(;;) { 52 | if(upkr_decode_bit(0)) { 53 | if(prev_was_match || upkr_decode_bit(256)) { 54 | offset = upkr_decode_length(257) - 1; 55 | if(offset == 0) { 56 | break; 57 | } 58 | } 59 | int length = upkr_decode_length(257 + 64); 60 | while(length--) { 61 | *write_ptr = write_ptr[-offset]; 62 | ++write_ptr; 63 | } 64 | prev_was_match = 1; 65 | } else { 66 | int byte = 1; 67 | while(byte < 256) { 68 | int bit = upkr_decode_bit(byte); 69 | byte = (byte << 1) + bit; 70 | } 71 | *write_ptr++ = byte; 72 | prev_was_match = 0; 73 | } 74 | } 75 | 76 | return write_ptr - (u8*)destination; 77 | } 78 | */ 79 | ; IN: IX = compressed_data, DE' = destination 80 | unpack: 81 | ; ** reset probs to 0x80, also reset HL (state) to zero, and set BC to probs+context 0 82 | ld hl,probs.c>>1 83 | ld bc,probs.e 84 | ld a,$80 85 | .reset_probs: 86 | dec bc 87 | ld (bc),a ; will overwrite one extra byte after the array because of odd length 88 | dec bc 89 | ld (bc),a 90 | dec l 91 | jr nz,.reset_probs 92 | exa 93 | ; BC = probs (context_index 0), state HL = 0, A' = 0x80 (no source bits left in upkr_current_byte) 94 | 95 | ; ** main loop to decompress data 96 | ; D = prev_was_match = uninitialised, literal is expected first => will reset D to "false" 97 | ; values for false/true of prev_was_match are: false = high(probs), true = 1 + high(probs) 98 | .decompress_data: 99 | ld c,0 100 | call decode_bit ; if(upkr_decode_bit(0)) 101 | jr c,.copy_chunk 102 | 103 | ; * extract byte from compressed data (literal) 104 | inc c ; C = byte = 1 (and also context_index) 105 | .decode_byte: 106 | call decode_bit ; bit = upkr_decode_bit(byte); 107 | rl c ; byte = (byte << 1) + bit; 108 | jr nc,.decode_byte ; while(byte < 256) 109 | ld a,c 110 | exx 111 | ld (de),a ; *write_ptr++ = byte; 112 | IFNDEF BACKWARDS_UNPACK : inc de : ELSE : dec de : ENDIF 113 | exx 114 | ld d,b ; prev_was_match = false 115 | jr .decompress_data 116 | 117 | ; * copy chunk of already decompressed data (match) 118 | .copy_chunk: 119 | ld a,b 120 | inc b ; context_index = 256 121 | ; if(prev_was_match || upkr_decode_bit(256)) { 122 | ; offset = upkr_decode_length(257) - 1; 123 | ; if (0 == offset) break; 124 | ; } 125 | cp d ; CF = prev_was_match 126 | call nc,decode_bit ; if not prev_was_match, then upkr_decode_bit(256) 127 | jr nc,.keep_offset ; if neither, keep old offset 128 | call decode_number ; context_index is already 257-1 as needed by decode_number 129 | dec de ; offset = upkr_decode_length(257) - 1; 130 | ld a,d 131 | or e 132 | ret z ; if(offset == 0) break 133 | ld (.offset),de 134 | .keep_offset: 135 | ; int length = upkr_decode_length(257 + 64); 136 | ; while(length--) { 137 | ; *write_ptr = write_ptr[-offset]; 138 | ; ++write_ptr; 139 | ; } 140 | ; prev_was_match = 1; 141 | ld c,low(257 + NUMBER_BITS - 1) ; context_index to second "number" set for lengths decoding 142 | call decode_number ; length = upkr_decode_length(257 + 64); 143 | push de 144 | exx 145 | IFNDEF BACKWARDS_UNPACK 146 | ; forward unpack (write_ptr++, upkr_data_ptr++) 147 | ld h,d ; DE = write_ptr 148 | ld l,e 149 | .offset+*: ld bc,0 150 | sbc hl,bc ; CF=0 from decode_number ; HL = write_ptr - offset 151 | pop bc ; BC = length 152 | ldir 153 | ELSE 154 | ; backward unpack (write_ptr--, upkr_data_ptr--) 155 | .offset+*: ld hl,0 156 | add hl,de ; HL = write_ptr + offset 157 | pop bc ; BC = length 158 | lddr 159 | ENDIF 160 | exx 161 | ld d,b ; prev_was_match = true 162 | djnz .decompress_data ; adjust context_index back to 0..255 range, go to main loop 163 | 164 | /* 165 | int upkr_decode_bit(int context_index) { 166 | while(upkr_state < 32768) { 167 | if(upkr_bits_left == 0) { 168 | upkr_current_byte = *upkr_data_ptr++; 169 | upkr_bits_left = 8; 170 | } 171 | upkr_state = (upkr_state << 1) + (upkr_current_byte >> 7); 172 | upkr_current_byte <<= 1; 173 | --upkr_bits_left; 174 | } 175 | 176 | int prob = upkr_probs[context_index]; 177 | int bit = (upkr_state & 255) >= prob ? 1 : 0; 178 | 179 | int prob_offset = 16; 180 | int state_offset = 0; 181 | int state_scale = prob; 182 | if(bit) { 183 | state_offset = -prob; 184 | state_scale = 256 - prob; 185 | prob_offset = 0; 186 | } 187 | upkr_state = state_offset + state_scale * (upkr_state >> 8) + (upkr_state & 255); 188 | upkr_probs[context_index] = prob_offset + prob - ((prob + 8) >> 4); 189 | 190 | return bit; 191 | } 192 | */ 193 | inc_c_decode_bit: 194 | ; ++low(context_index) before decode_bit (to get -1B by two calls in decode_number) 195 | inc c 196 | decode_bit: 197 | ; HL = upkr_state 198 | ; IX = upkr_data_ptr 199 | ; BC = probs+context_index 200 | ; A' = upkr_current_byte (!!! init to 0x80 at start, not 0x00) 201 | ; preserves DE 202 | ; ** while (state < 32768) - initial check 203 | push de 204 | bit 7,h 205 | jr nz,.state_b15_set 206 | exa 207 | ; ** while body 208 | .state_b15_zero: 209 | ; HL = upkr_state 210 | ; IX = upkr_data_ptr 211 | ; A = upkr_current_byte (init to 0x80 at start, not 0x00) 212 | add a,a ; upkr_current_byte <<= 1; // and testing if(upkr_bits_left == 0) 213 | jr nz,.has_bit ; CF=data, ZF=0 -> some bits + stop bit still available 214 | ; CF=1 (by stop bit) 215 | ld a,(ix) 216 | IFNDEF BACKWARDS_UNPACK : inc ix : ELSE : dec ix : ENDIF ; upkr_current_byte = *upkr_data_ptr++; 217 | adc a,a ; CF=data, b0=1 as new stop bit 218 | .has_bit: 219 | adc hl,hl ; upkr_state = (upkr_state << 1) + (upkr_current_byte >> 7); 220 | jp p,.state_b15_zero ; while (state < 32768) 221 | exa 222 | ; ** set "bit" 223 | .state_b15_set: 224 | ld a,(bc) ; A = upkr_probs[context_index] 225 | dec a ; prob is in ~7..249 range, never zero, safe to -1 226 | cp l ; CF = bit = prob-1 < (upkr_state & 255) <=> prob <= (upkr_state & 255) 227 | inc a 228 | ; ** adjust state 229 | push bc 230 | ld c,l ; C = (upkr_state & 255); (preserving the value) 231 | push af 232 | jr nc,.bit_is_0 233 | neg ; A = -prob == (256-prob), CF=1 preserved 234 | .bit_is_0: 235 | ld d,0 236 | ld e,a ; DE = state_scale ; prob || (256-prob) 237 | ld l,d ; H:L = (upkr_state>>8) : 0 238 | 239 | IFNDEF UPKR_UNPACK_SPEED 240 | 241 | ;; looped MUL for minimum unpack size 242 | ld b,8 ; counter 243 | .mulLoop: 244 | add hl,hl 245 | jr nc,.mul0 246 | add hl,de 247 | .mul0: 248 | djnz .mulLoop ; until HL = state_scale * (upkr_state>>8), also BC becomes (upkr_state & 255) 249 | 250 | ELSE 251 | 252 | ;;; unrolled MUL for better performance, +25 bytes unpack size 253 | ld b,d 254 | DUP 8 255 | add hl,hl 256 | jr nc,0_f 257 | add hl,de 258 | 0: 259 | EDUP 260 | 261 | ENDIF 262 | 263 | add hl,bc ; HL = state_scale * (upkr_state >> 8) + (upkr_state & 255) 264 | pop af ; restore prob and CF=bit 265 | jr nc,.bit_is_0_2 266 | dec d ; DE = -prob (also D = bit ? $FF : $00) 267 | add hl,de ; HL += -prob 268 | ; ^ this always preserves CF=1, because (state>>8) >= 128, state_scale: 7..250, prob: 7..250, 269 | ; so 7*128 > 250 and thus edge case `ADD hl=(7*128+0),de=(-250)` => CF=1 270 | .bit_is_0_2: 271 | ; *** adjust probs[context_index] 272 | rra ; + (bit<<4) ; part of -prob_offset, needs another -16 273 | and $FC ; clear/keep correct bits to get desired (prob>>4) + extras, CF=0 274 | rra 275 | rra 276 | rra ; A = (bit<<4) + (prob>>4), CF=(prob & 8) 277 | adc a,-16 ; A = (bit<<4) - 16 + ((prob + 8)>>4) ; -prob_offset = (bit<<4) - 16 278 | ld e,a 279 | pop bc 280 | ld a,(bc) ; A = prob (cheaper + shorter to re-read again from memory) 281 | sub e ; A = 16 - (bit<<4) + prob - ((prob + 8)>>4) ; = prob_offset + prob - ((prob + 8)>>4) 282 | ld (bc),a ; probs[context_index] = prob_offset + prob - ((prob + 8) >> 4); 283 | add a,d ; restore CF = bit (D = bit ? $FF : $00 && A > 0) 284 | pop de 285 | ret 286 | 287 | /* 288 | int upkr_decode_length(int context_index) { 289 | int length = 0; 290 | int bit_pos = 0; 291 | while(upkr_decode_bit(context_index)) { 292 | length |= upkr_decode_bit(context_index + 1) << bit_pos++; 293 | context_index += 2; 294 | } 295 | return length | (1 << bit_pos); 296 | } 297 | */ 298 | decode_number: 299 | ; HL = upkr_state 300 | ; IX = upkr_data_ptr 301 | ; BC = probs+context_index-1 302 | ; A' = upkr_current_byte (!!! init to 0x80 at start, not 0x00) 303 | ; return length in DE, CF=0 304 | ld de,$FFFF ; length = 0 with positional-stop-bit 305 | or a ; CF=0 to skip getting data bit and use only `rr d : rr e` to fix init DE 306 | .loop: 307 | call c,inc_c_decode_bit ; get data bit, context_index + 1 / if CF=0 just add stop bit into DE init 308 | rr d 309 | rr e ; DE = length = (length >> 1) | (bit << 15); 310 | call inc_c_decode_bit ; context_index += 2 311 | jr c,.loop 312 | .fix_bit_pos: 313 | ccf ; NC will become this final `| (1 << bit_pos)` bit 314 | rr d 315 | rr e 316 | jr c,.fix_bit_pos ; until stop bit is reached (all bits did land to correct position) 317 | ret ; return with CF=0 (important for unpack routine) 318 | 319 | DISPLAY "upkr.unpack total size: ",/D,$-unpack 320 | 321 | ; reserve space for probs array without emitting any machine code (using only EQU) 322 | 323 | IFDEF UPKR_PROBS_ORIGIN ; if specific address is defined by user, move probs array there 324 | probs: EQU ((UPKR_PROBS_ORIGIN) + 255) & -$100 ; probs array aligned to 256 325 | ELSE 326 | probs: EQU ($ + 255) & -$100 ; probs array aligned to 256 327 | ENDIF 328 | .real_c: EQU 1 + 255 + 1 + 2*NUMBER_BITS ; real size of probs array 329 | .c: EQU (.real_c + 1) & -2 ; padding to even size (required by init code) 330 | .e: EQU probs + .c 331 | 332 | DISPLAY "upkr.unpack probs array placed at: ",/A,probs,",\tsize: ",/A,probs.c 333 | 334 | /* 335 | archived: negligibly faster but +6B longer decode_number variant using HL' and BC' to 336 | do `number|=(1<