├── .gitignore
├── LzmaSpec.cpp
├── Makefile
├── README.md
├── contrib
    ├── hackyelf.py
    ├── linkmap.py
    └── parsemap.py
├── realcolor.hpp
└── small-lzma.png


/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__/
2 | 
3 | LzmaSpec
4 | 


--------------------------------------------------------------------------------
/LzmaSpec.cpp:
--------------------------------------------------------------------------------
  1 | // vim: set et ts=2 sw=2:
  2 | /* LzmaSpec.c -- LZMA Reference Decoder
  3 | 2015-06-14 : Igor Pavlov : Public domain */
  4 | 
  5 | // This code implements LZMA file decoding according to LZMA specification.
  6 | // This code is not optimized for speed.
  7 | 
  8 | #include <cstddef>
  9 | #include <cstdint>
 10 | #include <stdio.h>
 11 | #include <math.h>
 12 | #include <string.h>
 13 | #include <iostream>
 14 | #include <iomanip>
 15 | #include <vector>
 16 | #include <algorithm>
 17 | #ifndef _MSC_VER
 18 | #include <unistd.h>
 19 | #endif
 20 | #include "realcolor.hpp"
 21 | 
 22 | #ifdef _MSC_VER
 23 |   #pragma warning(disable : 4710) // function not inlined
 24 |   #pragma warning(disable : 4996) // This function or variable may be unsafe
 25 | #endif
 26 | 
 27 | typedef uint8_t Byte;
 28 | typedef uint16_t UInt16;
 29 | typedef uint32_t UInt32;
 30 | typedef uint64_t UInt64;
 31 | 
 32 | 
 33 | struct CInputStream
 34 | {
 35 |   FILE *File;
 36 |   UInt64 Processed;
 37 | 
 38 |   void Init() { Processed = 0; }
 39 | 
 40 |   Byte ReadByte()
 41 |   {
 42 |     int c = getc(File);
 43 |     if (c < 0)
 44 |       throw "Unexpected end of file";
 45 |     Processed++;
 46 |     return (Byte)c;
 47 |   }
 48 | };
 49 | 
 50 | 
 51 | struct COutStream
 52 | {
 53 |   std::vector<Byte> Data;
 54 | 
 55 |   void WriteByte(Byte b)
 56 |   {
 57 |     Data.push_back(b);
 58 |   }
 59 | };
 60 | 
 61 | 
 62 | class COutWindow
 63 | {
 64 |   Byte *Buf;
 65 |   UInt32 Pos;
 66 |   UInt32 Size;
 67 |   bool IsFull;
 68 | 
 69 | public:
 70 |   unsigned TotalPos;
 71 |   COutStream OutStream;
 72 | 
 73 |   COutWindow(): Buf(NULL) {}
 74 |   ~COutWindow() { delete []Buf; }
 75 |  
 76 |   void Create(UInt32 dictSize)
 77 |   {
 78 |     Buf = new Byte[dictSize];
 79 |     Pos = 0;
 80 |     Size = dictSize;
 81 |     IsFull = false;
 82 |     TotalPos = 0;
 83 |   }
 84 | 
 85 |   void PutByte(Byte b)
 86 |   {
 87 |     TotalPos++;
 88 |     Buf[Pos++] = b;
 89 |     if (Pos == Size)
 90 |     {
 91 |       Pos = 0;
 92 |       IsFull = true;
 93 |     }
 94 |     OutStream.WriteByte(b);
 95 |   }
 96 | 
 97 |   Byte GetByte(UInt32 dist) const
 98 |   {
 99 |     return Buf[dist <= Pos ? Pos - dist : Size - dist + Pos];
100 |   }
101 | 
102 |   void CopyMatch(UInt32 dist, unsigned len)
103 |   {
104 |     for (; len > 0; len--)
105 |       PutByte(GetByte(dist));
106 |   }
107 | 
108 |   bool CheckDistance(UInt32 dist) const
109 |   {
110 |     return dist <= Pos || IsFull;
111 |   }
112 | 
113 |   bool IsEmpty() const
114 |   {
115 |     return Pos == 0 && !IsFull;
116 |   }
117 | };
118 | 
119 | 
120 | #define kNumBitModelTotalBits 11
121 | #define kNumMoveBits 5
122 | 
123 | typedef UInt16 CProb;
124 | 
125 | #define PROB_INIT_VAL ((1 << kNumBitModelTotalBits) / 2)
126 | 
127 | #define INIT_PROBS(p) \
128 |  { for (unsigned i = 0; i < sizeof(p) / sizeof(p[0]); i++) p[i] = PROB_INIT_VAL; }
129 | 
130 | class CRangeDecoder
131 | {
132 |   UInt32 Range;
133 |   UInt32 Code;
134 | 
135 |   void Normalize();
136 | 
137 | public:
138 | 
139 |   CInputStream *InStream;
140 |   bool Corrupted;
141 |   float Perplexity;
142 | 
143 |   bool Init();
144 |   bool IsFinishedOK() const { return Code == 0; }
145 | 
146 |   UInt32 DecodeDirectBits(unsigned numBits);
147 |   unsigned DecodeBit(CProb *prob);
148 | };
149 | 
150 | bool CRangeDecoder::Init()
151 | {
152 |   Corrupted = false;
153 |   Range = 0xFFFFFFFF;
154 |   Code = 0;
155 |   Perplexity = 0.f;
156 | 
157 |   Byte b = InStream->ReadByte();
158 | 
159 |   for (int i = 0; i < 4; i++)
160 |     Code = (Code << 8) | InStream->ReadByte();
161 | 
162 |   if (b != 0 || Code == Range)
163 |     Corrupted = true;
164 |   return b == 0;
165 | }
166 | 
167 | #define kTopValue ((UInt32)1 << 24)
168 | 
169 | void CRangeDecoder::Normalize()
170 | {
171 |   if (Range < kTopValue)
172 |   {
173 |     Range <<= 8;
174 |     Code = (Code << 8) | InStream->ReadByte();
175 |   }
176 | }
177 | 
178 | UInt32 CRangeDecoder::DecodeDirectBits(unsigned numBits)
179 | {
180 |   Perplexity += numBits;
181 |   UInt32 res = 0;
182 |   do
183 |   {
184 |     Range >>= 1;
185 |     Code -= Range;
186 |     UInt32 t = 0 - ((UInt32)Code >> 31);
187 |     Code += Range & t;
188 | 
189 |     if (Code == Range)
190 |       Corrupted = true;
191 | 
192 |     Normalize();
193 |     res <<= 1;
194 |     res += t + 1;
195 |   }
196 |   while (--numBits);
197 |   return res;
198 | }
199 | 
200 | unsigned CRangeDecoder::DecodeBit(CProb *prob)
201 | {
202 |   unsigned v = *prob;
203 |   UInt32 bound = (Range >> kNumBitModelTotalBits) * v;
204 |   unsigned symbol;
205 |   if (Code < bound)
206 |   {
207 |     Perplexity += -log2(v/2048.f);
208 |     v += ((1 << kNumBitModelTotalBits) - v) >> kNumMoveBits;
209 |     Range = bound;
210 |     symbol = 0;
211 |   }
212 |   else
213 |   {
214 |     Perplexity += -log2(1.f-v/2048.f);
215 |     v -= v >> kNumMoveBits;
216 |     Code -= bound;
217 |     Range -= bound;
218 |     symbol = 1;
219 |   }
220 |   *prob = (CProb)v;
221 |   Normalize();
222 |   return symbol;
223 | }
224 | 
225 | 
226 | unsigned BitTreeReverseDecode(CProb *probs, unsigned numBits, CRangeDecoder *rc)
227 | {
228 |   unsigned m = 1;
229 |   unsigned symbol = 0;
230 |   for (unsigned i = 0; i < numBits; i++)
231 |   {
232 |     unsigned bit = rc->DecodeBit(&probs[m]);
233 |     m <<= 1;
234 |     m += bit;
235 |     symbol |= (bit << i);
236 |   }
237 |   return symbol;
238 | }
239 | 
240 | template <unsigned NumBits>
241 | class CBitTreeDecoder
242 | {
243 |   CProb Probs[(unsigned)1 << NumBits];
244 | 
245 | public:
246 | 
247 |   void Init()
248 |   {
249 |     INIT_PROBS(Probs);
250 |   }
251 | 
252 |   unsigned Decode(CRangeDecoder *rc)
253 |   {
254 |     unsigned m = 1;
255 |     for (unsigned i = 0; i < NumBits; i++)
256 |       m = (m << 1) + rc->DecodeBit(&Probs[m]);
257 |     return m - ((unsigned)1 << NumBits);
258 |   }
259 | 
260 |   unsigned ReverseDecode(CRangeDecoder *rc)
261 |   {
262 |     return BitTreeReverseDecode(Probs, NumBits, rc);
263 |   }
264 | };
265 | 
266 | #define kNumPosBitsMax 4
267 | 
268 | #define kNumStates 12
269 | #define kNumLenToPosStates 4
270 | #define kNumAlignBits 4
271 | #define kStartPosModelIndex 4
272 | #define kEndPosModelIndex 14
273 | #define kNumFullDistances (1 << (kEndPosModelIndex >> 1))
274 | #define kMatchMinLen 2
275 | 
276 | class CLenDecoder
277 | {
278 |   CProb Choice;
279 |   CProb Choice2;
280 |   CBitTreeDecoder<3> LowCoder[1 << kNumPosBitsMax];
281 |   CBitTreeDecoder<3> MidCoder[1 << kNumPosBitsMax];
282 |   CBitTreeDecoder<8> HighCoder;
283 | 
284 | public:
285 | 
286 |   void Init()
287 |   {
288 |     Choice = PROB_INIT_VAL;
289 |     Choice2 = PROB_INIT_VAL;
290 |     HighCoder.Init();
291 |     for (unsigned i = 0; i < (1 << kNumPosBitsMax); i++)
292 |     {
293 |       LowCoder[i].Init();
294 |       MidCoder[i].Init();
295 |     }
296 |   }
297 | 
298 |   unsigned Decode(CRangeDecoder *rc, unsigned posState)
299 |   {
300 |     if (rc->DecodeBit(&Choice) == 0)
301 |       return LowCoder[posState].Decode(rc);
302 |     if (rc->DecodeBit(&Choice2) == 0)
303 |       return 8 + MidCoder[posState].Decode(rc);
304 |     return 16 + HighCoder.Decode(rc);
305 |   }
306 | };
307 | 
308 | unsigned UpdateState_Literal(unsigned state)
309 | {
310 |   if (state < 4) return 0;
311 |   else if (state < 10) return state - 3;
312 |   else return state - 6;
313 | }
314 | unsigned UpdateState_Match   (unsigned state) { return state < 7 ? 7 : 10; }
315 | unsigned UpdateState_Rep     (unsigned state) { return state < 7 ? 8 : 11; }
316 | unsigned UpdateState_ShortRep(unsigned state) { return state < 7 ? 9 : 11; }
317 | 
318 | #define LZMA_DIC_MIN (1 << 12)
319 | 
320 | class CLzmaDecoder
321 | {
322 | public:
323 |   CRangeDecoder RangeDec;
324 |   COutWindow OutWindow;
325 |   std::vector<float> Perplexities;
326 |   std::vector<bool> Literals;
327 | 
328 |   bool markerIsMandatory;
329 |   unsigned lc, pb, lp;
330 |   UInt32 dictSize;
331 |   UInt32 dictSizeInProperties;
332 | 
333 |   void DecodeProperties(const Byte *properties)
334 |   {
335 |     unsigned d = properties[0];
336 |     if (d >= (9 * 5 * 5))
337 |       throw "Incorrect LZMA properties";
338 |     lc = d % 9;
339 |     d /= 9;
340 |     pb = d / 5;
341 |     lp = d % 5;
342 |     dictSizeInProperties = 0;
343 |     for (int i = 0; i < 4; i++)
344 |       dictSizeInProperties |= (UInt32)properties[i + 1] << (8 * i);
345 |     dictSize = dictSizeInProperties;
346 |     if (dictSize < LZMA_DIC_MIN)
347 |       dictSize = LZMA_DIC_MIN;
348 |   }
349 | 
350 |   CLzmaDecoder(): LitProbs(NULL) {}
351 |   ~CLzmaDecoder() { delete []LitProbs; }
352 | 
353 |   void Create()
354 |   {
355 |     OutWindow.Create(dictSize);
356 |     CreateLiterals();
357 |   }
358 | 
359 |   int Decode(bool unpackSizeDefined, UInt64 unpackSize);
360 | 
361 | private:
362 | 
363 |   CProb *LitProbs;
364 | 
365 |   void CreateLiterals()
366 |   {
367 |     LitProbs = new CProb[(UInt32)0x300 << (lc + lp)];
368 |   }
369 | 
370 |   void InitLiterals()
371 |   {
372 |     UInt32 num = (UInt32)0x300 << (lc + lp);
373 |     for (UInt32 i = 0; i < num; i++)
374 |       LitProbs[i] = PROB_INIT_VAL;
375 |   }
376 | 
377 |   void DecodeLiteral(unsigned state, UInt32 rep0)
378 |   {
379 |     unsigned prevByte = 0;
380 |     if (!OutWindow.IsEmpty())
381 |       prevByte = OutWindow.GetByte(1);
382 | 
383 |     unsigned symbol = 1;
384 |     unsigned litState = ((OutWindow.TotalPos & ((1 << lp) - 1)) << lc) + (prevByte >> (8 - lc));
385 |     CProb *probs = &LitProbs[(UInt32)0x300 * litState];
386 | 
387 |     if (state >= 7)
388 |     {
389 |       unsigned matchByte = OutWindow.GetByte(rep0 + 1);
390 |       do
391 |       {
392 |         unsigned matchBit = (matchByte >> 7) & 1;
393 |         matchByte <<= 1;
394 |         unsigned bit = RangeDec.DecodeBit(&probs[((1 + matchBit) << 8) + symbol]);
395 |         symbol = (symbol << 1) | bit;
396 |         if (matchBit != bit)
397 |           break;
398 |       }
399 |       while (symbol < 0x100);
400 |     }
401 |     while (symbol < 0x100)
402 |       symbol = (symbol << 1) | RangeDec.DecodeBit(&probs[symbol]);
403 |     OutWindow.PutByte((Byte)(symbol - 0x100));
404 |   }
405 | 
406 |   CBitTreeDecoder<6> PosSlotDecoder[kNumLenToPosStates];
407 |   CBitTreeDecoder<kNumAlignBits> AlignDecoder;
408 |   CProb PosDecoders[1 + kNumFullDistances - kEndPosModelIndex];
409 | 
410 |   void InitDist()
411 |   {
412 |     for (unsigned i = 0; i < kNumLenToPosStates; i++)
413 |       PosSlotDecoder[i].Init();
414 |     AlignDecoder.Init();
415 |     INIT_PROBS(PosDecoders);
416 |   }
417 | 
418 |   unsigned DecodeDistance(unsigned len)
419 |   {
420 |     unsigned lenState = len;
421 |     if (lenState > kNumLenToPosStates - 1)
422 |       lenState = kNumLenToPosStates - 1;
423 | 
424 |     unsigned posSlot = PosSlotDecoder[lenState].Decode(&RangeDec);
425 |     if (posSlot < 4)
426 |       return posSlot;
427 | 
428 |     unsigned numDirectBits = (unsigned)((posSlot >> 1) - 1);
429 |     UInt32 dist = ((2 | (posSlot & 1)) << numDirectBits);
430 |     if (posSlot < kEndPosModelIndex)
431 |       dist += BitTreeReverseDecode(PosDecoders + dist - posSlot, numDirectBits, &RangeDec);
432 |     else
433 |     {
434 |       dist += RangeDec.DecodeDirectBits(numDirectBits - kNumAlignBits) << kNumAlignBits;
435 |       dist += AlignDecoder.ReverseDecode(&RangeDec);
436 |     }
437 |     return dist;
438 |   }
439 | 
440 |   void PushPerplexities(unsigned len)
441 |   {
442 |     for (int i = 0; i < len; i++) {
443 |       Perplexities.push_back(RangeDec.Perplexity/len);
444 |     }
445 |     RangeDec.Perplexity = 0.f;
446 |   }
447 | 
448 |   CProb IsMatch[kNumStates << kNumPosBitsMax];
449 |   CProb IsRep[kNumStates];
450 |   CProb IsRepG0[kNumStates];
451 |   CProb IsRepG1[kNumStates];
452 |   CProb IsRepG2[kNumStates];
453 |   CProb IsRep0Long[kNumStates << kNumPosBitsMax];
454 | 
455 |   CLenDecoder LenDecoder;
456 |   CLenDecoder RepLenDecoder;
457 | 
458 |   void Init()
459 |   {
460 |     InitLiterals();
461 |     InitDist();
462 | 
463 |     INIT_PROBS(IsMatch);
464 |     INIT_PROBS(IsRep);
465 |     INIT_PROBS(IsRepG0);
466 |     INIT_PROBS(IsRepG1);
467 |     INIT_PROBS(IsRepG2);
468 |     INIT_PROBS(IsRep0Long);
469 | 
470 |     LenDecoder.Init();
471 |     RepLenDecoder.Init();
472 |   }
473 | };
474 | 
475 | 
476 | #define LZMA_RES_ERROR                   0
477 | #define LZMA_RES_FINISHED_WITH_MARKER    1
478 | #define LZMA_RES_FINISHED_WITHOUT_MARKER 2
479 | 
480 | int CLzmaDecoder::Decode(bool unpackSizeDefined, UInt64 unpackSize)
481 | {
482 |   if (!RangeDec.Init())
483 |     return LZMA_RES_ERROR;
484 | 
485 |   Init();
486 | 
487 |   UInt32 rep0 = 0, rep1 = 0, rep2 = 0, rep3 = 0;
488 |   unsigned state = 0;
489 | 
490 |   for (;;)
491 |   {
492 |     if (unpackSizeDefined && unpackSize == 0 && !markerIsMandatory)
493 |       if (RangeDec.IsFinishedOK())
494 |         return LZMA_RES_FINISHED_WITHOUT_MARKER;
495 | 
496 |     unsigned posState = OutWindow.TotalPos & ((1 << pb) - 1);
497 | 
498 |     if (RangeDec.DecodeBit(&IsMatch[(state << kNumPosBitsMax) + posState]) == 0)
499 |     {
500 |       if (unpackSizeDefined && unpackSize == 0)
501 |         return LZMA_RES_ERROR;
502 |       DecodeLiteral(state, rep0);
503 |       PushPerplexities(1);
504 |       Literals.push_back(true);
505 |       state = UpdateState_Literal(state);
506 |       unpackSize--;
507 |       continue;
508 |     }
509 | 
510 |     unsigned len;
511 | 
512 |     if (RangeDec.DecodeBit(&IsRep[state]) != 0)
513 |     {
514 |       if (unpackSizeDefined && unpackSize == 0)
515 |         return LZMA_RES_ERROR;
516 |       if (OutWindow.IsEmpty())
517 |         return LZMA_RES_ERROR;
518 |       if (RangeDec.DecodeBit(&IsRepG0[state]) == 0)
519 |       {
520 |         if (RangeDec.DecodeBit(&IsRep0Long[(state << kNumPosBitsMax) + posState]) == 0)
521 |         {
522 |           state = UpdateState_ShortRep(state);
523 |           OutWindow.PutByte(OutWindow.GetByte(rep0 + 1));
524 |           PushPerplexities(1);
525 |           Literals.push_back(false);
526 |           unpackSize--;
527 |           continue;
528 |         }
529 |       }
530 |       else
531 |       {
532 |         UInt32 dist;
533 |         if (RangeDec.DecodeBit(&IsRepG1[state]) == 0)
534 |           dist = rep1;
535 |         else
536 |         {
537 |           if (RangeDec.DecodeBit(&IsRepG2[state]) == 0)
538 |             dist = rep2;
539 |           else
540 |           {
541 |             dist = rep3;
542 |             rep3 = rep2;
543 |           }
544 |           rep2 = rep1;
545 |         }
546 |         rep1 = rep0;
547 |         rep0 = dist;
548 |       }
549 |       len = RepLenDecoder.Decode(&RangeDec, posState);
550 |       state = UpdateState_Rep(state);
551 |     }
552 |     else
553 |     {
554 |       rep3 = rep2;
555 |       rep2 = rep1;
556 |       rep1 = rep0;
557 |       len = LenDecoder.Decode(&RangeDec, posState);
558 |       state = UpdateState_Match(state);
559 |       rep0 = DecodeDistance(len);
560 |       if (rep0 == 0xFFFFFFFF)
561 |         return RangeDec.IsFinishedOK() ?
562 |             LZMA_RES_FINISHED_WITH_MARKER :
563 |             LZMA_RES_ERROR;
564 | 
565 |       if (unpackSizeDefined && unpackSize == 0)
566 |         return LZMA_RES_ERROR;
567 |       if (rep0 >= dictSize || !OutWindow.CheckDistance(rep0))
568 |         return LZMA_RES_ERROR;
569 |     }
570 |     len += kMatchMinLen;
571 |     bool isError = false;
572 |     if (unpackSizeDefined && unpackSize < len)
573 |     {
574 |       len = (unsigned)unpackSize;
575 |       isError = true;
576 |     }
577 |     OutWindow.CopyMatch(rep0 + 1, len);
578 |     Literals.insert(Literals.end(), len, false);
579 |     PushPerplexities(len);
580 |     unpackSize -= len;
581 |     if (isError)
582 |       return LZMA_RES_ERROR;
583 |   }
584 | }
585 | 
586 | //https://www.andrewnoske.com/wiki/Code_-_heatmaps_and_color_gradients
587 | class ColorGradient
588 | {
589 | private:
590 |   struct ColorPoint  // Internal class used to store colors at different points in the gradient.
591 |   {
592 |     float r,g,b;      // Red, green and blue values of our color.
593 |     float val;        // Position of our color along the gradient (between 0 and 1).
594 |     ColorPoint(float red, float green, float blue, float value)
595 |       : r(red), g(green), b(blue), val(value) {}
596 |   };
597 |   std::vector<ColorPoint> color;      // An array of color points in ascending value.
598 | 
599 | public:
600 |   //-- Default constructor:
601 |   ColorGradient()  {  createDefaultHeatMapGradient();  }
602 | 
603 |   //-- Inserts a new color point into its correct position:
604 |   void addColorPoint(float red, float green, float blue, float value)
605 |   {
606 |     for(int i=0; i<color.size(); i++)  {
607 |       if(value < color[i].val) {
608 |         color.insert(color.begin()+i, ColorPoint(red,green,blue, value));
609 |         return;  }}
610 |     color.push_back(ColorPoint(red,green,blue, value));
611 |   }
612 | 
613 |   //-- Inserts a new color point into its correct position:
614 |   void clearGradient() { color.clear(); }
615 | 
616 |   //-- Places a 5 color heapmap gradient into the "color" vector:
617 |   void createDefaultHeatMapGradient()
618 |   {
619 |     color.clear();
620 |     color.push_back(ColorPoint(0, 0, 0,   0.0f));      // Blue.
621 |     color.push_back(ColorPoint(0, 0, 1,   0.2f));     // Cyan.
622 |     color.push_back(ColorPoint(0, 1, 0,   0.5f));      // Green.
623 |     color.push_back(ColorPoint(1, 1, 0,   0.7f));     // Yellow.
624 |     color.push_back(ColorPoint(1, 0, 0,   0.9f));      // Red.
625 |   }
626 |   void createViridisHeatMapGradient()
627 |   {
628 |     color.clear();
629 |     color.push_back(ColorPoint(0x44/255.f,0x02/255.f,0x55/255.f,   0.0f));
630 |     color.push_back(ColorPoint(0x2C/255.f,0x70/255.f,0x8E/255.f,   0.33f));
631 |     color.push_back(ColorPoint(0x3D/255.f,0xBB/255.f,0x74/255.f,   0.66f));
632 |     color.push_back(ColorPoint(0xFA/255.f,0xE6/255.f,0x22/255.f,   1.f));
633 |   }
634 | 
635 |   //-- Inputs a (value) between 0 and 1 and outputs the (red), (green) and (blue)
636 |   //-- values representing that position in the gradient.
637 |   void getColorAtValue(const float value, float &red, float &green, float &blue)
638 |   {
639 |     if(color.size()==0)
640 |       return;
641 | 
642 |     for(int i=0; i<color.size(); i++)
643 |     {
644 |       ColorPoint &currC = color[i];
645 |       if(value < currC.val)
646 |       {
647 |         ColorPoint &prevC  = color[ std::max(0,i-1) ];
648 |         float valueDiff    = (prevC.val - currC.val);
649 |         float fractBetween = (valueDiff==0) ? 0 : (value - currC.val) / valueDiff;
650 |         red   = (prevC.r - currC.r)*fractBetween + currC.r;
651 |         green = (prevC.g - currC.g)*fractBetween + currC.g;
652 |         blue  = (prevC.b - currC.b)*fractBetween + currC.b;
653 |         return;
654 |       }
655 |     }
656 |     red   = color.back().r;
657 |     green = color.back().g;
658 |     blue  = color.back().b;
659 |     return;
660 |   }
661 | 
662 |   std::string get(const float value)
663 |   {
664 |     float r,g,b;
665 |     getColorAtValue(value, r,g,b);
666 |     std::stringstream ss;
667 |     ss << realcolor::bg(r,g,b) << realcolor::fg(1.f-r,1.f-g,1.f-b);
668 |     return ss.str();
669 |   }
670 | 
671 |   std::string printScale(int width)
672 |   {
673 |     std::stringstream ss;
674 |     for (int i = 0; i < width; i++) {
675 |       float val = (i*1.f)/width;
676 |       float r,g,b;
677 |       getColorAtValue(val, r,g,b);
678 |       ss << realcolor::fg(r,g,b) << "━";
679 |     }
680 |     ss << realcolor::reset;
681 |     return ss.str();
682 |   }
683 | };
684 | 
685 | static struct {
686 |   std::string infile;
687 |   std::string kkpout;
688 |   bool pretty;
689 |   bool jet;
690 |   bool literals;
691 | } options;
692 | 
693 | static void usage(char** argv) {
694 |   std::cerr << "usage: " << argv[0] << " [--raw] [--jet] [--kkp outfile.kkp] [--help] file.lzma" << std::endl;
695 | }
696 | 
697 | static int parse_args(int argc, char** argv) {
698 | #ifdef _MSC_VER
699 |   options.pretty = true;
700 | #else
701 |   options.pretty = isatty(STDOUT_FILENO);
702 | #endif
703 |   options.jet = false;
704 |   options.literals = false;
705 |   options.kkpout = "";
706 |   options.infile = "";
707 | 
708 |   if (argc < 2) {
709 |     usage(argv);
710 |     return 1;
711 |   }
712 | 
713 |   for (int i = 1; i < argc; ++i) {
714 |     const char* curarg = argv[i];
715 |     if (!strcmp(curarg, "--raw")) {
716 |       options.pretty = false;
717 |       continue;
718 |     }
719 |     if (!strcmp(curarg, "--jet")) {
720 |       options.jet = true;
721 |       continue;
722 |     }
723 |     if (!strcmp(curarg, "--lits") || !strcmp(curarg, "--literals")) {
724 |       options.literals = true;
725 |       continue;
726 |     }
727 |     if (!strcmp(curarg, "--help")) {
728 |       usage(argv);
729 |       return 0;
730 |     }
731 | 
732 |     if (!strcmp(curarg, "--kkp")) {
733 |       if (i == argc - 1) {
734 |         std::cerr << argv[0] << ": Need output file argument for --kkp flag!" << std::endl;
735 |         return 1;
736 |       }
737 |       options.kkpout = argv[i+1];
738 |       ++i;
739 |       continue;
740 |     }
741 | 
742 |     if (!options.infile.empty()) {
743 |       std::cerr << argv[0] << ": Input file already specified! Was parsed as '" << options.infile << "', now also got '" << curarg << "'." << std::endl;
744 |       return 1;
745 |     }
746 |     options.infile = curarg;
747 |   }
748 | 
749 |   if (options.infile.empty()) {
750 |     std::cerr << argv[0] << ": No input file specified!" << std::endl;
751 |     return 1;
752 |   }
753 | 
754 |   return -1;
755 | }
756 | 
757 | static int output_raw(CLzmaDecoder& lzmaDecoder, double maxPerplexity) {
758 |   for (int j = 0; j < lzmaDecoder.OutWindow.OutStream.Data.size(); j++) {
759 |     std::cout << lzmaDecoder.Perplexities[j]/maxPerplexity << std::endl;
760 |   }
761 |   std::cout << std::endl;
762 |   return 0;
763 | }
764 | 
765 | static int output_kkp(CLzmaDecoder& lzmaDecoder, double maxPerplexity) {
766 |   FILE* fo = fopen(options.kkpout.c_str(), "wb");
767 |   if (!fo) {
768 |     return 1;
769 |   }
770 | 
771 |   double totalPerpl = 0;
772 |   for (size_t j = 0; j < lzmaDecoder.OutWindow.OutStream.Data.size(); j++) {
773 |     totalPerpl += lzmaDecoder.Perplexities[j];
774 |   }
775 |   totalPerpl /= 8;
776 | 
777 |   uint32_t zero = 0;
778 |   uint32_t one = htole32((uint32_t)1);
779 |   uint32_t mone = 0xffffffffu;
780 |   uint32_t size32l = htole32((uint32_t)lzmaDecoder.OutWindow.OutStream.Data.size());
781 | 
782 |   // magic bytes
783 |   fwrite("KK64", 1, 4, fo);
784 |   // number of binary bytes
785 |   fwrite(&size32l, 4, 1, fo);
786 |   // number of source code files
787 |   fwrite(&zero, 4, 1, fo);
788 | 
789 |   // normally, source code info would be here
790 | 
791 |   // number of symbols (zero)
792 |   fwrite(&zero, 4, 1, fo);
793 | 
794 |   // normally, symbols would be here
795 | 
796 |   // all compressed bytes
797 |   for (size_t j = 0; j < lzmaDecoder.OutWindow.OutStream.Data.size(); ++j) {
798 |     uint8_t v = lzmaDecoder.OutWindow.OutStream.Data[j];
799 |     fwrite(&v, 1, 1, fo); // orig. data
800 |     fwrite(&zero, 2, 1, fo); // symbol index
801 |     double psz = lzmaDecoder.Perplexities[j] / 8;
802 |     fwrite(&psz, 8, 1, fo);
803 |     fwrite(&zero, 2, 1, fo); // source code line
804 |     fwrite(&zero, 2, 1, fo); // source code file index
805 |   }
806 | 
807 |   fclose(fo);
808 | 
809 |   return 0;
810 | }
811 | 
812 | static int output_console(CLzmaDecoder& lzmaDecoder, double maxPerplexity) {
813 |   ColorGradient grad;
814 |   if (options.jet) {
815 |     grad.createDefaultHeatMapGradient();
816 |   } else {
817 |     grad.createViridisHeatMapGradient();
818 |   }
819 | 
820 |   int colWidth = 64;
821 |   int scaleFreq = 16;
822 |   float minForCol = 1.;
823 |   float maxForCol = 0;
824 |   float avgForCol = 0;
825 | 
826 |   for (int j = 0; j < lzmaDecoder.OutWindow.OutStream.Data.size(); j++) {
827 |     if (j % colWidth == 0 && (j / colWidth)%scaleFreq == 0) {
828 |       std::cout << grad.printScale(colWidth) << std::endl;
829 |     }
830 |     bool literal = lzmaDecoder.Literals[j];
831 |     float heat = sqrt(lzmaDecoder.Perplexities[j]/maxPerplexity);
832 |     avgForCol += heat;
833 |     maxForCol = std::max(maxForCol, heat);
834 |     minForCol = std::min(minForCol, heat);
835 |     if (options.literals) heat = literal ? 1. : 0.;
836 | 
837 |     char byte = lzmaDecoder.OutWindow.OutStream.Data[j];
838 |     if (!std::isprint(byte)) {
839 |       byte = '.';
840 |     }
841 |     std::cout
842 |       << grad.get(heat)
843 |       // << std::setfill('0')
844 |       // << std::setw(2)
845 |       // << std::right
846 |       // << std::hex
847 |       << byte
848 |       << realcolor::reset;
849 |     // if (j % 16 == 16-1) {
850 |     //   std::cout << " ";
851 |     // }
852 |     if (j % colWidth == colWidth-1) {
853 |       std::cout << " "
854 |         << grad.get(minForCol) << " "
855 |         << grad.get(avgForCol/colWidth) << " "
856 |         << grad.get(maxForCol) << " "
857 |         << realcolor::reset << std::endl;
858 |       minForCol = 1;
859 |       maxForCol = 0;
860 |       avgForCol = 0;
861 |     }
862 |   }
863 |   std::cout << std::endl;
864 |   return 0;
865 | }
866 | 
867 | int main(int argc, char** argv)
868 | {
869 |   int r = parse_args(argc, argv);
870 |   if (r >= 0) return r;
871 | 
872 |   CInputStream inStream;
873 |   inStream.File = fopen(options.infile.c_str(), "rb");
874 |   if (inStream.File == 0)
875 |     throw "Can't open input file";
876 |   inStream.Init();
877 | 
878 |   CLzmaDecoder lzmaDecoder;
879 | 
880 |   Byte header[13];
881 |   int i;
882 |   for (i = 0; i < 13; i++)
883 |     header[i] = inStream.ReadByte();
884 | 
885 |   lzmaDecoder.DecodeProperties(header);
886 | 
887 |   UInt64 unpackSize = 0;
888 |   bool unpackSizeDefined = false;
889 |   for (i = 0; i < 8; i++)
890 |   {
891 |     Byte b = header[5 + i];
892 |     if (b != 0xFF)
893 |       unpackSizeDefined = true;
894 |     unpackSize |= (UInt64)b << (8 * i);
895 |   }
896 | 
897 |   lzmaDecoder.markerIsMandatory = !unpackSizeDefined;
898 | 
899 |   lzmaDecoder.RangeDec.InStream = &inStream;
900 | 
901 |   lzmaDecoder.Create();
902 | 
903 |   int res = lzmaDecoder.Decode(unpackSizeDefined, unpackSize);
904 | 
905 |   if (res == LZMA_RES_ERROR)
906 |     throw "LZMA decoding error";
907 | 
908 |   if (lzmaDecoder.RangeDec.Corrupted)
909 |   {
910 |     std::cerr << "Warning: LZMA stream is corrupted" << std::endl;
911 |   }
912 | 
913 |   double maxPerplexity = *std::max_element(lzmaDecoder.Perplexities.begin(), lzmaDecoder.Perplexities.end());
914 | 
915 |   if (!options.kkpout.empty()) {
916 |     return output_kkp(lzmaDecoder, maxPerplexity);
917 |   } else if (options.pretty) {
918 |     return output_console(lzmaDecoder, maxPerplexity);
919 |   } else {
920 |     return output_raw(lzmaDecoder, maxPerplexity);
921 |   }
922 | 
923 |   return 0;
924 | }
925 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | LzmaSpec : LzmaSpec.cpp realcolor.hpp
2 | 	g++ LzmaSpec.cpp -o LzmaSpec -lm
3 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # LZMA Vizualizer
 2 | 
 3 | Visualises per-byte perplexity of the compressed data in an LZMA file.
 4 | 
 5 | ## Usage
 6 | 
 7 | ```
 8 | ./LzmaSpec foo.lzma
 9 | ```
10 | 
11 | ## Example output
12 | 
13 | ![example](/small-lzma.png)
14 | 
15 | ## Analysing the compression ratios of symbols in an ELF file
16 | 
17 | `contrib/parsemap.py` can be used to show the compression ratio of separate
18 | symbols in an ELF file, given the LZMA-compressed ELF file and the
19 | corresponding linker map file. (See the `cc(1)` and `ld(1)` manpages for notes
20 | on how to output these when linking.)
21 | 
22 | ### Usage
23 | 
24 | ```
25 | usage: parsemap.py [-h] [--recurse RECURSE] [--lzmaspec LZMASPEC] lzma_file
26 |                    map_file
27 | 
28 | Shows a summary of the compression stats of every symbol in an ELF, given the
29 | compressed and uncompressed ELF files, as well as a linker map file.
30 | (`ld -Map', `cc -Wl,-Map', see respective manpages.)
31 | 
32 | positional arguments:
33 |   lzma_file            The LZMA-compressed ELF file
34 |   map_file             The linker map file
35 | 
36 | optional arguments:
37 |   -h, --help           show this help message and exit
38 |   --recurse RECURSE    Recursively analyse data in symbols. Syntax:
39 |                        --recurse symname=linker.map
40 |   --lzmaspec LZMASPEC  LzmaSpec binary to use (default: ./LzmaSpec)
41 | ```
42 | 
43 | ### Example output
44 | 
45 | ```
46 | $ contrib/parsemap.py --recurse dtn_snd=introtree/snd/bin/snd.ld.map \
47 |                       introtree/bin/main.smol{.vndh.z,.map}
48 | Symbol                    Uncompr. Size  Perplexity Perplexity/Size
49 | -------------------------------------------------------------------
50 | dtn_snd -> *(.igot)               10195      591.26            0.06
51 | main                                595      256.30            0.43
52 | dtn_snd -> Clinkster_GenerateMusic  707      247.99            0.35
53 | dtn_snd -> _start                   540      201.28            0.37
54 | dtn_frag_glsl                       385      144.75            0.38
55 | _DYNAMIC                            312      126.19            0.40
56 | dtn_snd -> Clinkster_WavFileHeader  164       62.93            0.38
57 | SDL_GL_CreateContext                164       36.57            0.22
58 | _smol_origin                        224       35.54            0.16
59 | _start                               39       21.01            0.54
60 | dtn_snd -> .text.clinkster.genMus    26       14.26            0.55
61 | dtn_snd -> *(.elf.end)             2502       11.09            0.00
62 | glClearColor                          8        3.50            0.44
63 | glCreateShaderProgramv                8        3.46            0.43
64 | __libc_start_main                     8        3.44            0.43
65 | glViewport                            8        3.22            0.40
66 | glUseProgram                          8        3.20            0.40
67 | glClear                               8        3.16            0.40
68 | glRecti                               8        3.10            0.39
69 | SDL_Init                              8        3.07            0.38
70 | SDL_DestroyWindow                     8        3.06            0.38
71 | SDL_CreateWindow                      8        3.05            0.38
72 | SDL_GetTicks                          8        3.00            0.37
73 | glUniform1f                           8        2.98            0.37
74 | SDL_GL_SetAttribute                   8        2.97            0.37
75 | SDL_GL_DeleteContext                  8        2.96            0.37
76 | SDL_GL_SwapWindow                     8        2.87            0.36
77 | SDL_PollEvent                         8        2.81            0.35
78 | SDL_Quit                              8        2.57            0.32
79 | _smol_data_start                     12        1.21            0.10
80 | glUniform2i                           8        1.05            0.13
81 | dtn_snd -> Clinkster_MusicBuffer      8        0.92            0.12
82 | dtn_snd -> *(.text.unlikely_.text.    1        0.78            0.78
83 | dtn_snd -> *(.plt.sec)                1        0.77            0.77
84 | dtn_snd -> *(.text.hot_.text.hot.*    1        0.70            0.70
85 | dtn_snd -> *(.text.exit_.text.exit    1        0.66            0.66
86 | dtn_snd -> [0x08048094]               1        0.62            0.62
87 | dtn_snd -> *(.text.startup_.text.s    1        0.61            0.61
88 | _smol_text_start                      1        0.57            0.57
89 | dtn_snd -> *(.exception_ranges_.ex    1        0.49            0.49
90 | dtn_snd -> *(.text)                   1        0.48            0.48
91 | dtn_snd -> .rodata.clinkster.velfa    1        0.32            0.32
92 | _smol_text_end                        1        0.27            0.27
93 | dtn_snd -> __bss_start                2        0.23            0.12
94 | dtn_snd -> _edata                     1        0.12            0.12
95 | dtn_snd -> *(.data1)                  1        0.12            0.12
96 | dtn_snd -> *(.gnu.warning)            1        0.01            0.01
97 | ```
98 | 
99 | 


--------------------------------------------------------------------------------
/contrib/hackyelf.py:
--------------------------------------------------------------------------------
  1 | 
  2 | # custom elf parser because a standard one wouldn't be trustable because the
  3 | # ELFs we're parsing will be a bit wonky anyway
  4 | 
  5 | from struct import unpack
  6 | from typing import *
  7 | 
  8 | 
  9 | ELFCLASS32 = 1
 10 | ELFCLASS64 = 2
 11 | 
 12 | EM_386    =  3
 13 | EM_X86_64 = 62
 14 | 
 15 | PT_NULL    = 0
 16 | PT_LOAD    = 1
 17 | PT_DYNAMIC = 2
 18 | PT_INTERP  = 3
 19 | 
 20 | DT_NULL    =  0
 21 | DT_NEEDED  =  1
 22 | DT_PLTGOT  =  3
 23 | DT_STRTAB  =  5
 24 | DT_SYMTAB  =  6
 25 | DT_RELA    =  7
 26 | DT_RELASZ  =  8
 27 | DT_RELAENT =  9
 28 | DT_STRSZ   = 10
 29 | DT_SYMENT  = 11
 30 | DT_SONAME  = 14
 31 | DT_REL     = 17
 32 | DT_RELSZ   = 18
 33 | DT_RELENT  = 19
 34 | DT_PLTREL  = 20
 35 | DT_DEBUG   = 21
 36 | DT_TEXTREL = 22
 37 | DT_JMPREL  = 23
 38 | DT_BIND_NOW= 24
 39 | 
 40 | SHT_NULL     =  0
 41 | SHT_PROGBITS =  1
 42 | SHT_SYMTAB   =  2
 43 | SHT_STRTAB   =  3
 44 | SHT_RELA     =  4
 45 | SHT_DYNAMIC  =  6
 46 | SHT_NOBITS   =  8
 47 | SHT_REL      =  9
 48 | SHT_DYNSYM   = 11
 49 | 
 50 | SHF_WRITE     = 1<<0
 51 | SHF_ALLOC     = 1<<1
 52 | SHF_EXECINSTR = 1<<2
 53 | SHF_MERGE     = 1<<4
 54 | SHF_STRINGS   = 1<<5
 55 | SHF_INFO_LINK = 1<<6
 56 | 
 57 | STB_LOCAL  = 0
 58 | STB_GLOBAL = 1
 59 | STB_WEAK   = 2
 60 | 
 61 | STT_NOTYPE = 0
 62 | STT_OBJECT = 1
 63 | STT_FUNC   = 2
 64 | STT_SECTION= 3
 65 | STT_FILE   = 4
 66 | STT_COMMON = 5
 67 | STT_TLS    = 6
 68 | STT_GNU_IFUNC = 10
 69 | 
 70 | STV_DEFAULT   = 0
 71 | STV_INTERNAL  = 1
 72 | STV_HIDDEN    = 2
 73 | STV_PROTECTED = 3
 74 | 
 75 | class Phdr(NamedTuple):
 76 |     ptype: int
 77 |     off  : int
 78 |     vaddr: int
 79 |     paddr: int
 80 |     filesz: int
 81 |     memsz: int
 82 |     flags: int
 83 |     align: int
 84 | 
 85 | class Dyn(NamedTuple):
 86 |     tag: int
 87 |     val: int
 88 | 
 89 | class Shdr(NamedTuple):
 90 |     name: Union[int, str]
 91 |     type: int
 92 |     flags: int
 93 |     addr: int
 94 |     offset: int
 95 |     size: int
 96 |     link: int
 97 |     info: int
 98 |     addralign: int
 99 |     entsize: int
100 | 
101 | class Sym(NamedTuple):
102 |     name: str
103 |     value: int
104 |     size: int
105 |     type: int
106 |     binding: int
107 |     visibility: int
108 |     shndx: int
109 | 
110 | class Rel(NamedTuple):
111 |     offset: int
112 |     symbol: Sym
113 |     type: int
114 | class Rela(NamedTuple):
115 |     offset: int
116 |     symbol: Sym
117 |     type: int
118 |     addend: int
119 | Reloc = Union[Rel, Rela]
120 | 
121 | class ELF(NamedTuple):
122 |     data  : bytes
123 |     ident : bytes
124 |     eclass: int
125 |     mach  : int
126 |     entry : int
127 |     phdrs : Sequence[Phdr]
128 |     dyn   : Sequence[Dyn]
129 |     shdrs : Sequence[Shdr]
130 |     symtab: Sequence[Sym]
131 |     dynsym: Sequence[Sym]
132 |     relocs: Sequence[Reloc]
133 |     is32bit: bool
134 | 
135 | def readstr(data: bytes, off: int) -> str:
136 |     strb = bytearray()
137 |     while data[off] != 0 and off < len(data):
138 |         strb.append(data[off])
139 |         off = off + 1
140 |     return strb.decode('utf-8')
141 | 
142 | # yeah, there's some code duplication here
143 | # idgaf
144 | 
145 | def parse_phdr32(data: bytes, phoff:int, phentsz:int, phnum:int) -> Sequence[Phdr]:
146 |     ps = []
147 |     for off in range(phoff, phoff+phentsz*phnum, phentsz):
148 |         ptype, off, vaddr, paddr, filesz, memsz, flags, align = \
149 |             unpack('<IIIIIIII', data[off:off+8*4])
150 |         p = Phdr(ptype, off, vaddr, paddr, filesz, memsz, flags, align)
151 |         ps.append(p)
152 | 
153 |     return ps
154 | 
155 | def parse_dyn32(data: bytes, dynp: Phdr) -> Dyn:
156 |     ds = []
157 | 
158 |     off = dynp.off
159 |     while True:
160 |         tag, val = unpack('<II', data[off:off+2*4])
161 |         ds.append(Dyn(tag, val))
162 | 
163 |         if tag == DT_NULL: break
164 |         off = off + 2*4
165 | 
166 |     return ds
167 | 
168 | def parse_reloc32(data: bytes, reloff: int, nrel: int, entsz: int, syms: Sequence[Sym], rela: bool) -> Reloc:
169 |     rr=[]
170 | 
171 |     for off in range(reloff, reloff+entsz*nrel, entsz):
172 |         off, inf, add = unpack('<IIi', data[off:(off+12)]) if rela \
173 |             else (*unpack('<Ii', data[off:(off+8)]),0)
174 | 
175 |         sym = syms[inf >> 8]
176 |         type = inf & 0xff
177 |         rr.append(Rela(off, sym, type, add) if rela else Rel(off, sym, type))
178 | 
179 |     return rr
180 | 
181 | def parse_shdr32(data: bytes, shoff: int, shentsz: int, shnum: int,
182 |                  shstrndx: int) -> Sequence[Shdr]:
183 |     if shnum*shentsz+shoff > len(data) or shentsz==0 or shnum==0 or shoff==0:
184 |         print("snum*shentsz+shoff",shnum*shentsz+shoff)
185 |         print("len(data)",len(data))
186 |         print("shentsz",shentsz)
187 |         print("shnum",shnum)
188 |         print("shoff",shoff)
189 |         return []
190 | 
191 |     ss = []
192 |     for off in range(shoff, shoff+shentsz*shnum, shentsz):
193 |         noff, typ, flags, addr, off, size, link, info, align, entsz = \
194 |             unpack('<IIIIIIIIII', data[off:off+10*4])
195 |         s = Shdr(noff, typ, flags, addr, off, size, link, info, align, entsz)
196 |         ss.append(s)
197 | 
198 |     if shstrndx < shnum:
199 |         shstr = ss[shstrndx]
200 |         for i in range(len(ss)):
201 |             sname = readstr(data, shstr.offset + ss[i].name) \
202 |                 if ss[i].name < shstr.size else None
203 |             ss[i] = Shdr(sname, ss[i].type, ss[i].flags, ss[i].addr,
204 |                          ss[i].offset, ss[i].size, ss[i].link, ss[i].info,
205 |                          ss[i].addralign, ss[i].entsize)
206 | 
207 |     return ss
208 | 
209 | def parse_sym32(data: bytes, sym: Shdr, strt: Shdr) -> Sequence[Sym]:
210 |     ss = []
211 |     for off in range(sym.offset, sym.offset+sym.size, sym.entsize):
212 |         noff, val, sz, info, other, shndx = \
213 |             unpack('<IIIBBH', data[off:off+3*4+2+2])
214 | 
215 |         sn = readstr(data, strt.offset + noff) \
216 |             if noff < strt.size else None
217 |         s = Sym(sn, val, sz, (info & 15), (info >> 4), other, shndx)
218 |         ss.append(s)
219 |     return ss#sorted(ss, key=lambda x:x.value)
220 | 
221 | def parse_32(data: bytes) -> ELF:
222 |     ident  = data[:16]
223 |     eclass = data[4]
224 |     mach   = unpack('<H', data[18:18+2])[0]
225 |     entry  = unpack('<I', data[24:24+4])[0]
226 | 
227 |     phoff   = unpack('<I', data[28:28+4])[0]
228 |     shoff   = unpack('<I', data[32:32+4])[0]
229 |     phentsz = unpack('<H', data[42:42+2])[0]
230 |     phnum   = unpack('<H', data[44:44+2])[0]
231 |     shentsz = unpack('<H', data[46:46+2])[0]
232 |     shnum   = unpack('<H', data[48:48+2])[0]
233 |     shstrndx= unpack('<H', data[50:50+2])[0]
234 | 
235 |     phdrs = [] if phentsz == 0 else parse_phdr32(data, phoff, phentsz, phnum)
236 |     dyn   = None
237 | 
238 |     for p in phdrs:
239 |         if p.ptype == PT_DYNAMIC:
240 |             dyn = parse_dyn32(data, p)
241 |             break
242 | 
243 |     shdrs = parse_shdr32(data, shoff, shentsz, shnum, shstrndx)
244 |     #print("shdrs",shdrs)
245 | 
246 |     symtabsh = [s for s in shdrs if s.type == SHT_SYMTAB and s.name == ".symtab"]
247 |     strtabsh = [s for s in shdrs if s.type == SHT_STRTAB and s.name == ".strtab"]
248 |     dynsymsh = [s for s in shdrs if s.type == SHT_SYMTAB and s.name == ".dynsym"]
249 |     dynstrsh = [s for s in shdrs if s.type == SHT_STRTAB and s.name == ".dynstr"]
250 |     relash   = [s for s in shdrs if s.type == SHT_RELA]
251 |     relsh    = [s for s in shdrs if s.type == SHT_REL]
252 | 
253 |     #print("symtab",symtabsh)
254 |     #print("strtab",strtabsh)
255 | 
256 |     assert len(symtabsh) < 2
257 |     assert len(strtabsh) < 2
258 |     assert len(dynsymsh) < 2
259 |     assert len(dynstrsh) < 2
260 | 
261 |     symtab, dynsym = None, None
262 |     if len(symtabsh) and len(strtabsh):
263 |         symtab = parse_sym32(data, symtabsh[0], strtabsh[0]) \
264 |             if len(shdrs) > 0 else []
265 |     if len(dynsymsh) and len(dynstrsh):
266 |         dynsym = parse_sym32(data, symtabsh[0], strtabsh[0]) \
267 |             if len(shdrs) > 0 else []
268 | 
269 |     relocs = []
270 | 
271 |     # TODO: use sh.link to use the correct symbol table
272 |     for sh in relash:
273 |         relocs += parse_reloc32(data, sh.offset, sh.size//sh.entsize,
274 |                                 sh.entsize, symtab, True)
275 |     for sh in relsh:
276 |         relocs += parse_reloc32(data, sh.offset, sh.size//sh.entsize,
277 |                                 sh.entsize, symtab, False)
278 |     # TODO: relocs from DT_RELA, DT_REL
279 | 
280 |     return ELF(data, ident, eclass, mach, entry, phdrs, dyn, shdrs,
281 |                symtab, dynsym, relocs, True)
282 | 
283 | def parse_phdr64(data: bytes, phoff:int, phentsz:int, phnum:int) -> Sequence[Phdr]:
284 |     ps = []
285 |     for off in range(phoff, phoff+phentsz*phnum, phentsz):
286 |         # TODO # what is TODO exactly??
287 |         ptype, flags, off, vaddr, paddr, filesz, memsz, align = \
288 |             unpack('<IIQQQQQQ', data[off:off+2*4+6*8])
289 |         p = Phdr(ptype, off, vaddr, paddr, filesz, memsz, flags, align)
290 |         ps.append(p)
291 | 
292 |     return ps
293 | 
294 | def parse_dyn64(data: bytes, dynp: Phdr) -> Dyn:
295 |     ds = []
296 | 
297 |     off = dynp.off
298 |     while True:
299 |         tag, val = unpack('<QQ', data[off:off+2*8])
300 |         ds.append(Dyn(tag, val))
301 | 
302 |         if tag == DT_NULL: break
303 |         off = off + 2*8
304 | 
305 |     return ds
306 | 
307 | def parse_reloc64(data: bytes, reloff: int, nrel: int, entsz: int, syms: Sequence[Sym], rela: bool) -> Reloc:
308 |     rr=[]
309 | 
310 |     for off in range(reloff, reloff+entsz*nrel, entsz):
311 |         off, inf, add = unpack('<QQq', data[off:(off+24)]) if rela \
312 |             else (*unpack('<Qq', data[off:(off+16)]),0)
313 | 
314 |         sym = syms[inf >> 32]
315 |         type = inf & 0xffffffff
316 |         rr.append(Rela(off, sym, type, add) if rela else Rel(off, sym, type))
317 | 
318 |     return rr
319 | 
320 | def parse_shdr64(data: bytes, shoff: int, shentsz: int, shnum: int,
321 |                  shstrndx: int) -> Sequence[Shdr]:
322 | 
323 |     if shnum*shentsz+shoff > len(data) or shentsz==0 or shnum==0 or shoff==0:
324 |         return []
325 | 
326 |     ss = []
327 |     for off in range(shoff, shoff+shentsz*shnum, shentsz):
328 |         noff, typ, flags, addr, off, size, link, info, align, entsz = \
329 |             unpack('<IIQQQQIIQQ', data[off:off+4*4+6*8])
330 |         s = Shdr(noff, typ, flags, addr, off, size, link, info, align, entsz)
331 |         ss.append(s)
332 | 
333 |     if shstrndx < shnum:
334 |         shstr = ss[shstrndx]
335 |         for i in range(len(ss)):
336 |             sname = readstr(data, shstr.offset + ss[i].name) \
337 |                 if ss[i].name < shstr.size else None
338 |             ss[i] = Shdr(sname, ss[i].type, ss[i].flags, ss[i].addr,
339 |                          ss[i].offset, ss[i].size, ss[i].link, ss[i].info,
340 |                          ss[i].addralign, ss[i].entsize)
341 | 
342 |     return ss
343 | 
344 | def parse_sym64(data: bytes, sym: Shdr, strt: Shdr) -> Sequence[Sym]:
345 |     ss = []
346 |     for off in range(sym.offset, sym.offset+sym.size, sym.entsize):
347 |         noff, info, other, shndx, value, sz = \
348 |             unpack('<IBBHQQ', data[off:off+4+2+2+8*2])
349 | 
350 |         sn = readstr(data, strt.offset + noff) \
351 |             if noff < strt.size else None
352 |         s = Sym(sn, value, sz, (info & 15), (info >> 4), other, shndx)
353 |         ss.append(s)
354 |     return ss#sorted(ss, key=lambda x:x.value)
355 | 
356 | def parse_64(data: bytes) -> ELF:
357 |     ident  = data[:16]
358 |     eclass = data[4]
359 |     mach   = unpack('<H', data[18:18+2])[0]
360 |     entry  = unpack('<Q', data[24:24+8])[0]
361 | 
362 |     phoff   = unpack('<Q', data[32:32+8])[0]
363 |     shoff   = unpack('<Q', data[40:40+8])[0]
364 |     phentsz = unpack('<H', data[54:54+2])[0]
365 |     phnum   = unpack('<H', data[56:56+2])[0]
366 |     shentsz = unpack('<H', data[58:58+2])[0]
367 |     shnum   = unpack('<H', data[60:60+2])[0]
368 |     shstrndx= unpack('<H', data[62:62+2])[0]
369 | 
370 |     phdrs = [] if phentsz == 0 else parse_phdr64(data, phoff, phentsz, phnum)
371 |     dyn   = None
372 | 
373 |     for p in phdrs:
374 |         if p.ptype == PT_DYNAMIC:
375 |             dyn = parse_dyn64(data, p)
376 |             break
377 | 
378 |     shdrs = [] if shentsz == 0 else parse_shdr64(data, shoff, shentsz, shnum, shstrndx)
379 | 
380 |     symtabsh = [s for s in shdrs if s.type == SHT_SYMTAB and s.name == ".symtab"]
381 |     strtabsh = [s for s in shdrs if s.type == SHT_STRTAB and s.name == ".strtab"]
382 |     dynsymsh = [s for s in shdrs if s.type == SHT_SYMTAB and s.name == ".dynsym"]
383 |     dynstrsh = [s for s in shdrs if s.type == SHT_STRTAB and s.name == ".dynstr"]
384 |     relash   = [s for s in shdrs if s.type == SHT_RELA]
385 |     relsh    = [s for s in shdrs if s.type == SHT_REL]
386 | 
387 |     assert len(symtabsh) < 2
388 |     assert len(strtabsh) < 2
389 |     assert len(dynsymsh) < 2
390 |     assert len(dynstrsh) < 2
391 | 
392 |     symtab, dynsym = None, None
393 |     if len(symtabsh) and len(strtabsh):
394 |         symtab = parse_sym64(data, symtabsh[0], strtabsh[0]) \
395 |             if len(shdrs) > 0 else []
396 |     if len(dynsymsh) and len(dynstrsh):
397 |         dynsym = parse_sym64(data, symtabsh[0], strtabsh[0]) \
398 |             if len(shdrs) > 0 else []
399 | 
400 |     relocs = []
401 | 
402 |     # TODO: use sh.link to use the correct symbol table
403 |     for sh in relash:
404 |         relocs += parse_reloc32(data, sh.offset, sh.size//sh.entsize,
405 |                                 sh.entsize, symtab, True)
406 |     for sh in relsh:
407 |         relocs += parse_reloc32(data, sh.offset, sh.size//sh.entsize,
408 |                                 sh.entsize, symtab, False)
409 |     # TODO: relocs from DT_RELA, DT_REL
410 | 
411 |     return ELF(data, ident, eclass, mach, entry, phdrs, dyn, shdrs,
412 |                symtab, dynsym, relocs, False)
413 | 
414 | def parse(data: bytes) -> ELF:
415 |     assert data[:4] == b'\x7FELF', "Not a valid ELF file" # good enough
416 | 
417 |     ecls  = data[4]
418 |     if ecls == ELFCLASS32: return parse_32(data)
419 |     elif ecls == ELFCLASS64: return parse_64(data)
420 |     else:
421 |         emch = unpack('<H', data[18:18+2])[0]
422 |         if emch == EM_386: return parse_32(data)
423 |         elif emch == EM_X86_64: return parse_64(data)
424 | 
425 |         assert False, "bad E_CLASS %d" % ecls
426 | 
427 | 


--------------------------------------------------------------------------------
/contrib/linkmap.py:
--------------------------------------------------------------------------------
  1 | 
  2 | import re
  3 | from typing import *
  4 | 
  5 | class CommonSym(NamedTuple):
  6 |     name: str
  7 |     size: int
  8 |     file: str
  9 | class Discard(NamedTuple):
 10 |     name: str
 11 |     size: int
 12 |     file: str
 13 | class MemCfg(NamedTuple):
 14 |     name: str
 15 |     org : int
 16 |     size: int
 17 | class MMap(NamedTuple):
 18 |     sect: str
 19 |     org : int
 20 |     #size: int
 21 |     sym : str
 22 |     file: str
 23 | class XRef(NamedTuple):
 24 |     name: str
 25 |     deff: str
 26 |     reff: Sequence[str]
 27 | class ImpObj(NamedTuple):
 28 |     ar: str
 29 |     obj: str
 30 | class ImpRsnObj(NamedTuple):
 31 |     sym: str
 32 |     obj: ImpObj
 33 | class ImpRsnSym(NamedTuple):
 34 |     sym: str
 35 | class ArImp(NamedTuple):
 36 |     impobj: ImpObj
 37 |     reason: Union[ImpRsnSym, ImpRsnObj]
 38 | 
 39 | class LinkMap(NamedTuple):
 40 |     common : Sequence[CommonSym]
 41 |     discard: Sequence[Discard]
 42 |     memcfg : Sequence[MemCfg]
 43 |     mmap   : Sequence[MMap]
 44 |     xref   : Sequence[XRef]
 45 |     arimp  : Sequence[ArImp]
 46 | 
 47 | def parse_common( ls: Sequence[str]) -> Sequence[CommonSym]: return [] # TODO
 48 | def parse_discard(ls: Sequence[str]) -> Sequence[Discard  ]: return [] # TODO
 49 | def parse_memcfg( ls: Sequence[str]) -> Sequence[MemCfg   ]: return [] # TODO
 50 | def parse_xref(   ls: Sequence[str]) -> Sequence[XRef     ]: return [] # TODO
 51 | def parse_arimp(  ls: Sequence[str]) -> Sequence[ArImp    ]: return [] # TODO
 52 | 
 53 | def parse_mmap(ls: Sequence[str]) -> Sequence[MMap]:
 54 |     rrr = []
 55 | 
 56 |     bigsect = None
 57 |     section = None
 58 |     curfile = None
 59 |     #size = -1
 60 | 
 61 |     for l in ls:
 62 |         def matcher(mobj):
 63 |             return mobj.group(0).replace(' ', '_')
 64 |         l = re.sub(r"\*\(.*\)", matcher, l)
 65 |         #print(repr(l))
 66 |         s = l.strip(); w = s.split()
 67 | 
 68 |         if s.startswith('LOAD ') or s.startswith('OUTPUT(') or \
 69 |            s.startswith('START GROUP') or s.startswith('END GROUP'):
 70 |             continue#break
 71 | 
 72 |         if l[0] != ' ':
 73 |             bigsect = w[0]
 74 |             del w[0]
 75 |         elif l[1] != ' ':
 76 |             section = w[0]
 77 |             del w[0]
 78 | 
 79 |         if len(w) == 0 or w[0] == "[!provide]":
 80 |             continue # addr placed on next line for prettyprinting reasons
 81 | 
 82 |         #print(repr(l), w[0])
 83 |         assert w[0].startswith("0x"), "welp, bad symbol addr %s"%w[0]
 84 | 
 85 |         addr = int(w[0], 16)
 86 | 
 87 |         size = -1
 88 |         symn = ""
 89 |         if w[1].startswith("0x"): # filename will prolly follow
 90 |             size = int(w[1], 16)
 91 |             curfile = w[2] if len(w) > 2 else ""
 92 |         else: symn = w[1]
 93 | 
 94 |         if len(symn) > 0:
 95 |             rrr.append(MMap(section, addr, symn, curfile))
 96 | 
 97 |     return sorted(rrr, key=lambda m: m.org)
 98 | 
 99 | def parse(s: str) -> LinkMap:
100 |     COMMON  = 0
101 |     DISCARD = 1
102 |     MEMCFG  = 2
103 |     MMAP    = 3
104 |     XREF    = 4
105 |     ARIMP   = 5
106 | 
107 |     curpt = -1
108 | 
109 |     commonl, discardl, memcfgl, mmapl, xrefl, arimpl = [], [], [], [], [], []
110 | 
111 |     for l in s.split('\n'):
112 |         if len(l.strip()) == 0: continue
113 | 
114 |         ls = l.strip()
115 |         if ls == "Allocating common symbols": curpt = COMMON
116 |         elif ls == "Discarded input sections": curpt = DISCARD
117 |         elif ls == "Memory Configuration": curpt = MEMCFG
118 |         elif ls == "Linker script and memory map": curpt = MMAP
119 |         elif ls == "Cross Reference Table": curpt = XREF
120 |         elif ls == "As-needed library included to satisfy reference by file (symbol)": curpt = ARIMP
121 |         elif ls == "Archive member included to satisfy reference by file (symbol)": curpt = ARIMP
122 |         elif curpt == COMMON :  commonl.append(l)
123 |         elif curpt == DISCARD: discardl.append(l)
124 |         elif curpt == MEMCFG :  memcfgl.append(l)
125 |         elif curpt == MMAP   :    mmapl.append(l)
126 |         elif curpt == XREF   :    xrefl.append(l)
127 |         elif curpt == ARIMP  :   arimpl.append(l)
128 |         else:
129 |             assert False, "bad line %s" % ls
130 | 
131 |     return LinkMap(parse_common(commonl), parse_discard(discardl),
132 |                    parse_memcfg(memcfgl), parse_mmap(mmapl), parse_xref(xrefl),
133 |                    parse_arimp(arimpl))
134 | 
135 | 


--------------------------------------------------------------------------------
/contrib/parsemap.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | 
  3 | import sys, subprocess, argparse, lzma
  4 | from typing import *
  5 | 
  6 | import linkmap, hackyelf
  7 | from linkmap import MMap, LinkMap
  8 | from hackyelf import ELF, Phdr, Dyn
  9 | 
 10 | class SymOff(NamedTuple):
 11 |     sym: str
 12 |     off: int
 13 |     bss: bool
 14 | class SymWeight(NamedTuple):
 15 |     sof: SymOff
 16 |     wgt: float
 17 |     start: int
 18 |     dsize: int
 19 |     wwgt: float
 20 | 
 21 | def mem2off(elf: ELF, memaddr: int) -> Optional[int]:
 22 |     for p in elf.phdrs:
 23 |         if memaddr >= p.vaddr and memaddr < p.vaddr + p.filesz:
 24 |             offoff = memaddr - p.vaddr
 25 |             return p.off + offoff
 26 |         elif memaddr >= p.vaddr and memaddr < p.vaddr + p.memsz:
 27 |             offoff = memaddr - p.vaddr
 28 |             return -(p.off + offoff) # BSS
 29 | 
 30 |     return None
 31 | 
 32 | def mkst(recs: Dict[str, str], elf: ELF, mmap: Sequence[MMap], pfix="") -> Sequence[SymOff]:
 33 |     lll = []
 34 |     for x in mmap:
 35 |         off = mem2off(elf, x.org)
 36 |         if off is None: # not found?
 37 |             # -> normal for eg *_size syms defined in a linnker script
 38 |             #print("W: sym %s$%s @ 0x%x not found!" % (x.sect, x.sym, x.org))
 39 |             continue
 40 | 
 41 |         if x.sym in recs:
 42 |             elf2 = hackyelf.parse(elf.data[abs(off):])
 43 |             mmap2 = None
 44 |             with open(recs[x.sym], 'r') as rcsm:
 45 |                 mmap2 = linkmap.parse(rcsm.read()).mmap
 46 | 
 47 |             for y in mkst(recs, elf2, mmap2, x.sym+' -> '):
 48 |                 lll.append(SymOff(pfix+y.sym, abs(off)+y.off, \
 49 |                                   off<0 or y.off<0))
 50 |         else:
 51 |             xn = x.sym if x.sym != '.' else (x.sect or ("[0x%08x]"%x.org))
 52 |             lll.append(SymOff(pfix+xn, abs(off), off < 0))
 53 | 
 54 |     return sorted(lll, key=lambda s: s.off)
 55 | 
 56 | def weightsyms(symtab: Sequence[SymOff], weights: Sequence[float]):
 57 |     curs = 0
 58 |     totalw = 0.0
 59 |     lll = []
 60 |     start = 0
 61 |     for i in range(len(weights)):
 62 |         # next sym!
 63 |         if len(symtab) > curs+1 and i >= symtab[curs+1].off:
 64 |             if i-start > 0:
 65 |                 lll.append(SymWeight(symtab[curs], totalw, start, i-start, totalw/(i-start)))
 66 |             start = i
 67 |             curs = curs + 1
 68 |             #print("going to %s, was %f" % (symtab[curs].sym, totalw))
 69 |             totalw = 0.0
 70 | 
 71 |         totalw = totalw + weights[i]
 72 | 
 73 |     if start < len(weights):
 74 |         lll.append(SymWeight(symtab[curs], totalw, start, len(weights)-start, totalw/(len(weights)-start)))
 75 |     return sorted(lll, key=lambda x: x.wgt, reverse=True)
 76 | 
 77 | def splitr(s):
 78 |     assert '=' in s, "Bad --recurse format %s, see --help" % repr(s)
 79 | 
 80 |     i = s.index('=')
 81 |     return s[:i], s[i+1:]
 82 | 
 83 | def main(opts):
 84 |     recs = dict([splitr(s) for s in (opts.recurse or [])])
 85 | 
 86 |     weights = [float(x.strip()) for x in \
 87 |                subprocess.check_output([opts.lzmaspec, "--raw", opts.lzma_file]) \
 88 |                     .decode('utf-8').split('\n') \
 89 |                if len(x) > 0]
 90 | 
 91 |     elfb = None
 92 |     with lzma.open(opts.lzma_file, 'rb') as lf: elfb = lf.read()
 93 | 
 94 |     maps = opts.map_file.read()
 95 |     elf  = hackyelf.parse(elfb)
 96 |     mmap = linkmap.parse(maps).mmap
 97 | 
 98 |     symtab = mkst(recs, elf, mmap)
 99 | 
100 |     print("Symbol                    Uncompr. Size\t\tPerplexity\tPerplexity/Size")
101 |     print("-"*79)
102 |     totals, totalw, totalww = 0,0,0
103 |     for x in weightsyms(symtab, weights):
104 |         symn = x.sof.sym
105 |         symn = (symn + ' ' * (34 - len(symn)))[:34] # 34 is good enough
106 |         print("%s%5d\t\t%10.2f\t%15.2f" % (symn, x.dsize, x.wgt, x.wwgt))
107 |         totals += x.dsize
108 |         totalw += x.wgt
109 |         totalww+= x.wwgt
110 |     print("-"*79)
111 |     print("Total:%33d\t\t%10.2f\t%14.1f%%" % (totals, totalw, 100*totalw/totals))
112 | 
113 |     return 0
114 | 
115 | if __name__ == '__main__':
116 |     p = argparse.ArgumentParser(description="""\
117 | Shows a summary of the compression stats of every symbol in an ELF,
118 | given the compressed and uncompressed ELF files, as well as a linker
119 | map file. (`ld -Map', `cc -Wl,-Map', see respective manpages.)
120 | """)
121 | 
122 |     p.add_argument("lzma_file", type=str, \
123 |                    help="The LZMA-compressed ELF file")
124 |     p.add_argument("map_file", type=argparse.FileType('r'), \
125 |                    help="The linker map file")
126 | 
127 |     p.add_argument("--recurse", action='append', help="Recursively analyse "+\
128 |                    "data in symbols. Syntax: --recurse symname=linker.map")
129 | 
130 |     p.add_argument("--lzmaspec", type=str, default="./LzmaSpec", \
131 |                    help="LzmaSpec binary to use (default: ./LzmaSpec)")
132 | 
133 |     exit(main(p.parse_args()))
134 | 
135 | 


--------------------------------------------------------------------------------
/realcolor.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <iostream>
 4 | #include <cstdio>
 5 | #include <functional>
 6 | #include <sstream>
 7 | 
 8 | namespace realcolor
 9 | {
10 | 	namespace
11 | 	{
12 | 		constexpr int red(int rgb)
13 | 		{
14 | 			return (rgb >> 16) & 0xFF;
15 | 		}
16 | 
17 | 		constexpr int green(int rgb)
18 | 		{
19 | 			return (rgb >> 8) & 0xFF;
20 | 		}
21 | 
22 | 		constexpr int blue(int rgb)
23 | 		{
24 | 			return rgb & 0xFF;
25 | 		}
26 | 	};
27 | 
28 | 	inline std::ostream& reset(std::ostream& stream)
29 | 	{
30 | 		stream << "\x1b[0m";
31 | 		return stream;
32 | 	}
33 | 
34 | 	inline std::string fg(int r, int g, int b)
35 | 	{
36 | 		std::stringstream color;
37 | 		color << "\x1b[38;2;" << r << ";" << g << ";" << b << "m";
38 | 		return color.str();
39 | 	}
40 | 
41 | 	inline std::string bg(int r, int g, int b)
42 | 	{
43 | 		std::stringstream color;
44 | 		color << "\x1b[48;2;" << r << ";" << g << ";" << b << "m";
45 | 		return color.str();
46 | 	}
47 | 
48 | 	//don't be scared of this template magic, this is just so we can pass all floating point types in (floats, doubles, etc)
49 | 	template<typename floatType>
50 | 	inline typename std::enable_if<std::is_floating_point<floatType>::value, std::string>::type
51 | 	fg(floatType r, floatType g, floatType b)
52 | 	{
53 | 		return fg(static_cast<int>(r*0xFF), static_cast<int>(g*0xFF), static_cast<int>(b*0xFF));
54 | 	}
55 | 
56 | 	template<typename floatType>
57 | 	inline typename std::enable_if<std::is_floating_point<floatType>::value, std::string>::type
58 | 	bg(floatType r, floatType g, floatType b)
59 | 	{
60 | 		return bg(static_cast<int>(r*0xFF), static_cast<int>(g*0xFF), static_cast<int>(b*0xFF));
61 | 	}
62 | 
63 | 	inline std::string fg(int rgb)
64 | 	{
65 | 		return fg(red(rgb), green(rgb), blue(rgb));
66 | 	}
67 | 
68 | 	inline std::string bg(int rgb)
69 | 	{
70 | 		return bg(red(rgb), green(rgb), blue(rgb));
71 | 	}
72 | };
73 | 


--------------------------------------------------------------------------------
/small-lzma.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/blackle/LZMA-Vizualizer/fe4ea87e6a068bc5f7bcd6e009c713d4ceb30cb8/small-lzma.png


--------------------------------------------------------------------------------