├── LICENSE
├── Lib3
    ├── common.inc
    ├── coro2pair.inc
    ├── coro3_pin.inc
    ├── coro3_pin_0.inc
    ├── coro3_setjmp_x32.h
    ├── coro3_setjmp_x64.h
    ├── coro3b.inc
    ├── coro_fh_inp.inc
    ├── coro_fh_out.inc
    ├── coro_fhp2.inc
    └── file_api_std.inc
├── README.md
├── c.bat
├── c32.bat
├── g
├── g.bat
├── g32.bat
├── gc.bat
├── icl64.cfg
├── shar64
├── src
    ├── archive.inc
    ├── dir_scan.inc
    ├── direntry.inc
    ├── dirparse.inc
    ├── dirparse2.inc
    ├── dirstore.inc
    ├── main_utf8_wrap.inc
    ├── memblk.inc
    ├── setjmp.lib
    ├── shar.cpp
    ├── stream_unwrap.inc
    ├── stream_wrap.inc
    ├── stream_wrapper.inc
    ├── usage.inc
    └── usage.txt
├── t.bat
├── txt2inc.pl
├── vc.bat
└── zstd
    ├── bitstream.h
    ├── dirent.h
    ├── entropy_common.inc
    ├── error_private.h
    ├── error_private.inc
    ├── fse.h
    ├── fse_compress.inc
    ├── fse_decompress.inc
    ├── huf.h
    ├── huf_compress.inc
    ├── huf_decompress.inc
    ├── mem.h
    ├── xxhash.h
    ├── xxhash.inc
    ├── zbuff.h
    ├── zbuff_compress.inc
    ├── zbuff_decompress.inc
    ├── zstd.cpp
    ├── zstd.h
    ├── zstd.inc
    ├── zstd_common.inc
    ├── zstd_compress.inc
    ├── zstd_decompress.inc
    ├── zstd_errors.h
    ├── zstd_internal.h
    └── zstd_opt.h


/Lib3/common.inc:
--------------------------------------------------------------------------------
  1 | 
  2 | #pragma once
  3 | 
  4 | #include <stdio.h>
  5 | #include <stdlib.h>
  6 | #include <stddef.h>
  7 | #include <string.h>
  8 | #include <memory.h>
  9 | #undef EOF
 10 | 
 11 | #pragma pack(1)
 12 | 
 13 | typedef unsigned short word;
 14 | typedef unsigned int   uint;
 15 | typedef unsigned char  byte;
 16 | typedef unsigned long long qword;
 17 | typedef signed long long sqword;
 18 | 
 19 | 
 20 | template <class T> void bzero( T &_p ) { int i; byte* p = (byte*)&_p; for( i=0; i<sizeof(_p); i++ ) p[i]=0; }
 21 | //template <class T, int N> void bzero( T (&p)[N] ) { int i; for( i=0; i<N; i++ ) p[i]=0; }
 22 | 
 23 | template <class T, int N> void bzero( T (&p)[N] ) { 
 24 |   byte* q = (byte*)&p; uint _N=N*sizeof(T);
 25 |   uint i; for( i=0; i<_N; i++ ) q[i]=0; 
 26 | }
 27 | 
 28 | template <class T> void bzero( T* p, int N ) { int i; for( i=0; i<N; i++ ) p[i]=0; }
 29 | 
 30 | template <class T, int N, int M> void bzero( T (&p)[N][M] ) { int i; for( i=0; i<N*M; i++ ) p[0][i]=0; }
 31 | 
 32 | template <class T> T Min( T x, T y ) { return (x<y) ? x : y; }
 33 | template <class T> T Max( T x, T y ) { return (x>y) ? x : y; }
 34 | 
 35 | #define macro_Min( x, y ) (((x)<(y)) ? (x) : (y))
 36 | #define macro_Max( x, y ) (((x)>(y)) ? (x) : (y))
 37 | 
 38 | template <class T,int N> constexpr int DIM( T (&wr)[N] ) { return sizeof(wr)/sizeof(wr[0]); };
 39 | #define AlignUp(x,r) ((x)+((r)-1))/(r)*(r)
 40 | template<byte a,byte b,byte c,byte d> struct wc { 
 41 |   static const unsigned int n=(d<<24)+(c<<16)+(b<<8)+a; 
 42 |   static const unsigned int x=(a<<24)+(b<<16)+(c<<8)+d;
 43 | };
 44 | 
 45 | #ifdef __GNUC__
 46 |  #define INLINE   __attribute__((always_inline)) 
 47 |  #define NOINLINE __attribute__((noinline))
 48 |  #define ALIGN(n) __attribute__((aligned(n)))
 49 | // #define __assume_aligned(x,y) x=(byte*)__builtin_assume_aligned((void*)x,y)
 50 |  #define __assume_aligned(x,y) (x=decltype(x)(__builtin_assume_aligned((void*)x,y)))
 51 |  #define restrict __restrict
 52 | #else
 53 |  #define INLINE   __forceinline
 54 |  #define NOINLINE __declspec(noinline)
 55 |  #define ALIGN(n) __declspec(align(n))
 56 | #endif
 57 | 
 58 | #define if_e0(x) if(__builtin_expect((x),0))
 59 | #define if_e1(x) if(__builtin_expect((x),1))
 60 | #define for_e0(x,y,z) for( (x); __builtin_expect((y),0); (z) )
 61 | #define for_e1(x,y,z) for( (x); __builtin_expect((y),1); (z) )
 62 | 
 63 | #if defined(_MSC_VER) && !defined(__INTEL_COMPILER)
 64 |  #define __builtin_expect(x,y) (x)
 65 | // #define __assume_aligned(x,y) 
 66 |  #define __assume_aligned(x,y) __assume( (((byte*)x)-((byte*)0))%(y)==0 )
 67 |  #define restrict __restrict
 68 |  //#include "intrin.h"
 69 |  #ifndef COMMON_SKIP_BSF
 70 |  extern "C" {
 71 |  byte __cdecl _BitScanForward( uint* _Index, uint _Mask);
 72 |  byte __cdecl _BitScanForward64( uint* _Index, qword _Mask );
 73 |  }
 74 |  #endif
 75 | #endif
 76 | 
 77 | #if !defined(_MSC_VER) && !defined(__INTEL_COMPILER)
 78 |  #define __assume(x) (x)
 79 | #endif
 80 | 
 81 | #ifdef INC_FLEN
 82 | static uint flen( FILE* f ) {
 83 |   fseek( f, 0, SEEK_END );
 84 |   uint len = ftell(f);
 85 |   fseek( f, 0, SEEK_SET );
 86 |   return len;
 87 | }
 88 | #endif
 89 | 
 90 | #ifdef INC_LOG2I
 91 | static uint log2i( uint x ) {
 92 | #if ((defined __GNUC__) || (defined __INTEL_COMPILER))
 93 |  #ifdef __GNUC__
 94 |   return 31-__builtin_clz(x);
 95 |  #else
 96 |   return _bit_scan_reverse(x);
 97 |  #endif
 98 | #else
 99 |   uint i; 
100 |   for( i=0; i<32; i++,x>>=1 ) if( x==0 ) break;
101 |   return i-1;
102 | #endif
103 | }
104 | #endif
105 | 
106 | #if defined(__x86_64) || defined(_M_X64)
107 |  #define X64
108 |  #define X64flag 1
109 | #else
110 |  #undef X64
111 |  #define X64flag 0
112 | #endif
113 | 


--------------------------------------------------------------------------------
/Lib3/coro2pair.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | template < class Model1, class Model2 > 
 3 | struct CoroutinePair : Coroutine {
 4 | 
 5 |   ALIGN(4096) Model1 M1;
 6 |   ALIGN(4096) Model2 M2;
 7 | 
 8 |   enum { midbufsize = 1<<16 };
 9 |   ALIGN(4096) byte midbuf[midbufsize+16];
10 | 
11 |   void init( void ) {
12 | //printf( "%I64X: coropair init\n", this );
13 |     coro_init();
14 |     M1.coro_init();
15 |     M2.coro_init();
16 |   }
17 | 
18 |   void do_process( void ) {
19 |     uint r1,r2,l;
20 |     static byte* sEOF = (byte*)"\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF";
21 | 
22 | //printf( "%I64X: inpptr=%I64X outptr=%I64X il=%i ol=%i \n", this, inpptr,outptr, getinplen(), getoutleft() ); fflush(stdout);
23 | 
24 |     M1.coro_init();
25 |     M2.coro_init();
26 | 
27 |     M1.addout( midbuf, midbufsize );
28 |     M2.addout( (byte*)outptr, getoutleft() );
29 |     M2.coro_call(&M2); // should be at r1
30 | 
31 | //printf( "outptr=%I64X M2.outptr=%I64X\n", outptr, M2.outptr );
32 | 
33 |     while( 1 ) {
34 |       r1 = M1.coro_call(&M1); 
35 | //printf( "!r1=%i f_quit=%i l=%i!\n", r1, M1.f_quit, M1.getoutsize() );
36 | 
37 |       if( r1==1 ) {
38 | 
39 |         //inpptr=inpend; 
40 |         chkinp(); // yield(this,1);
41 |         l = getinpleft();
42 |         if( l==0 ) { M1.f_quit=1; M1.addinp( sEOF, 0 ); } else
43 |         M1.addinp( (byte*)inpptr, l ); 
44 |         inpptr+=l; 
45 | 
46 |       } else if( (r1==2) || (r1==0) ) {
47 | 
48 |         if( r1==0 ) inpptr=M1.inpptr;
49 | 
50 |         l = M1.getoutsize(); //M1.outptr-M1.outbeg;
51 | 
52 |         Retry:
53 |         if( l==0 ) { M2.f_quit=1; M2.addinp( sEOF, 0 ); } else
54 |         M2.addinp( (byte*)M1.outbeg, l );
55 | 
56 |         while(1) {
57 | //printf( "{\n" ); fflush(stdout);
58 |           r2 = M2.coro_call(&M2); 
59 | //printf( "}r2=%i f_quit=%i l=%i!\n", r2, M2.f_quit, M2.getoutsize() );
60 | 
61 |           if( r2==2 ) {
62 |             outend = outptr = M2.outptr; chkout(); //yield(this,2);
63 |             M2.addout( (byte*)outptr, getoutleft() );
64 |             continue;
65 |           }
66 |           if( (r2==1) && (r1==0) ) { l=0; goto Retry; }
67 |           break;
68 |         }
69 | 
70 |         if( (r1==0) || (r2==0) ) break;
71 | 
72 |         M1.addout( midbuf, midbufsize );
73 |       }
74 |     }
75 | 
76 |     outbeg = M2.outbeg; outend = outptr = M2.outptr;
77 | 
78 |     yield(this,0);
79 |   }
80 | 
81 | };
82 | 


--------------------------------------------------------------------------------
/Lib3/coro3_pin.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | #include "coro3_pin_0.inc"
 3 | 
 4 | struct coro3_pin: coro3_pin_0 {
 5 |   typedef Coroutine wrap;
 6 | 
 7 |   void pin_init( wrap* that, uint _r_code ) {
 8 |     ptr=beg=end=0; f_EOF=0;
 9 |     base_offs = ((char*)this) - ((char*)that);
10 |     r_code = _r_code;
11 |   }
12 | 
13 |   void yield_r( void ) {
14 |     wrap& W = *(wrap*)(((char*)this) - base_offs );
15 |     yield( (void*)&W, r_code );
16 |   }
17 | 
18 |   inline uint f_quit( void );
19 | 
20 | #define coro3_pin_DEFINE_f_quit                     \
21 |   inline uint coro3_pin::f_quit( void ) {           \
22 |     wrap& W = *(wrap*)(((char*)this) - base_offs ); \
23 |     return f_EOF | W.f_quit;                        \
24 |   }                                                 
25 | 
26 |   void chkinp( void ) { if_e0( ptr>=end ) yield_r(); }
27 | 
28 |   void chkout( uint d=0 ) { if_e0( ptr>=end-d ) yield_r(); }
29 | 
30 |   byte get0( void ) { return *ptr++; }
31 |   void put0( uint c ) { *ptr++ = c; }
32 | 
33 |   uint get( void ) { 
34 |     m0:
35 |     if_e0( ptr>=end ) {
36 |       if_e0( f_quit() ) return uint(-1);
37 |       yield_r();
38 |       goto m0;
39 |     }
40 |     return *ptr++; 
41 |   }
42 | 
43 |   void put( uint c ) { 
44 |     *ptr++ = c; chkout(); 
45 |   }
46 | 
47 | };
48 | 


--------------------------------------------------------------------------------
/Lib3/coro3_pin_0.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | #pragma once
 3 | 
 4 | struct coro3_pin_0 {
 5 |   byte* ptr;
 6 |   byte* beg;
 7 |   byte* end;
 8 |   uint  f_EOF;
 9 |   word  base_offs;
10 |   word  r_code;
11 | 
12 |   uint getinplen()  { return end-ptr; } //-V110
13 |   uint getinpleft() { return end-ptr; } //-V110
14 |   uint getinpsize() { return ptr-beg; } //-V110
15 | 
16 |   uint getoutlen()  { return end-ptr; } //-V110
17 |   uint getoutleft() { return end-ptr; } //-V110
18 |   uint getoutsize() { return ptr-beg; } //-V110
19 | 
20 |   void addinp( byte* inp,uint inplen ) { addbuf(inp,inplen); }
21 |   void addout( byte* out,uint outlen ) { addbuf(out,outlen); }
22 | 
23 |   void addbuf( byte* buf,uint len ) {
24 |     beg = ptr = buf;
25 |     end = &buf[len];
26 |   }
27 | 
28 | };
29 | 


--------------------------------------------------------------------------------
/Lib3/coro3_setjmp_x32.h:
--------------------------------------------------------------------------------
 1 | 
 2 | struct my_jmpbuf {
 3 |   uint eip,esp;
 4 | };
 5 | 
 6 | #define ASM __asm__ volatile
 7 | 
 8 | INLINE
 9 | int my_setjmp( my_jmpbuf* regs ) {
10 |   int r;
11 | 
12 |   ASM ("\
13 |    movl %%esp,4(%1); \
14 |    call 1f; \
15 | 1: popl 0(%1); \
16 |   " : "=a"(r) : "b"(regs),"a"(0) : "%ecx","%edx","%esi","%edi","%ebp"
17 |   );
18 | 
19 |   return r;
20 | }
21 | 
22 | INLINE
23 | void my_jmp( my_jmpbuf* regs, int ) {
24 |   ASM ("\
25 |   xchg %0,%%esp; \
26 |   jmp *%1; \
27 |   " :  : "d"(regs->esp),"b"( ((byte*)regs->eip)+2 ),"a"(1) : 
28 |   );
29 | }
30 | 
31 | typedef my_jmpbuf m_jmp_buf[1];
32 | #define jmp_buf m_jmp_buf
33 | #define longjmp my_jmp
34 | #define setjmp  my_setjmp
35 | 
36 | 


--------------------------------------------------------------------------------
/Lib3/coro3_setjmp_x64.h:
--------------------------------------------------------------------------------
 1 | 
 2 | struct my_jmpbuf {
 3 |   qword rip,rsp;
 4 | };
 5 | 
 6 | #define ASM __asm__ volatile
 7 | 
 8 | INLINE
 9 | int my_setjmp( my_jmpbuf* regs ) {
10 |   qword r;
11 | 
12 |   ASM ("\
13 |    movq %%rsp,8(%1); \
14 |    call 1f; \
15 | 1: popq 0(%1); \
16 |   " : "=a"(r) : "b"(regs),"a"(0) : "%rcx","%rdx","%rsi","%rdi","%rbp","%r8","%r9","%r10","%r11","%r12","%r13","%r14","%r15"
17 |   );
18 | 
19 |   return r;
20 | }
21 | 
22 | INLINE
23 | void my_jmp( my_jmpbuf* regs, int ) {
24 |   ASM ("\
25 |   xchg %0,%%rsp; \
26 |   jmp *%1; \
27 |   " :  : "d"(regs->rsp),"b"( ((byte*)regs->rip)+2 ),"a"(1) : 
28 |   );
29 | }
30 | 
31 | typedef my_jmpbuf m_jmp_buf[1];
32 | #define jmp_buf m_jmp_buf
33 | #define longjmp my_jmp
34 | #define setjmp  my_setjmp
35 | 
36 | 


--------------------------------------------------------------------------------
/Lib3/coro3b.inc:
--------------------------------------------------------------------------------
  1 | 
  2 | //#define CORO_NOASM 1
  3 | 
  4 | #if defined(_MSC_VER) || defined(__clang__)
  5 | #pragma runtime_checks( "scu", off )
  6 | #pragma check_stack(off)
  7 | #pragma strict_gs_check(off)
  8 | #endif
  9 | 
 10 | #if ((defined __GNUC__) || (defined __INTEL_COMPILER) || (defined __clang__)) && (!defined CORO_NOASM)
 11 |   #ifdef X64
 12 |     #include "coro3_setjmp_x64.h"
 13 | //    #include "coro3_setjmp_x64b.h"
 14 |   #else
 15 |     #include "coro3_setjmp_x32.h"
 16 | //    #include "coro3_setjmp_x32b.h"
 17 |   #endif
 18 | #else 
 19 |   #ifndef CORO_NOASM
 20 |   #define CORO_NOASM 1
 21 |   #endif
 22 |   #include <setjmp.h>
 23 | #endif
 24 | 
 25 | struct Coroutine;
 26 | static void yield( void* p, int value );
 27 | 
 28 | #include "coro3_pin.inc"
 29 | 
 30 | static uint coro_call0( Coroutine* that );
 31 | static void call_do_process0( Coroutine* that );
 32 | 
 33 | enum CAPI { 
 34 |   INIT=0,  // have to run Init()
 35 |   QUIT=1,  // all done, have to run Quit()
 36 |   FLUSH=2,
 37 |   PROG=3,  // processing possible
 38 |   DONE=4,  // Quit() complete
 39 |   ALLOC=8880,
 40 |   FREE=8881,
 41 | };
 42 | 
 43 | struct Coroutine {
 44 | 
 45 |   union {
 46 |     struct {
 47 |     byte* inpptr;
 48 |     byte* inpbeg;
 49 |     byte* inpend;
 50 |     uint  inp_f_EOF;
 51 |     uint  inp_pad_x64_;
 52 | 
 53 |     byte* outptr;
 54 |     byte* outbeg;
 55 |     byte* outend;
 56 |     uint  out_f_EOF;
 57 |     uint  out_pad_x64_;
 58 |     };
 59 |     coro3_pin pin[4];
 60 |   };
 61 | 
 62 |   volatile uint  state;
 63 |   volatile uint  f_quit;
 64 |            uint  f_init; // state of main routine
 65 | 
 66 |   ALIGN(32) jmp_buf PointA;
 67 |   ALIGN(32) jmp_buf PointB;
 68 | 
 69 |   ALIGN(8)
 70 |   volatile char* stkptrH;
 71 |   volatile char* stkptrL; // remembered _sp value for this instance
 72 | 
 73 |   typedef void (Coroutine::*t_do_process)( void );
 74 |   t_do_process p_do_process;
 75 | 
 76 |   enum{ STKPAD=4096*4 }; // coroutine stack size
 77 |   enum{ STKPAD0=1<<16 }; // stack padding from frontend to coroutine
 78 | 
 79 |   ALIGN(8) byte stk[STKPAD];
 80 | 
 81 |   void coro_init( void ) {
 82 |     f_init = CAPI::INIT;
 83 |     f_quit = 0;
 84 |     state = 0; 
 85 |     for( uint i=0; i<DIM(pin); i++ ) pin[i].pin_init( this, 1+i );
 86 |   }
 87 | 
 88 |   template <typename T> 
 89 |   INLINE
 90 |   uint coro_call( T* that ) {
 91 |     p_do_process = (t_do_process)&T::do_process;
 92 |     return coro_call0(that);
 93 |   }
 94 | 
 95 | //---------------------
 96 | 
 97 |   void chkinp( void ) { pin[0].chkinp(); }
 98 |   void chkout( uint d=0 ) { pin[1].chkout(d); }
 99 |   uint get( void ) { return pin[0].get(); }
100 |   void put( uint c ) { pin[1].put(c); }
101 | 
102 |   byte get0( void ) { return pin[0].get0(); }
103 |   void put0( uint c ) { pin[1].put0(c); }
104 | 
105 |   uint getinplen() { return pin[0].getinplen(); } //-V110
106 |   uint getoutlen() { return pin[1].getoutlen(); } //-V110
107 |   uint getinpleft() { return pin[0].getinpleft(); } //-V524 //-V110
108 |   uint getoutleft() { return pin[1].getoutleft(); } //-V524 //-V110
109 |   uint getinpsize() { return pin[0].getinpsize(); } //-V110
110 |   uint getoutsize() { return pin[1].getoutsize(); } //-V110
111 | 
112 |   void addinp( byte* inp,uint inplen ) { pin[0].addinp(inp,inplen); }
113 |   void addout( byte* out,uint outlen ) { pin[1].addout(out,outlen); }
114 | 
115 | };
116 | 
117 | 
118 | NOINLINE
119 | static void yield( void* p, int value ) { 
120 |   Coroutine& q = *(Coroutine*)p;
121 |   char curtmp; q.stkptrL=(&curtmp)-16;
122 |   if( setjmp(q.PointB)==0 ) { 
123 |     q.state=value; 
124 |     memcpy( q.stk, (char*)q.stkptrL, q.stkptrH-q.stkptrL );
125 |     longjmp(q.PointA,1); 
126 |     __assume(0);
127 |   }
128 | }
129 | 
130 | NOINLINE
131 | static uint coro_call0( Coroutine* that ) {
132 |   if_e1( setjmp(that->PointA)==0 ) {
133 |     if_e1( that->state ) { // calls usually take this path, since other runs only on init
134 |       memcpy( (char*)that->stkptrL, that->stk, that->stkptrH-that->stkptrL );
135 |       longjmp(that->PointB,1); 
136 |       __assume(0);
137 |     }
138 |     call_do_process0(that);
139 |     __assume(0);
140 |   }
141 |   return that->state;
142 | }
143 | 
144 | 
145 | NOINLINE
146 | static void call_do_process0( Coroutine* that ) {
147 |   // call_do_process0 needs to be an actual separate function to allocate stack pad in its frame
148 |   byte stktmp[Coroutine::STKPAD0]; 
149 |   that->stkptrH = ((char*)stktmp);
150 | 
151 |   // do_process also needs a separate stack frame, to avoid merging stktmp into it, but ptr call is ok
152 |   (that->*(that->p_do_process))();
153 | 
154 |   // do_process ends with yield(0) (can't normally return to changed frontend stack)
155 |   // so tell compiler that this point can't be reached
156 |   __assume(0);
157 | }
158 | 
159 | 
160 | coro3_pin_DEFINE_f_quit
161 | #undef coro3_pin_DEFINE_f_quit
162 | 
163 | //#include "coro3_init.inc"
164 | 
165 | 


--------------------------------------------------------------------------------
/Lib3/coro_fh_inp.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | template < class Model > 
 3 | struct CoroFileProcInp : Model {
 4 |   using Model::f_quit;
 5 | 
 6 |   enum { inpbufsize = 1<<16, outbufsize = 1<<16 };
 7 |   ALIGN(4096) byte inpbuf[inpbufsize];
 8 | 
 9 |   // read from file, process with coroutine, write using That coroutine
10 |   qword processfile( filehandle f, Coroutine* _That ) {
11 |     uint l,r;
12 |     qword c_out = 0;
13 |     Coroutine& Q = *_That;
14 | 
15 |     coro_init();
16 |     goto AddOut;
17 | 
18 |     while(1) {
19 |       r = coro_call(this); 
20 |       if( r==1 ) {
21 |         l = f.sread( inpbuf, inpbufsize );
22 |         if( l==0 ) f_quit=1; // get/put don't support repeated failures
23 |         addinp( inpbuf, l ); 
24 |       } else { // r0=quit, r3=error
25 |         l = getoutsize();
26 |         Q.outptr += l; c_out += l;
27 |         if( r!=2 ) break;
28 | AddOut: Q.chkout(); addout( (byte*)Q.outptr, Q.getoutlen() );
29 |       } // if
30 |     } // while
31 | 
32 |     return c_out;
33 |   } // func
34 | 
35 | };
36 | 
37 | 


--------------------------------------------------------------------------------
/Lib3/coro_fh_out.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | 
 3 | template < class Model > 
 4 | struct CoroFileProcOut : Model {
 5 |   using Model::f_quit;
 6 |   using Model::inpptr;
 7 |   using Model::inpend;
 8 | 
 9 |   enum { inpbufsize = 1<<16, outbufsize = 1<<16 };
10 |   ALIGN(4096) byte outbuf[outbufsize];
11 | 
12 |   void init( void ) {
13 |     coro_init();
14 |   }
15 | 
16 |   // read from given pin of That coroutine, process, write to file
17 |   qword processfile( filehandle g, Coroutine* _That, uint i=0 ) {
18 |     uint l,r;
19 |     qword c_out = 0;
20 |     Coroutine& Q = *_That;
21 | 
22 |     addout( outbuf, outbufsize );
23 | 
24 |     while(1) {
25 |       r = coro_call(this); 
26 |       if( r==1 ) {
27 |         Q.pin[i].chkinp(); l = Q.pin[i].getinplen();
28 |         if( l==0 ) f_quit=1;
29 |         addinp( (byte*)Q.pin[i].ptr, l ); Q.pin[i].ptr+=l;
30 |       } else { // r0=quit, r3=error
31 |         l = getoutsize();
32 |         if( l>0 ) g.writ( outbuf, l ); 
33 |         c_out += l;
34 |         if( r!=2 ) break;
35 |         addout( outbuf, outbufsize );
36 |       } // if
37 |     } // while
38 | 
39 |     Q.pin[i].ptr -= inpend-inpptr;
40 |     addinp( 0, 0 ); // get it to redo chkinp
41 | 
42 |     return c_out;
43 |   } // func
44 | 
45 | };
46 | 
47 | 


--------------------------------------------------------------------------------
/Lib3/coro_fhp2.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | template < class Model > 
 3 | struct CoroFileProc : Model {
 4 |   using Model::f_quit;
 5 |   using Model::outbeg;
 6 | 
 7 |   enum{ inpbufsize=1<<16, outbufsize=1<<16 };
 8 | 
 9 |   ALIGN(4096) byte inpbuf[inpbufsize];
10 |   ALIGN(4096) byte outbuf[outbufsize];
11 | 
12 |   void processfile( filehandle f, filehandle g, byte* _inpptr=0, byte* _inpend=0 ) {
13 |     uint l,r;
14 | 
15 |     coro_init();
16 |     addout( outbuf, outbufsize );
17 |     if( _inpptr!=0 ) addinp( _inpptr, _inpend-_inpptr );
18 | 
19 |     while(1) {
20 |       r = coro_call(this); 
21 | //printf( "r=%i\n", r );
22 |       if( r==1 ) {
23 |         l = f.sread( inpbuf, inpbufsize );
24 |         if( l==0 ) f_quit=1; // get/put don't support repeated failures
25 |         addinp( inpbuf, l ); 
26 |       } else { // r0=quit, r3=error
27 |         l = getoutsize();
28 |         if( l>0 ) g.writ( (byte*)outbeg, l ); 
29 |         if( r!=2 ) break;
30 |         addout( outbuf, outbufsize );
31 |       } // if
32 |     } // while
33 | 
34 |   } // func
35 | 
36 | };
37 | 
38 | 


--------------------------------------------------------------------------------
/Lib3/file_api_std.inc:
--------------------------------------------------------------------------------
  1 | 
  2 | #if defined(_MSC_VER)
  3 | #define fseeko64 _fseeki64 
  4 | #define ftello64 _ftelli64
  5 | #endif
  6 | 
  7 | struct filehandle0 {
  8 | 
  9 |   FILE* f;
 10 | 
 11 |   operator int( void ) {
 12 |     return ((byte*)f)-((byte*)0);
 13 |   }
 14 | 
 15 |   template< typename CHAR >
 16 |   int open( const CHAR* name ) {
 17 |     if( strcmp(name,"-")==0 ) f = stdin; else
 18 |     f = fopen( name, "rb" );
 19 |     return ((byte*)f)-((byte*)0);
 20 |   }
 21 | 
 22 |   template< typename CHAR >
 23 |   int make( const CHAR* name ) {
 24 |     if( strcmp(name,"-")==0 ) f = stdout; else
 25 |     f = fopen( name, "wb" ); 
 26 |     return ((byte*)f)-((byte*)0);
 27 |   }
 28 | 
 29 |   int close( void ) {
 30 |     return fclose(f);
 31 |   }
 32 | 
 33 |   qword size( void ) { 
 34 |     qword p = tell();
 35 |     seek( 0, SEEK_END );
 36 |     qword s = tell();
 37 |     seek( p, SEEK_SET );
 38 |     return s; 
 39 |   }
 40 | 
 41 |   void seek( qword ofs, uint typ=SEEK_SET ) {
 42 |     fseeko64( f, ofs, typ );
 43 |   }
 44 | 
 45 |   qword tell( void ) {
 46 |     return ftello64(f);
 47 |   }
 48 | 
 49 |   template< typename BUF >
 50 |   uint read( BUF& buf ) {
 51 |     return read( &buf, sizeof(buf) )!=sizeof(buf);
 52 |   }
 53 | 
 54 |   uint read( void* _buf, uint len ) {
 55 |     return fread(_buf,1,len,f);
 56 |   }
 57 | 
 58 |   template< typename BUF >
 59 |   uint sread( BUF& buf ) { return sread( &buf, sizeof(buf) )!=sizeof(buf); }
 60 | 
 61 |   uint sread( void* _buf, uint len ) { 
 62 |     byte* buf = (byte*)_buf;
 63 |     uint l=0, r;
 64 | 
 65 |     do {
 66 |       r = read( buf+l, len-l );
 67 |       l += r;
 68 |     } while( (r>0) && (l<len) );
 69 | 
 70 |     return l;
 71 |   }
 72 | 
 73 |   int Getc( void ) {
 74 |     byte c;
 75 |     uint l = read(c);
 76 |     return l ? -1 : c;
 77 |   }
 78 | 
 79 |   uint writ( void* _buf, uint len ) {
 80 |     return fwrite( _buf, 1,len, f );
 81 |   }
 82 | 
 83 |   template< typename BUF >
 84 |   uint writ( BUF& buf ) {
 85 |     return writ( &buf, sizeof(buf) )!=sizeof(buf);
 86 |   }
 87 | 
 88 | };
 89 | 
 90 | 
 91 | struct filehandle : filehandle0 {
 92 |   filehandle() { f=0; }
 93 | 
 94 |   filehandle( int ) { f=0; }
 95 | 
 96 |   template< typename CHAR >
 97 |   filehandle( const CHAR* name, uint f_wr=0 ) {
 98 |     f_wr ? make(name) : open(name);
 99 |   }
100 | };
101 | 
102 | 
103 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # shar2
 2 | Shar2 archiver
 3 | <code><pre>
 4 | Shar2 version 0 [18.11.2019 22:36]. File archiving utility.
 5 | Copyright (c) 2019 ConeXware, Inc.  All Rights Reserved.
 6 | Written by Eugene D. Shelwien.
 7 | 
 8 | Usage:
 9 | 
10 |   shar a archive path/  -- concatenate files/subdirs at path into "archive"
11 |   shar a - path/        -- output archive data to stdout
12 |   shar a0 archive path/ -- store files uncompressed
13 |   shar a3 archive path/ -- compress files with zstd level 3 (solid)
14 | 
15 | !!! atm only a directory can be added to archive
16 | !!! last name without trailing slash is turned to "."
17 | !!! because wildcard support is not implemented yet
18 | 
19 |   shar x archive base -- extract files/dirs from archive to base\
20 |   shar x - base       -- extract from stdin to base\
21 | </pre></code>
22 | Notes:
23 | 
24 | 1. File/dir attributes, streams, security info, timestamps are not preserved.
25 | Links/junctions are not preserved (added as what they point to).
26 | 
27 | 2. Extracted files or archive file at "a" command are overwritten without questions.
28 | 
29 | 3. shar2 archive format uses signatures to terminate the files and data streams.
30 | Thus it's possible to edit the stored archive file, then still extract the archive.
31 | 


--------------------------------------------------------------------------------
/c.bat:
--------------------------------------------------------------------------------
 1 | @echo off
 2 | 
 3 | del shar.exe
 4 | 
 5 | perl txt2inc.pl src/usage.txt src/usage.inc
 6 | 
 7 | set ICLCFG=icl64.cfg
 8 | 
 9 | set icl=C:\IntelH0721\bin-intel64\icl2a.bat
10 | set icl=C:\IntelI0124\bin-intel64\icl2a.bat 
11 | set icl=C:\IntelJ0070\bin-intel64\icl2a.bat 
12 | set icl=C:\IntelJ0117\bin-intel64\icl2a.bat 
13 | set icl=C:\VC2015\bin\amd64\c2-17.bat /ILib3 /DCOMMON_SKIP_BSF setjmp.lib
14 | set icl=C:\IntelJ2190\bin-intel64\icl2d.bat 
15 | 
16 | call %icl% src/shar.cpp zstd/zstd.cpp
17 | 
18 | del *.exp *.obj
19 | 


--------------------------------------------------------------------------------
/c32.bat:
--------------------------------------------------------------------------------
 1 | @echo off
 2 | 
 3 | del shar32.exe
 4 | 
 5 | set ICLCFG=icl64.cfg
 6 | 
 7 | set icl=C:\IntelB1054\bin\ia32\icl.bat /Qstd=c++0x /Dconstexpr=
 8 | set icl=C:\IntelH0721\bin-ia32\icl2a.bat 
 9 | set icl=C:\IntelI0124\bin-ia32\icl2a.bat 
10 | set icl=C:\IntelJ0070\bin-ia32\icl2a.bat 
11 | set icl=C:\IntelJ0117\bin-ia32\icl2b.bat 
12 | set icl=C:\IntelJ2190\bin-ia32\icl2d.bat 
13 | 
14 | call %icl% src/shar.cpp zstd/zstd.cpp /Feshar32.exe
15 | 
16 | del *.exp *.obj
17 | 


--------------------------------------------------------------------------------
/g:
--------------------------------------------------------------------------------
 1 | 
 2 | rm -v shar64
 3 | g++ -std=gnu++11 -O2 -s -static -fno-exceptions -fno-rtti -fpermissive -fno-stack-protector -fno-stack-check -fno-check-new -I./Lib3 -D_mkdir=mkdir src/shar.cpp zstd/zstd.cpp -o shar64 2>err
 4 | ls -al shar64 err
 5 | 
 6 | rm -v shar32
 7 | g++ -m32 -std=gnu++11 -O2 -s -static -fno-exceptions -fno-rtti -fpermissive -fno-stack-protector -fno-stack-check -fno-check-new -I./Lib3 -D_mkdir=mkdir src/shar.cpp zstd/zstd.cpp -o shar32 2>err32
 8 | ls -al shar32 err32
 9 | 
10 | 


--------------------------------------------------------------------------------
/g.bat:
--------------------------------------------------------------------------------
 1 | @echo off
 2 | 
 3 | del *.exe
 4 | 
 5 | set incs=-DNDEBUG -DSTRICT -DNDEBUG -DWIN32 -I./Lib3 -D_mkdir=mkdir
 6 | rem -march=sse2 -mtune=sse2 -msse2
 7 | 
 8 | set opts=-fstrict-aliasing -fomit-frame-pointer -ffast-math ^
 9 | -fno-exceptions -fno-rtti -fpermissive  ^
10 | -fno-stack-protector -fno-stack-check -fno-check-new -ftree-vectorize
11 | 
12 | 
13 | rem -fprofile-use -fprofile-correction 
14 | 
15 | rem -flto -ffat-lto-objects -Wl,-flto -fuse-linker-plugin -Wl,-O -Wl,--sort-common -Wl,--as-needed -ffunction-sections
16 | 
17 | :set gcc=C:\MinGW710\bin\g++.exe -m32 
18 | set gcc=C:\MinGW810\bin\g++.exe -m32
19 | set gcc=C:\MinGW810x\bin\g++.exe 
20 | set gcc=C:\MinGW820\bin\g++.exe -march=pentium2
21 | set gcc=C:\MinGW820x\bin\g++.exe -march=k8
22 | set path=%gcc%\..\
23 | 
24 | del *.exe *.o
25 | 
26 | %gcc% -std=gnu++11 -O9 -s %incs% %opts% -static src/shar.cpp zstd/zstd.cpp -o shar.exe
27 | 
28 | 


--------------------------------------------------------------------------------
/g32.bat:
--------------------------------------------------------------------------------
 1 | @echo off
 2 | 
 3 | del *.exe
 4 | 
 5 | set incs=-DNDEBUG -DSTRICT -DNDEBUG -DWIN32 -I./Lib3 -D_mkdir=mkdir
 6 | rem -march=sse2 -mtune=sse2 -msse2
 7 | 
 8 | set opts=-fstrict-aliasing -fomit-frame-pointer -ffast-math ^
 9 | -fno-exceptions -fno-rtti -fpermissive  ^
10 | -fno-stack-protector -fno-stack-check -fno-check-new -ftree-vectorize
11 | 
12 | 
13 | rem -fprofile-use -fprofile-correction 
14 | 
15 | rem -flto -ffat-lto-objects -Wl,-flto -fuse-linker-plugin -Wl,-O -Wl,--sort-common -Wl,--as-needed -ffunction-sections
16 | 
17 | :set gcc=C:\MinGW710\bin\g++.exe -m32 
18 | set gcc=C:\MinGW810\bin\g++.exe -m32
19 | set gcc=C:\MinGW810x\bin\g++.exe 
20 | set gcc=C:\MinGW820x\bin\g++.exe -march=k8
21 | set gcc=C:\MinGW820\bin\g++.exe -march=pentium2 -m32
22 | set path=%gcc%\..\
23 | 
24 | del *.exe *.o
25 | 
26 | %gcc% -std=gnu++11 -O9 -s %incs% %opts% -static src/shar.cpp zstd/zstd.cpp -o shar32.exe
27 | 
28 | 


--------------------------------------------------------------------------------
/gc.bat:
--------------------------------------------------------------------------------
 1 | @echo off
 2 | 
 3 | set LIB=C:\VC2015\lib\amd64;C:\VC2015\sdk\Lib\x64;C:\VC2015\ucrt\x64
 4 | 
 5 | set incs=-DSTRICT -DNDEBUG -DWIN32 -D_WIN32 -DBIT64 -I./Lib3 -DCOMMON_SKIP_BSF -DUNICODE -D_UNICODE ^
 6 | -D_CRT_SECURE_NO_WARNINGS ^
 7 | -D_CRT_SECURE_NO_DEPRECATE ^
 8 | -D_CRT_DISABLE_PERFCRIT_LOCKS ^
 9 | -D_CRT_NONSTDC_NO_DEPRECATE ^
10 | -D_SECURE_SCL=0 ^
11 | -D_ITERATOR_DEBUG_LEVEL=0 ^
12 | -D_SECURE_SCL_THROWS=0 ^
13 | -D_HAS_ITERATOR_DEBUGGING=0 ^
14 | -I. -IC:\VC2019\include ^
15 | -IC:\VC2019\ucrt\include ^
16 | -IC:\VC2019\sdk\Include ^
17 | -DUNICODE -D_UNICODE
18 | 
19 | 
20 | rem -fstrict-aliasing 
21 | set opts=-fomit-frame-pointer -fno-stack-protector -fno-stack-check -fgnu-keywords -Wno-invalid-token-paste
22 | 
23 | set arch=-march=k8 -mtune=k8 -fms-compatibility -fms-compatibility-version=19 -fms-extensions -Wno-ignored-attributes -m64
24 | 
25 | set gcc=C:\clang801x\bin\clang++.exe
26 | 
27 | del *.exe *.o
28 | 
29 | %gcc% -v -s %arch% -O9 -std=c++11 %incs% %opts% src/shar.cpp zstd/zstd.cpp C:\VC2019\lib\amd64\oldnames.lib -static -o shar.exe
30 | 
31 | del *.o
32 | 
33 | 
34 | 
35 | 


--------------------------------------------------------------------------------
/icl64.cfg:
--------------------------------------------------------------------------------
  1 | #icnewfeat.lib
  2 | 
  3 | #/FAs
  4 | #/Qopt-report9
  5 | 
  6 | #/Qopt-prefetch:5
  7 | 
  8 | /I.\Lib3
  9 | #/I.\Lib
 10 | 
 11 | /MP8
 12 | /Qoverride-limits
 13 | 
 14 | #/DNOASM
 15 | /DNDEBUG
 16 | #/DCONTEXT=void
 17 | #/DPCONTEXT=void*
 18 | 
 19 | /wd161
 20 | /wd864
 21 | /wd94
 22 | /wd264
 23 | /wd791
 24 | /wd47
 25 | /wd3199
 26 | /wd3920
 27 | 
 28 | /MT
 29 | #/arch:SSE2
 30 | #/arch:ia32
 31 | /QxSSE2
 32 | #/QxHOST
 33 | #/QxSSE4.1
 34 | #/arch:SSE4.1
 35 | #/tune:corei7
 36 | #/QxN
 37 | #/arch:SSSE3
 38 | #/QxSSE3
 39 | 
 40 | #/D_X86_
 41 | 
 42 | #/FAs
 43 | 
 44 | #/Og
 45 | /Ob2ity
 46 | /O3 /Qipo
 47 | /GF /GA /GL
 48 | /Gy 
 49 | /GS- /GR- 
 50 | #/GX-
 51 | /EHs-c-a- 
 52 | #/EHsc /Qsafeseh-
 53 | /Gr
 54 | #/Oa
 55 | /Qvla
 56 | /Qintel-extensions
 57 | /Qopt-mem-layout-trans:3
 58 | /Qopt-dynamic-align
 59 | /Qinline-calloc
 60 | /Qfnalign:16
 61 | /Qcomplex_limited_range
 62 | /Qunroll-aggressive
 63 | /Qalias-const
 64 | /Qopt-multi-version-aggressive
 65 | /Qopt-subscript-in-range
 66 | #/Qopt-ra-region-strategy:routine
 67 | 
 68 | 
 69 | /Qlong_double
 70 | /Qrestrict
 71 | /Qvc14.1
 72 | /Qstd=c++14
 73 | 
 74 | /wd791
 75 | /wd1913
 76 | 
 77 | #/Qvec_report2
 78 | 
 79 | /D_CRT_SECURE_NO_WARNINGS
 80 | /D_CRT_SECURE_NO_DEPRECATE
 81 | /D_CRT_DISABLE_PERFCRIT_LOCKS
 82 | /D_CRT_NONSTDC_NO_DEPRECATE
 83 | /D_SECURE_SCL=0
 84 | /D_ITERATOR_DEBUG_LEVEL=0
 85 | /D_SECURE_SCL_THROWS=0
 86 | /D_HAS_ITERATOR_DEBUGGING=0
 87 | #/D_HAS_EXCEPTIONS=0
 88 | /DVC_EXTRALEAN
 89 | #/DWIN32_LEAN_AND_MEAN
 90 | #/DWIN32_EXTRA_LEAN
 91 | /DSTRICT
 92 | /DNDEBUG
 93 | /DWIN32
 94 | /D_WIN32_WINNT=0x0500
 95 | #/D_WIN64
 96 | #/D_AMD64_
 97 | 
 98 | 
 99 | 
100 | #/Qip
101 | #/Oa
102 | #/Qip
103 | #/Qipo
104 | #/fast
105 | #/Qip-no-inlining
106 | #/Qip-no-pinlining
107 | #/Qopt-prefetch:4
108 | #/Qprof-psa-use
109 | #/Qalign
110 | #/align
111 | #/Qsfalign16
112 | #/Qfnalign
113 | #/Qnobss-init
114 | #/Qprof-src-root-cwd
115 | #/Qssp
116 | 
117 | /Qprof-src-dir-
118 | #/Qprof-dir OPT
119 | #/Qprof-gen
120 | #/Qprof-use
121 | 
122 | /QIfist
123 | #/Qparallel
124 | 
125 | /IC:\IntelH0721\include\icc
126 | #/IC:\IntelI0019\include\icc 
127 | #/IC:\IntelH0048\include 
128 | /IC:\VC2019\include
129 | /IC:\VC2019\ucrt\include
130 | /IC:\VC2019\sdk\Include
131 | 
132 | #/IC:\VC71\include
133 | #/IC:\VC71\INCLUDE1
134 | 
135 | /link /OPT:REF /OPT:ICF=99 /LARGEADDRESSAWARE /SAFESEH:NO /incremental:no /fixed /base:0x400000
136 | 
137 | #
138 | #/link /FIXED:NO
139 | #/link /MERGE:.rdata=.data
140 | #/link /MERGE:.data1=.data
141 | 
142 | #user32.lib
143 | #gdi32.lib
144 | #comctl32.lib
145 | #comdlg32.lib
146 | #advapi32.lib
147 | #shell32.lib
148 | 


--------------------------------------------------------------------------------
/shar64:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Shelwien/shar2/e66ae7eacd2c6073157c5a619512f4265728bbe5/shar64


--------------------------------------------------------------------------------
/src/archive.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | template< template< int f_DEC > class zstd_Codec >
 3 | struct Archive {
 4 | 
 5 |   enum{ pathbufsize=1<<16 };
 6 |   char pathbuf[pathbufsize];
 7 | 
 8 |   struct zstd_Codec_Enc : CoroutinePair< zstd_Codec<0>, Wrapper<0,0x9A> > {
 9 |     uint Init( uint cLevel=0, uint winlog=-1 ) { init(); return this->M1.Init(cLevel,winlog); }
10 |     void Quit( void ) { return this->M1.Quit(); }
11 |   };
12 | 
13 | 
14 |   struct zstd_Codec_Dec : CoroutinePair< Wrapper<1,0x9A>, zstd_Codec<1> > {
15 |     uint Init( uint cLevel=0, uint winlog=-1 ) { init(); return this->M2.Init(cLevel,winlog); }
16 |     void Quit( void ) { return this->M2.Quit(); }
17 |   };
18 | 
19 |   #include "dirstore.inc"
20 |   #include "dirparse.inc"
21 |   #include "dirparse2.inc"
22 | 
23 |   ALIGN(4096) union {
24 |     dirstore A;
25 |     CoroFileProc< CoroutinePair<zstd_Codec_Dec,dirparse> > X;
26 |     CoroFileProc< dirparse2 > X2;
27 |   };
28 | 
29 |   NOINLINE
30 |   uint Shar_Extract( filehandle& f, char* path, uint _r=0 ) {
31 |     uint i,c, r=0, cfg_level=0, cfg_memory=0;
32 | 
33 |     if( _r!=0 ) r=_r; else {
34 |       char chksign[sign_len];
35 |       f.read( chksign, sign_len ); chksign[sign_len-1]=0;
36 |       if( strcmp(chksign,signature)!=0 ) return 3;
37 |       f.read( &r, 1 ); // codec type
38 |     }
39 | 
40 |     if( r!=zstd_Codec<0>::ID ) return r;
41 | 
42 |     // create target path
43 |     strcpy( pathbuf, path );
44 |     for( i=0;; i++ ) {
45 |       c = pathbuf[i];
46 |       if( (i>0) && ((c=='/')||(c=='\\')||(c==0)) ) {
47 |         if( pathbuf[i-1]!=':' ) {
48 |           pathbuf[i]=0;
49 |           _mkdir( pathbuf );
50 |           pathbuf[i]=c;
51 |         }
52 |       }
53 |       if( c==0 ) break;
54 |     }
55 | 
56 |     X.M1.Init( cfg_level, cfg_memory ); // zstd
57 |     X.M2.init( pathbuf );
58 |     X.processfile( f, 0 );
59 |     X.M1.Quit();
60 | 
61 |     // ren/move dumped files to proper subdirs and names
62 |     X2.init( pathbuf );
63 |     X2.processfile( f, 0, X.M1.inpptr, X.M1.inpend );
64 | 
65 |     return 0;
66 |   }
67 | 
68 |   NOINLINE
69 |   uint Shar_Create( filehandle& f, char* path, uint cfg_level, uint cfg_memory ) {
70 |     uint r;
71 |     filehandle& g = f;
72 | 
73 |     A.Z.Init( cfg_level, cfg_memory );
74 | 
75 |     g.writ( (void*)signature, sign_len );
76 | 
77 |     r = zstd_Codec<0>::ID;
78 |     g.writ( &r, 1 );
79 | 
80 |     A.init( path );
81 |     A.processfile( g );
82 |     A.flush_out2( g );
83 | 
84 |     A.Z.Quit();
85 | 
86 |     return 0;
87 |   }
88 | 
89 | };
90 | 


--------------------------------------------------------------------------------
/src/dir_scan.inc:
--------------------------------------------------------------------------------
  1 | 
  2 | #ifdef _WIN32
  3 | 
  4 | #define AreFileApisANSI(x) 0
  5 | #define GetACP(x) CP_UTF8
  6 | #define GetOEMCP(x) CP_UTF8
  7 | #include "../zstd/dirent.h"
  8 | #undef AreFileApisANSI
  9 | #undef GetACP
 10 | #undef GetOEMCP
 11 | 
 12 | #else
 13 | 
 14 | #include <dirent.h>
 15 | #include <sys/stat.h>
 16 | #define _mkdir(X) mkdir(X,S_IRWXU|S_IRWXG|S_IROTH|S_IXOTH)
 17 | 
 18 | #endif
 19 | 
 20 | struct dirscan {
 21 | 
 22 |   typedef DIR* pDIR;
 23 |   typedef dirent* pdirent;
 24 | 
 25 |   enum{ MAX_DEPTH=256, MAX_FNAME=4096 };
 26 | 
 27 |   char pathbuf[MAX_FNAME];
 28 | 
 29 |   pDIR dir0[MAX_DEPTH];
 30 |   uint endpos[MAX_DEPTH];
 31 |   uint parent[MAX_DEPTH];
 32 |   pdirent ent;
 33 | 
 34 |   uint state; // 0=after init, 1=after r1
 35 |   uint cur_depth;
 36 |   uint path_end; // \0 index for path_buf
 37 |   uint f_dir; // 1==dir, 0=file?
 38 |   uint i_file; // file index // number of files
 39 |   uint i_parent; // file's parent index
 40 |   char* name; // current filename;
 41 |   char* path; // original path
 42 | 
 43 |   uint init( char* path ) {
 44 |     uint c,i,j,k;
 45 | 
 46 |     //coro_init();
 47 |     state=0;
 48 | 
 49 |     // copy the whole path, remember last slash pos
 50 |     for( i=0,j=0; (i+1<MAX_FNAME) && ((c=path[i])!=0); i++ ) {
 51 |       if( c=='\\' ) c='/';
 52 |       pathbuf[i]=c;
 53 |       if( c=='/' ) j=i;
 54 | #ifdef _WIN32
 55 |       if( c==':' ) j=i+1;
 56 | #endif
 57 |     }
 58 |     // cut off last name/mask
 59 |     if( j==0 ) {
 60 |       strcpy( pathbuf, "." ); j++;
 61 |     } else {
 62 |       pathbuf[j]=0;
 63 |     }
 64 | 
 65 |     cur_depth = 0;
 66 |     path_end = j;
 67 |     i_file = 1;
 68 |     i_parent = 0;
 69 | 
 70 |     return 0;
 71 |   }
 72 | 
 73 |   uint scan( void ) {
 74 |     uint c,i,j; char* s;
 75 | 
 76 |     if( state==1 ) {
 77 |       i_file++; // file counter
 78 |       if( f_dir && (cur_depth+1<MAX_DEPTH) ) {
 79 |         parent[cur_depth]=i_parent;
 80 |         endpos[cur_depth]=path_end;
 81 |         path_end = strlen(pathbuf);
 82 |         //pathbuf[path_end++]='/';
 83 |         //for( i=0; (path_end+1<MAX_FNAME) && ((c=name[i])!=0); i++ ) pathbuf[path_end++]=c;
 84 |         //pathbuf[path_end]=0;
 85 |         cur_depth++; 
 86 |         i_parent = i_file-1;
 87 |         state=0; //goto Recurse;
 88 |       }
 89 |     }
 90 | 
 91 |     if( state==0 ) {
 92 |       dir0[cur_depth] = opendir(pathbuf);
 93 |       //printf( "!%s!dir=%I64X!\n", pathbuf, dir0[cur_depth] );
 94 |     }
 95 | 
 96 |     while(1) {
 97 |       ent = 0; if( dir0[cur_depth]!=0 ) ent = readdir(dir0[cur_depth]);
 98 |       if( ent==0 ) {
 99 |         if( cur_depth==0 ) break;
100 |         closedir( dir0[cur_depth] ); dir0[cur_depth]=0;
101 |         cur_depth--;
102 |         path_end = endpos[cur_depth];
103 |         i_parent = parent[cur_depth];
104 |         pathbuf[path_end]=0;
105 |         continue;
106 |       }
107 | 
108 |       s = ent->d_name;
109 |       c = (uint&)s[0];
110 |       if( ((c&0xFFFF)=='.') || ((c&0xFFFFFF)=='..') ) continue;
111 | 
112 |       i=path_end;
113 |       pathbuf[i++]='/';
114 |       for( i=i,j=0; (i+1<MAX_FNAME) && ((c=s[j])!=0); j++ ) pathbuf[i++]=c;
115 |       pathbuf[i++]=0;
116 | 
117 | #ifdef _DIRENT_HAVE_D_TYPE
118 |       if( (ent->d_type!=DT_UNKNOWN) && (ent->d_type!=DT_LNK) ) {
119 |          // don't have to stat if we have d_type info, unless it's a symlink (since we stat, not lstat)
120 |         f_dir = (ent->d_type==DT_DIR);
121 |       } else 
122 | #endif
123 |       {
124 |         struct stat stbuf;
125 |         stat(pathbuf, &stbuf); // stat follows symlinks, lstat doesn't.
126 |         f_dir = S_ISDIR(stbuf.st_mode);
127 |       }
128 | 
129 |       //f_dir = (ent->d_type==DT_DIR);
130 |       name = s;
131 | 
132 |       //printf( "!%c!%s/%s!\n", f_dir?'D':'F', pathbuf, s );
133 |       //yield(this,1);
134 |       state=1; return 1;
135 |     } // while
136 | 
137 |     //yield(this,0);
138 |     return 0;
139 |   }
140 | 
141 | };
142 | 


--------------------------------------------------------------------------------
/src/direntry.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | const char signature[] = "Shar2\n\x1A";
 3 | const uint sign_len = sizeof(signature);
 4 | 
 5 | template< class Coroutine >
 6 | uint vlen_put( Coroutine& C, qword x ) {
 7 |   uint len = 0;
 8 |   while(1) {
 9 |     C.put( (x&0x7F) | ((x>0x7F)<<7) ); len++;
10 |     x>>=7;
11 |     if( x==0 ) break;
12 |   }
13 |   return len;
14 | }
15 | 
16 | template< class Coroutine >
17 | qword vlen_get( Coroutine& C ) {  
18 |   uint c,i=0;
19 |   qword x=0;
20 |   while(1) {
21 |     c = byte(C.get());
22 |     x |= qword(c&0x7F)<<i; i+=7;
23 |     if( (c&0x80)==0 ) break;
24 |   }
25 |   return x;
26 | }
27 | 
28 | struct entry {
29 | 
30 |   uint Parent; // directory index
31 |   qword usize; // unpacked size
32 | //  qword csize; // packed size
33 | 
34 |   char type; // entry type: 'F'=file, 'D'=dir
35 | 
36 | //  char method; // compression method: 'c'=copy, 'z'=zstd
37 | 
38 |   char* name;
39 | 
40 |   char namebuf[dirscan::MAX_FNAME];
41 | 
42 |   void init( uint f_dir=0, char* fnam=0, uint _parent=0 ) {
43 |     Parent = _parent;
44 |     usize = 0; 
45 | //    csize = 0;
46 |     type = f_dir?'D':'F'; 
47 | //    method = 'c';
48 |     name = (fnam==0) ? namebuf : fnam;
49 |   }
50 | 
51 |   // return the number of written bytes
52 |   template< class Coroutine >
53 |   uint store( Coroutine& C ) {
54 |     uint i,l=0;
55 |     l += vlen_put( C, Parent );
56 |     l++; C.put( type );
57 | //    C.put( method );
58 | //    if( type!='D' ) l += vlen_put( C, usize );
59 | 
60 |     for( i=0;; i++ ) { l++; C.put( name[i] ); if( name[i]==0 ) break; }
61 |     return l;
62 |   }
63 | 
64 |   template< class Coroutine >
65 |   uint parse( Coroutine& C ) {
66 |     uint i;
67 |     C.chkinp(); if( C.getinplen()==0 ) return 1;
68 |     name = namebuf;
69 |     Parent = vlen_get(C);
70 |     type = C.get();
71 | //    if( type!='D' ) usize=vlen_get(C); else 
72 |     usize=0;
73 | //    method = C.get();
74 |     for( i=0; i<sizeof(namebuf); i++ ) { name[i]=C.get(); if( name[i]==0 ) break; }
75 |     return 0;
76 |   }
77 | 
78 | };
79 | 


--------------------------------------------------------------------------------
/src/dirparse.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | struct dirparse : Coroutine {
 3 | 
 4 |   uint path_end; // \0 index for path_buf
 5 | 
 6 |   char pathbuf[dirscan::MAX_FNAME];
 7 | 
 8 |   CoroFileProcOut< Wrapper<1> > W;
 9 | 
10 |   uint init( char* path ) {
11 |     coro_init();
12 |     strcpy( pathbuf, path );
13 | 
14 |     path_end = strlen(pathbuf);
15 | 
16 |     return 0;
17 |   }
18 | 
19 |   void do_process( void ) {
20 | 
21 |     W.init();
22 | 
23 |     filehandle g;
24 | 
25 |     uint i_file = 0;
26 |     for( i_file=0; /*i_file<n_files*/; /*i_file++*/ ) {
27 | 
28 |       while( (getinplen()==0) ) { chkinp(); if( f_quit ) goto Done; }
29 | 
30 |       pathbuf[path_end] = '/';
31 |       sprintf( &pathbuf[1+path_end], "temp.%08X", i_file++ );
32 | 
33 |       //printf( "!%c!%i!%s!%i!\n", E.type, E.Parent, pathbuf, uint(E.usize) );
34 | 
35 |       g.make( pathbuf ); if( g ) {
36 |         W.processfile( g, this, /*pin*/0 );
37 |         g.close();
38 |       } // if open
39 | 
40 |     } // for
41 | 
42 | Done:
43 | 
44 |     yield(this,0);
45 |   }
46 | 
47 | };
48 | 
49 | 


--------------------------------------------------------------------------------
/src/dirparse2.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | struct dirparse2 : Coroutine {
 3 | 
 4 |   entry E;
 5 | 
 6 |   uint cur_depth;
 7 |   uint path_end; // \0 index for path_buf
 8 |   uint path2_end; // \0 index for path_buf2
 9 |   uint endpos[dirscan::MAX_DEPTH];
10 |   uint parent[dirscan::MAX_DEPTH];
11 | 
12 |   char pathbuf[dirscan::MAX_FNAME];
13 |   char path2buf[dirscan::MAX_FNAME];
14 | 
15 |   uint init( char* path ) {
16 |     coro_init();
17 |     strcpy( pathbuf, path );
18 |     strcpy( path2buf, path );
19 | 
20 |     path2_end = path_end = strlen(pathbuf);
21 |     cur_depth = 0;
22 |     parent[0] = 0;
23 |     endpos[0] = path_end;
24 | 
25 |     return 0;
26 |   }
27 | 
28 |   void do_process( void ) {
29 | 
30 |     uint r, i_file = 0;
31 |     path2buf[path2_end] = '/';
32 | 
33 |     for( i_file=0; /*i_file<n_files*/; /*i_file++*/ ) {
34 | 
35 |      if( E.parse(*this) ) break;
36 | 
37 |      while( (cur_depth>0) && (E.Parent<parent[cur_depth-1]) ) {
38 |        cur_depth--;
39 |        path_end = endpos[cur_depth];
40 |        pathbuf[path_end] = 0;
41 |      }
42 | 
43 |      pathbuf[path_end] = '/';
44 |      strcpy( &pathbuf[1+path_end], E.name );
45 | 
46 |      if( E.type=='D' ) {
47 |        fprintf(stderr, "!%c!%i!%s!\n", E.type, E.Parent, pathbuf );
48 |        _mkdir( pathbuf );
49 |      } else {
50 |        sprintf( &path2buf[1+path2_end], "temp.%08X", i_file++ );
51 | 
52 |        r = rename( path2buf, pathbuf );
53 |        if( r!=0 ) {
54 |          remove( pathbuf );
55 |          r = rename( path2buf, pathbuf );
56 |        }
57 | 
58 | //fprintf(stderr, "!err=%i!\n", r );
59 |        fprintf(stderr, "!%c!%i!%s!%s!\n", E.type, E.Parent, pathbuf, path2buf );
60 |      }
61 | 
62 |      if( E.type=='D' ) {
63 |        endpos[cur_depth] = path_end;
64 |        parent[cur_depth] = 1+i_file;
65 |        path_end = strlen(pathbuf);
66 |        cur_depth++;
67 |      } else {
68 |        pathbuf[path_end] = 0;
69 |      }
70 | 
71 |     } // for
72 | 
73 |     yield(this,0);
74 |   }
75 | 
76 | };
77 | 
78 | 


--------------------------------------------------------------------------------
/src/dirstore.inc:
--------------------------------------------------------------------------------
  1 | 
  2 | struct dirstore : Coroutine {
  3 |   #define xOut pin[2]
  4 | 
  5 |   qword n_bytes;
  6 |   memblk* h_blk;
  7 | 
  8 |   entry E;
  9 | 
 10 |   dirscan D;
 11 | 
 12 |   zstd_Codec_Enc Z;
 13 | 
 14 |   enum{ inpbufsize=1<<16, outbufsize=1<<16,  bufsize=1<<16 };
 15 |   ALIGN(4096) memblk out2buf;
 16 |   ALIGN(4096) byte inpbuf[inpbufsize];
 17 |   ALIGN(4096) byte outbuf[outbufsize];
 18 |   ALIGN(4096) byte Zoutbuf[outbufsize];
 19 |   ALIGN(4096) byte buf[bufsize];
 20 | 
 21 |   CoroFileProcInp< Wrapper<0> > W;
 22 | 
 23 |   uint init( char* path ) {
 24 |     n_bytes = 0;
 25 |     coro_init();
 26 |     uint r = D.init(path);
 27 |     return r;
 28 |   }
 29 | 
 30 |   void do_process( void ) {
 31 |     uint i,l,n;
 32 |     filehandle f;
 33 | 
 34 |     while( D.scan() ) {
 35 | 
 36 |       E.init( D.f_dir, D.name, D.i_parent );
 37 | 
 38 |       if( E.type!='D' ) {
 39 |         // copy a file to archive
 40 |         f.open( D.pathbuf ); if( f ) {
 41 |           E.usize += W.processfile( f, this );
 42 |           f.close();
 43 |         } // if open
 44 |       } // if file
 45 | 
 46 |       fprintf(stderr, "!%c!%i!%s!%i!\n", E.type, D.i_parent, D.pathbuf, uint(E.usize) );
 47 | 
 48 |       n_bytes += E.store(xOut);
 49 |     }
 50 | 
 51 |     --D.i_file; // started from 1
 52 |     fprintf(stderr, "N_files=%i\n", D.i_file );
 53 |     fprintf(stderr, "n_bytes=%i\n", uint(n_bytes) );
 54 | 
 55 | //    l = vlen_put( xOut, D.i_file );
 56 | //    l+= vlen_put( xOut, n_bytes );
 57 | //    xOut.put( l );
 58 | 
 59 |     yield(this,0);
 60 |   }
 61 | 
 62 |   void flush_out2( filehandle& h ) {
 63 |     h_blk = &out2buf;
 64 |     while( h_blk!=0 ) {
 65 |       memblk& p = *h_blk;
 66 |       h.writ( p.data, p.len );
 67 |       h_blk = p.next;
 68 |     }
 69 |   }
 70 | 
 71 |   void new_memblk( memblk* p ) {
 72 |     if( h_blk!=0 ) h_blk->next = p; 
 73 |     h_blk = p;
 74 |     xOut.addout( h_blk->data, memblklen ); 
 75 |     h_blk->len=0;
 76 |     h_blk->next=0;
 77 |   }
 78 | 
 79 |   void processfile( filehandle g ) {
 80 |     uint l=0,r,rz;
 81 |     coro_init();
 82 |     addout( outbuf, outbufsize );
 83 |     Z.addout( Zoutbuf, outbufsize );
 84 |     h_blk=0; new_memblk( &out2buf );
 85 | 
 86 |     while( 1 ) {
 87 |       r = coro_call(this); 
 88 | 
 89 |       if( r==1 ) break; // shouldn't happen
 90 | 
 91 |       if( (r==2) || (r==0) ) {
 92 | 
 93 |         l = getoutsize();
 94 | ZRetry:
 95 |         Z.addinp( (byte*)outbeg, l ); if( r==0 ) Z.f_quit=1;
 96 | 
 97 |         while(1) {
 98 |           rz = Z.coro_call(&Z);
 99 |           if( rz==1 ) break;
100 |           if( g ) g.writ( Zoutbuf, Z.getoutsize() );
101 |           Z.addout( Zoutbuf, outbufsize );
102 |           if( rz==0 ) break;
103 |         }
104 |         if( (r==0) && (rz==1) ) { l=0; Z.f_quit=1; goto ZRetry; }
105 | 
106 |         addout( outbuf, outbufsize );
107 |       }
108 |   
109 |       if( (r==3) || (r==0) ) {
110 |         h_blk->len = xOut.getoutsize();
111 |         if( r==3 ) {
112 |           memblk* p = new memblk; if( p==0 ) break; // memory error
113 |           new_memblk( p );
114 |         }
115 |       }
116 | 
117 |       if( r==0 ) break;
118 |     }
119 | 
120 |   } // processfile
121 | 
122 |   #undef xOut
123 | };
124 | 
125 | 


--------------------------------------------------------------------------------
/src/main_utf8_wrap.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | // from paq8px_v183.cpp
 3 | 
 4 | //#include <windows.h>
 5 | #ifdef _WIN32
 6 | #include <io.h>
 7 | #include <fcntl.h>
 8 | #endif
 9 | 
10 | int main( int argc, char** argv ) {
11 | 
12 | #ifdef _WIN32
13 |   // On Windows, argv is encoded in the effective codepage, therefore unsuitable 
14 |   // for acquiring command line arguments (file names
15 |   // in our case) not representable in that particular codepage.
16 |   // -> We will recreate argv as UTF8 (like in Linux)
17 |   uint i, /*oldcp = GetConsoleOutputCP(),*/ buffersize, retval;
18 | //  SetConsoleOutputCP(CP_UTF8);
19 |   int argc_utf8 = 0;
20 |   char** argv_utf8 = 0 ;
21 |   #pragma comment(lib,"shell32.lib")
22 |   wchar_t** szArglist = CommandLineToArgvW( GetCommandLineW(), &argc_utf8 );
23 | 
24 |   if( szArglist==0 ) {
25 |     fprintf(stderr, "CommandLineToArgvW failed\n" ); return 1;
26 |   } else {
27 |     if( argc!=argc_utf8 ) { fprintf(stderr, "Error preparing command line arguments." ); return 1; }
28 |     argv_utf8 = new char*[argc_utf8+1];
29 |     for( i=0; i<argc_utf8; i++ ) { 
30 |       wchar_t* s = szArglist[i];
31 |       buffersize = WideCharToMultiByte(CP_UTF8,0,s,-1,NULL,0,NULL,NULL);
32 |       argv_utf8[i] = new char[buffersize];
33 |       WideCharToMultiByte(CP_UTF8,0,s,-1,argv_utf8[i],buffersize,NULL,NULL);
34 |       //printf("%d: %s\n", i, argv_utf8[i]); //debug: see if conversion is successful
35 |     }
36 |     argv_utf8[argc_utf8]=0;
37 | 
38 |     setlocale(LC_ALL, ".UTF8");
39 |     setmode( fileno(stdin), O_BINARY );
40 |     setmode( fileno(stdout), O_BINARY );
41 | 
42 |     retval = main_utf8(argc_utf8, argv_utf8);
43 |     //for( i=0; i<argc_utf8; i++ ) delete[] argv_utf8[i]; 
44 |     //delete[] argv_utf8;
45 |     //SetConsoleOutputCP(oldcp);
46 |     return retval;
47 |   }
48 | 
49 | #else
50 | 
51 |   return main_utf8(argc, argv);
52 | 
53 | #endif
54 | 
55 | }
56 | 


--------------------------------------------------------------------------------
/src/memblk.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | struct memblk;
 3 | 
 4 | enum{ 
 5 |   memblksize=1<<13,
 6 |   memblklen = memblksize - sizeof(memblk*) - sizeof(uint)
 7 | };
 8 | 
 9 | struct memblk {
10 |   memblk* next;
11 |   uint len;
12 |   byte data[memblklen];
13 | };
14 | 
15 | 


--------------------------------------------------------------------------------
/src/setjmp.lib:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Shelwien/shar2/e66ae7eacd2c6073157c5a619512f4265728bbe5/src/setjmp.lib


--------------------------------------------------------------------------------
/src/shar.cpp:
--------------------------------------------------------------------------------
  1 | 
  2 | #ifdef _WIN32
  3 | #include <windows.h>
  4 | #include <locale.h>
  5 | #endif
  6 | 
  7 | #include "common.inc"
  8 | 
  9 | #ifdef _WIN32
 10 | #include <io.h>
 11 | #include <direct.h>
 12 | //#include "file_api_win.inc"
 13 | #include "file_api_std.inc"
 14 | #else
 15 | #include "file_api_std.inc"
 16 | #endif
 17 | 
 18 | #include "coro3b.inc"
 19 | 
 20 | #include "stream_wrapper.inc"
 21 | #include "coro2pair.inc"
 22 | 
 23 | #include "coro_fhp2.inc"
 24 | #include "coro_fh_inp.inc"
 25 | #include "coro_fh_out.inc"
 26 | 
 27 | #include "memblk.inc"
 28 | 
 29 | #include "dir_scan.inc"
 30 | #include "direntry.inc"
 31 | 
 32 | #define FILE BUGG // have to use file_api
 33 | 
 34 | #include "archive.inc"
 35 | 
 36 | #include "../zstd/zstd.inc"
 37 | 
 38 | template< int f_DEC >
 39 | struct copy_Codec : Coroutine {
 40 | 
 41 |   enum{ ID='C' };
 42 | 
 43 |   uint Init( uint cLevel=0, uint winlog=-1 ) {
 44 |     coro_init(); 
 45 |     return 0;
 46 |   }
 47 | 
 48 |   void Quit( void ) {
 49 |   }
 50 | 
 51 |   void do_process( void ) {
 52 |     uint c;
 53 | 
 54 |     while(1) {
 55 |       c = get(); if( c==-1 ) break; 
 56 |       put( c );
 57 |     }
 58 | 
 59 |     yield(this,0);
 60 |   }
 61 | 
 62 | };
 63 | 
 64 | ALIGN(4096) 
 65 | static union {
 66 |   Archive< zstd_Codec > A_zstd;
 67 |   Archive< copy_Codec > A_copy;
 68 | };
 69 | 
 70 | #include "usage.inc"
 71 | 
 72 | // shar a|x archive path
 73 | 
 74 | int main_utf8( int argc, char** argv ) {
 75 | 
 76 |   uint cfg_level = 3;
 77 |   uint cfg_memory = 23; // 1<<23
 78 | 
 79 |   if( argc<3 ) {
 80 |     printf( msg_logo );
 81 |     printf( msg_usage );
 82 |     return 1;
 83 |   }
 84 | 
 85 |   uint r, f_UNP = (argv[1][0]=='x');
 86 |   char* path = (char*)( argc<4 ? f_UNP? "" : "./" : argv[3] );
 87 | 
 88 |   if( strlen(argv[1])>=2 ) cfg_level=atoi(&argv[1][1]);
 89 | 
 90 |   if( f_UNP ) {
 91 |     filehandle f( argv[2] ); if( f==0 ) return 2; // for actual data
 92 | 
 93 |     r = A_zstd.Shar_Extract( f, path, 0 );
 94 | 
 95 |     if( r==3 ) {
 96 |       fprintf(stderr,"Signature mismatch!\n" );
 97 |       return 3;
 98 |     }
 99 | 
100 |     switch( r ) {
101 |     case 0: break; // done
102 |     case zstd_Codec<0>::ID: 
103 |       r = A_zstd.Shar_Extract( f, path, r );
104 |       break;
105 |     case copy_Codec<0>::ID: 
106 |       r = A_copy.Shar_Extract( f, path, r );
107 |       break;
108 |     default:
109 |       fprintf(stderr,"Unknown compression method!\n" );
110 |       return 3;
111 |     };
112 | 
113 |   } else {
114 | 
115 |     filehandle g( argv[2], 1 ); if( g==0 ) return 2;
116 |   
117 |     if( cfg_level==0 ) {
118 |       r=A_copy.Shar_Create( g, path, 0, cfg_memory );
119 |     } else {
120 |       r=A_zstd.Shar_Create( g, path, cfg_level, cfg_memory );
121 |     }
122 |   }
123 | 
124 |   return 0;
125 | }
126 | 
127 | #include "main_utf8_wrap.inc"
128 | 
129 | 


--------------------------------------------------------------------------------
/src/stream_unwrap.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | /*
 3 | 
 4 | 0 = empty cache, waiting for input
 5 | 1 = symbol cached, waiting for output request
 6 | 
 7 | 2 = CHR0 detected
 8 | 3 = CHR0,CHR1 detected
 9 | 4 = CHR0,CHR1,CHR2 detected
10 | 
11 | 13 = CHR2..CHR0 to output
12 | 12 = CHR1..CHR0 to output
13 | 11 = CHR0 to output
14 | 10 = all clear, return to 0
15 | 
16 | */
17 | 
18 | template< class wrapper_enum >
19 | struct stream_unwrapper : wrapper_enum {
20 | using wrapper_enum::EOF;
21 | using wrapper_enum::CHR0;
22 | using wrapper_enum::CHR1;
23 | using wrapper_enum::CHR2;
24 | using wrapper_enum::CHR3;
25 | using wrapper_enum::CHR4;
26 | using wrapper_enum::CHR0123;
27 | using wrapper_enum::CHR3210;
28 | 
29 |   uint state;
30 |   uint count;
31 |   uint cache;
32 |   uint cache2;
33 | 
34 |   void init( void ) { 
35 |     state=0; count=0;
36 |     cache=0; cache2=0;
37 |   }
38 | 
39 |   uint need_inp( void ) {
40 |     return (state!=1) && (state<10);
41 |   }
42 | 
43 |   void pass_inp( uint c ) {
44 | 
45 |     cache = c; 
46 | 
47 |     switch( state ) {
48 |     case 0: if( c==CHR0 ) state=2-1; break;
49 |     case 2: if( c==CHR1 ) state=3-1; else count=2,state=9,cache2=c; break;
50 |     case 3: if( c==CHR2 ) state=4-1; else count=3,state=9,cache2=c; break;
51 |     case 4: 
52 |       if( c==CHR3 ) cache=EOF, state=1-1;  else
53 |       if( c==CHR4 ) cache2=CHR2, count=3,state=9;  else  
54 |                     cache2=c,    count=4,state=9;
55 |       break;
56 |     }
57 | 
58 |     state++;
59 | 
60 |   }
61 | 
62 |   uint read_out( void ) {
63 |     uint r = cache; state--;
64 | 
65 |     if( state>=9 ) {
66 |       r = byte( CHR3210 >> ((state-9)*8) );
67 |       state+=2; count--;
68 |       if( count==0 ) r=cache2, state=0; else
69 |       if( (count==1) && (cache2==CHR0) ) state=2;
70 |     }
71 | 
72 |     return r;
73 |   }
74 | 
75 | };
76 | 


--------------------------------------------------------------------------------
/src/stream_wrap.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | /*
 3 | 
 4 | 0 = empty cache, waiting for input
 5 | 1 = symbol cached, waiting for output request
 6 | 
 7 | 7 = CHR0 ready for output (after EOF)
 8 | 6 = CHR1 ready for output 
 9 | 5 = CHR2 ready for output
10 | 4 = CHR3 ready for output
11 | 3 = EOF reached
12 | 
13 | */
14 | 
15 | template< class wrapper_enum >
16 | struct stream_wrapper : wrapper_enum {
17 | using wrapper_enum::EOF;
18 | using wrapper_enum::CHR0;
19 | using wrapper_enum::CHR1;
20 | using wrapper_enum::CHR2;
21 | using wrapper_enum::CHR3;
22 | using wrapper_enum::CHR4;
23 | using wrapper_enum::CHR0123;
24 | using wrapper_enum::CHR3210;
25 | 
26 |   uint state;
27 |   uint cache;
28 | 
29 |   void init( void ) { 
30 |     state=0; 
31 |     cache=0;
32 |   }
33 | 
34 |   uint need_inp( void ) {
35 |     return state==0;
36 |   }
37 | 
38 |   void pass_inp( uint c ) {
39 |     if( c!=EOF ) {
40 |       cache = (cache<<8) | c;
41 |       state = 1;
42 |       if( (cache|1)==CHR0123 ) {
43 |         cache = (cache<<8) | CHR4;
44 |         state = 2;
45 |       }
46 |     } else {
47 |       cache = CHR3210;
48 |       state = 7;
49 |     }
50 |   }
51 | 
52 |   uint read_out( void ) {
53 |     uint r = (state!=3) ? byte(cache) : EOF;
54 |     state--; if( state>0 ) cache>>=8;
55 |     return r;
56 |   }
57 | 
58 | };
59 | 


--------------------------------------------------------------------------------
/src/stream_wrapper.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | #undef EOF
 3 | 
 4 | template< int _CHR0=0x99, int _CHR1=0x19, int _CHR2=0xD3, int _CHR3=0xFF, int _CHR4=0xFE >
 5 | struct wrapper_enum {
 6 |   enum { EOF=0x100 };
 7 | 
 8 |   //99 19 D3
 9 |   enum { 
10 |     CHR0=_CHR0, CHR1=_CHR1, CHR2=_CHR2, CHR3=_CHR3, CHR4=_CHR4,
11 |     CHR0123=CHR0*0x1000000U+CHR1*0x10000+CHR2*0x100+CHR3, 
12 |     CHR3210=CHR3*0x1000000U+CHR2*0x10000+CHR1*0x100+CHR0
13 |   };
14 | };
15 | 
16 | #include "stream_wrap.inc"
17 | #include "stream_unwrap.inc"
18 | 
19 | template< int MODE, int _CHR0=0x99 > struct wrapper_base;
20 | template< int _CHR0 > struct wrapper_base<0,_CHR0> : stream_wrapper< wrapper_enum<_CHR0> > {};
21 | template< int _CHR0 > struct wrapper_base<1,_CHR0> : stream_unwrapper< wrapper_enum<_CHR0> > {};
22 | 
23 | template <int ProcMode, int _CHR0=0x99>
24 | struct Wrapper : Coroutine {
25 | 
26 |   enum{ DECODE=ProcMode };
27 | 
28 |   wrapper_base<DECODE,_CHR0> W;
29 | 
30 |   qword c_out; // number of input bytes
31 | 
32 |   void do_process( void ) {
33 | 
34 |     uint c;
35 |    
36 |     c_out = 0; 
37 | 
38 |     W.init();
39 | 
40 |     while(1) {
41 | 
42 |       while( W.need_inp() ) {
43 |         c = get(); c_out+=1-f_quit;
44 |         W.pass_inp( f_quit ? W.EOF : c );
45 |       }
46 | 
47 |       c = W.read_out(); if( c==W.EOF ) break;
48 | 
49 |       put( c );
50 | 
51 |     }
52 | 
53 | //printf( "c_out=%i f_quit=%i\n", uint(c_out), f_quit );
54 | 
55 |     yield(this,0);
56 |   }
57 | 
58 | };
59 | 
60 | 


--------------------------------------------------------------------------------
/src/usage.inc:
--------------------------------------------------------------------------------
 1 | 
 2 | char msg_logo[] = 
 3 | "Shar2 version 0 [21.11.2019 02:11]. File archiving utility.\n"
 4 | "Copyright (c) 2019 ConeXware, Inc.  All Rights Reserved.\n"
 5 | "Written by Eugene D. Shelwien.\n"
 6 | "\n"
 7 | ;
 8 | 
 9 | char msg_usage[] = 
10 | "Usage:\n"
11 | "\n"
12 | "  shar a archive path/  -- concatenate files/subdirs at path into \"archive\"\n"
13 | "  shar a - path/        -- output archive data to stdout\n"
14 | "  shar a0 archive path/ -- store files uncompressed\n"
15 | "  shar a3 archive path/ -- compress files with zstd level 3 (solid)\n"
16 | "\n"
17 | "!!! atm only a directory can be added to archive\n"
18 | "!!! last name without trailing slash is turned to \".\"\n"
19 | "!!! because wildcard support is not implemented yet\n"
20 | "\n"
21 | "  shar x archive base -- extract files/dirs from archive to base\\\n"
22 | "  shar x - base       -- extract from stdin to base\\\n"
23 | "\n"
24 | "Notes:\n"
25 | "\n"
26 | "1. File/dir attributes, streams, security info, timestamps are not preserved.\n"
27 | "Links/junctions are not preserved (added as what they point to).\n"
28 | "\n"
29 | "2. Extracted files or archive file at \"a\" command are overwritten without questions.\n"
30 | "\n"
31 | "3. shar2 archive format uses signatures to terminate the files and data streams.\n"
32 | "Thus it's possible to edit the stored archive file, then still extract the archive.\n"
33 | ;
34 | 


--------------------------------------------------------------------------------
/src/usage.txt:
--------------------------------------------------------------------------------
 1 | ;
 2 | [[msg_logo]]
 3 | Shar2 version 0 [$DATE]. File archiving utility.
 4 | Copyright (c) 2019 ConeXware, Inc.  All Rights Reserved.
 5 | Written by Eugene D. Shelwien.
 6 | 
 7 | ;
 8 | [[msg_usage]]
 9 | Usage:
10 | 
11 |   shar a archive path/  -- concatenate files/subdirs at path into "archive"
12 |   shar a - path/        -- output archive data to stdout
13 |   shar a0 archive path/ -- store files uncompressed
14 |   shar a3 archive path/ -- compress files with zstd level 3 (solid)
15 | 
16 | !!! atm only a directory can be added to archive
17 | !!! last name without trailing slash is turned to "."
18 | !!! because wildcard support is not implemented yet
19 | 
20 |   shar x archive base -- extract files/dirs from archive to base\
21 |   shar x - base       -- extract from stdin to base\
22 | 
23 | Notes:
24 | 
25 | 1. File/dir attributes, streams, security info, timestamps are not preserved.
26 | Links/junctions are not preserved (added as what they point to).
27 | 
28 | 2. Extracted files or archive file at "a" command are overwritten without questions.
29 | 
30 | 3. shar2 archive format uses signatures to terminate the files and data streams.
31 | Thus it's possible to edit the stored archive file, then still extract the archive.
32 | 


--------------------------------------------------------------------------------
/t.bat:
--------------------------------------------------------------------------------
 1 | @echo off
 2 | 
 3 | rd /S /Q tmp
 4 | mkdir tmp
 5 | del sharive sharive2 >nul 2>&1
 6 | 
 7 | shar.exe a99 sharive ..\Lib\
 8 | 
 9 | cat sharive | shar.exe x - tmp
10 | 
11 | shar.exe a99 - tmp\ | cat >sharive2
12 | 
13 | for %%a in (sharive,sharive2) do echo %%~za - %%a
14 | md5sum sharive sharive2
15 | 
16 | rd /S /Q tmp
17 | del sharive sharive2 >nul 2>&1
18 | 
19 | 


--------------------------------------------------------------------------------
/txt2inc.pl:
--------------------------------------------------------------------------------
 1 | 
 2 | sub getdate {
 3 |   my ($s,$m,$h,$d,$mn,$y) = localtime(time);
 4 |   return sprintf "%02i.%02i.%04i %02i:%02i", $d,$mn+1,$y+1900, $h,$m;
 5 | }
 6 | 
 7 | $curdate = getdate();
 8 | 
 9 | open( I, "<$ARGV[0]" ) || die;
10 | open( O, ">$ARGV[1]" ) || die;
11 | 
12 | $flag=0;
13 | print O "\n";
14 | while( <I> ) {
15 |   if( /\[\[(.*)\]\]/ ) {
16 |     print O ";\n\n" if $flag;
17 |     print O "char $1\[\] = \n"; $flag=1;
18 |   } else {
19 |     s/[\r\n]//g;
20 |     s/\s+$//;
21 |     if( /^;$/ ) {
22 |     } else {
23 |       s/\\/\\\\/g;
24 |       s/"/\\"/g;
25 |       s/%/%%/g;
26 |       s/\$DATA/%s/g;
27 |       s/\$DATE/$curdate/g;
28 |       print O "\"$_\\n\"\n";
29 |     }
30 |   }
31 | }
32 | print O ";\n" if $flag;
33 | 


--------------------------------------------------------------------------------
/vc.bat:
--------------------------------------------------------------------------------
 1 | @echo off
 2 | 
 3 | del shar.exe
 4 | 
 5 | set icl=C:\VC2015\bin\amd64\c2-17.bat /ILib3 /DCOMMON_SKIP_BSF src/setjmp.lib
 6 | 
 7 | call %icl% src/shar.cpp zstd/zstd.cpp
 8 | 
 9 | del *.exp *.obj
10 | 


--------------------------------------------------------------------------------
/zstd/bitstream.h:
--------------------------------------------------------------------------------
  1 | /* ******************************************************************
  2 |    bitstream
  3 |    Part of FSE library
  4 |    header file (to include)
  5 |    Copyright (C) 2013-2016, Yann Collet.
  6 | 
  7 |    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
  8 | 
  9 |    Redistribution and use in source and binary forms, with or without
 10 |    modification, are permitted provided that the following conditions are
 11 |    met:
 12 | 
 13 |        * Redistributions of source code must retain the above copyright
 14 |    notice, this list of conditions and the following disclaimer.
 15 |        * Redistributions in binary form must reproduce the above
 16 |    copyright notice, this list of conditions and the following disclaimer
 17 |    in the documentation and/or other materials provided with the
 18 |    distribution.
 19 | 
 20 |    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 21 |    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 22 |    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 23 |    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 24 |    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 25 |    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 26 |    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 27 |    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 28 |    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 29 |    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 30 |    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 31 | 
 32 |    You can contact the author at :
 33 |    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
 34 | ****************************************************************** */
 35 | #ifndef BITSTREAM_H_MODULE
 36 | #define BITSTREAM_H_MODULE
 37 | 
 38 | #if defined (__cplusplus)
 39 | extern "C" {
 40 | #endif
 41 | 
 42 | 
 43 | /*
 44 | *  This API consists of small unitary functions, which must be inlined for best performance.
 45 | *  Since link-time-optimization is not available for all compilers,
 46 | *  these functions are defined into a .h to be included.
 47 | */
 48 | 
 49 | /*-****************************************
 50 | *  Dependencies
 51 | ******************************************/
 52 | #include "mem.h"            /* unaligned access routines */
 53 | #include "error_private.h"  /* error codes and messages */
 54 | 
 55 | 
 56 | /*=========================================
 57 | *  Target specific
 58 | =========================================*/
 59 | #if defined(__BMI__) && defined(__GNUC__)
 60 | #  include <immintrin.h>   /* support for bextr (experimental) */
 61 | #endif
 62 | 
 63 | 
 64 | /*-******************************************
 65 | *  bitStream encoding API (write forward)
 66 | ********************************************/
 67 | /* bitStream can mix input from multiple sources.
 68 | *  A critical property of these streams is that they encode and decode in **reverse** direction.
 69 | *  So the first bit sequence you add will be the last to be read, like a LIFO stack.
 70 | */
 71 | typedef struct
 72 | {
 73 |     size_t bitContainer;
 74 |     int    bitPos;
 75 |     char*  startPtr;
 76 |     char*  ptr;
 77 |     char*  endPtr;
 78 | } BIT_CStream_t;
 79 | 
 80 | MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
 81 | MEM_STATIC void   BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
 82 | MEM_STATIC void   BIT_flushBits(BIT_CStream_t* bitC);
 83 | MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
 84 | 
 85 | /* Start with initCStream, providing the size of buffer to write into.
 86 | *  bitStream will never write outside of this buffer.
 87 | *  `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
 88 | *
 89 | *  bits are first added to a local register.
 90 | *  Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
 91 | *  Writing data into memory is an explicit operation, performed by the flushBits function.
 92 | *  Hence keep track how many bits are potentially stored into local register to avoid register overflow.
 93 | *  After a flushBits, a maximum of 7 bits might still be stored into local register.
 94 | *
 95 | *  Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
 96 | *
 97 | *  Last operation is to close the bitStream.
 98 | *  The function returns the final size of CStream in bytes.
 99 | *  If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
100 | */
101 | 
102 | 
103 | /*-********************************************
104 | *  bitStream decoding API (read backward)
105 | **********************************************/
106 | typedef struct
107 | {
108 |     size_t   bitContainer;
109 |     unsigned bitsConsumed;
110 |     const char* ptr;
111 |     const char* start;
112 | } BIT_DStream_t;
113 | 
114 | typedef enum { BIT_DStream_unfinished = 0,
115 |                BIT_DStream_endOfBuffer = 1,
116 |                BIT_DStream_completed = 2,
117 |                BIT_DStream_overflow = 3 } BIT_DStream_status;  /* result of BIT_reloadDStream() */
118 |                /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
119 | 
120 | MEM_STATIC size_t   BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
121 | MEM_STATIC size_t   BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
122 | MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
123 | MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
124 | 
125 | 
126 | /* Start by invoking BIT_initDStream().
127 | *  A chunk of the bitStream is then stored into a local register.
128 | *  Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
129 | *  You can then retrieve bitFields stored into the local register, **in reverse order**.
130 | *  Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
131 | *  A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
132 | *  Otherwise, it can be less than that, so proceed accordingly.
133 | *  Checking if DStream has reached its end can be performed with BIT_endOfDStream().
134 | */
135 | 
136 | 
137 | /*-****************************************
138 | *  unsafe API
139 | ******************************************/
140 | MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
141 | /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
142 | 
143 | MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
144 | /* unsafe version; does not check buffer overflow */
145 | 
146 | MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
147 | /* faster, but works only if nbBits >= 1 */
148 | 
149 | 
150 | 
151 | /*-**************************************************************
152 | *  Internal functions
153 | ****************************************************************/
154 | MEM_STATIC unsigned BIT_highbit32 (register U32 val)
155 | {
156 | #   if defined(_MSC_VER)   /* Visual */
157 |     unsigned long r=0;
158 |     _BitScanReverse ( &r, val );
159 |     return (unsigned) r;
160 | #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* Use GCC Intrinsic */
161 |     return 31 - __builtin_clz (val);
162 | #   else   /* Software version */
163 |     static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
164 |     U32 v = val;
165 |     v |= v >> 1;
166 |     v |= v >> 2;
167 |     v |= v >> 4;
168 |     v |= v >> 8;
169 |     v |= v >> 16;
170 |     return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
171 | #   endif
172 | }
173 | 
174 | /*=====    Local Constants   =====*/
175 | static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,  0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF };   /* up to 26 bits */
176 | 
177 | 
178 | /*-**************************************************************
179 | *  bitStream encoding
180 | ****************************************************************/
181 | /*! BIT_initCStream() :
182 |  *  `dstCapacity` must be > sizeof(void*)
183 |  *  @return : 0 if success,
184 |               otherwise an error code (can be tested using ERR_isError() ) */
185 | MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity)
186 | {
187 |     bitC->bitContainer = 0;
188 |     bitC->bitPos = 0;
189 |     bitC->startPtr = (char*)startPtr;
190 |     bitC->ptr = bitC->startPtr;
191 |     bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr);
192 |     if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall);
193 |     return 0;
194 | }
195 | 
196 | /*! BIT_addBits() :
197 |     can add up to 26 bits into `bitC`.
198 |     Does not check for register overflow ! */
199 | MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
200 | {
201 |     bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
202 |     bitC->bitPos += nbBits;
203 | }
204 | 
205 | /*! BIT_addBitsFast() :
206 |  *  works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
207 | MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
208 | {
209 |     bitC->bitContainer |= value << bitC->bitPos;
210 |     bitC->bitPos += nbBits;
211 | }
212 | 
213 | /*! BIT_flushBitsFast() :
214 |  *  unsafe version; does not check buffer overflow */
215 | MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
216 | {
217 |     size_t const nbBytes = bitC->bitPos >> 3;
218 |     MEM_writeLEST(bitC->ptr, bitC->bitContainer);
219 |     bitC->ptr += nbBytes;
220 |     bitC->bitPos &= 7;
221 |     bitC->bitContainer >>= nbBytes*8;   /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
222 | }
223 | 
224 | /*! BIT_flushBits() :
225 |  *  safe version; check for buffer overflow, and prevents it.
226 |  *  note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
227 | MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
228 | {
229 |     size_t const nbBytes = bitC->bitPos >> 3;
230 |     MEM_writeLEST(bitC->ptr, bitC->bitContainer);
231 |     bitC->ptr += nbBytes;
232 |     if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
233 |     bitC->bitPos &= 7;
234 |     bitC->bitContainer >>= nbBytes*8;   /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
235 | }
236 | 
237 | /*! BIT_closeCStream() :
238 |  *  @return : size of CStream, in bytes,
239 |               or 0 if it could not fit into dstBuffer */
240 | MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
241 | {
242 |     BIT_addBitsFast(bitC, 1, 1);   /* endMark */
243 |     BIT_flushBits(bitC);
244 | 
245 |     if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */
246 | 
247 |     return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
248 | }
249 | 
250 | 
251 | /*-********************************************************
252 | * bitStream decoding
253 | **********************************************************/
254 | /*! BIT_initDStream() :
255 | *   Initialize a BIT_DStream_t.
256 | *   `bitD` : a pointer to an already allocated BIT_DStream_t structure.
257 | *   `srcSize` must be the *exact* size of the bitStream, in bytes.
258 | *   @return : size of stream (== srcSize) or an errorCode if a problem is detected
259 | */
260 | MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
261 | {
262 |     if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
263 | 
264 |     if (srcSize >=  sizeof(bitD->bitContainer)) {  /* normal case */
265 |         bitD->start = (const char*)srcBuffer;
266 |         bitD->ptr   = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
267 |         bitD->bitContainer = MEM_readLEST(bitD->ptr);
268 |         { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
269 |           bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
270 |           if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
271 |     } else {
272 |         bitD->start = (const char*)srcBuffer;
273 |         bitD->ptr   = bitD->start;
274 |         bitD->bitContainer = *(const BYTE*)(bitD->start);
275 |         switch(srcSize)
276 |         {
277 |             case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
278 |             case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
279 |             case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
280 |             case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
281 |             case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
282 |             case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) <<  8;
283 |             default:;
284 |         }
285 |         { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
286 |           bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
287 |           if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
288 |         bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
289 |     }
290 | 
291 |     return srcSize;
292 | }
293 | 
294 | MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
295 | {
296 |     return bitContainer >> start;
297 | }
298 | 
299 | MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
300 | {
301 | #if defined(__BMI__) && defined(__GNUC__)   /* experimental */
302 | #  if defined(__x86_64__)
303 |     if (sizeof(bitContainer)==8)
304 |         return _bextr_u64(bitContainer, start, nbBits);
305 |     else
306 | #  endif
307 |         return _bextr_u32(bitContainer, start, nbBits);
308 | #else
309 |     return (bitContainer >> start) & BIT_mask[nbBits];
310 | #endif
311 | }
312 | 
313 | MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
314 | {
315 |     return bitContainer & BIT_mask[nbBits];
316 | }
317 | 
318 | /*! BIT_lookBits() :
319 |  *  Provides next n bits from local register.
320 |  *  local register is not modified.
321 |  *  On 32-bits, maxNbBits==24.
322 |  *  On 64-bits, maxNbBits==56.
323 |  *  @return : value extracted
324 |  */
325 |  MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
326 | {
327 | #if defined(__BMI__) && defined(__GNUC__)   /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
328 |     return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
329 | #else
330 |     U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
331 |     return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
332 | #endif
333 | }
334 | 
335 | /*! BIT_lookBitsFast() :
336 | *   unsafe version; only works only if nbBits >= 1 */
337 | MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
338 | {
339 |     U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
340 |     return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
341 | }
342 | 
343 | MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
344 | {
345 |     bitD->bitsConsumed += nbBits;
346 | }
347 | 
348 | /*! BIT_readBits() :
349 |  *  Read (consume) next n bits from local register and update.
350 |  *  Pay attention to not read more than nbBits contained into local register.
351 |  *  @return : extracted value.
352 |  */
353 | MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
354 | {
355 |     size_t const value = BIT_lookBits(bitD, nbBits);
356 |     BIT_skipBits(bitD, nbBits);
357 |     return value;
358 | }
359 | 
360 | /*! BIT_readBitsFast() :
361 | *   unsafe version; only works only if nbBits >= 1 */
362 | MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
363 | {
364 |     size_t const value = BIT_lookBitsFast(bitD, nbBits);
365 |     BIT_skipBits(bitD, nbBits);
366 |     return value;
367 | }
368 | 
369 | /*! BIT_reloadDStream() :
370 | *   Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ).
371 | *   This function is safe, it guarantees it will not read beyond src buffer.
372 | *   @return : status of `BIT_DStream_t` internal register.
373 |               if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
374 | MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
375 | {
376 | 	if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))  /* should not happen => corruption detected */
377 | 		return BIT_DStream_overflow;
378 | 
379 |     if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
380 |         bitD->ptr -= bitD->bitsConsumed >> 3;
381 |         bitD->bitsConsumed &= 7;
382 |         bitD->bitContainer = MEM_readLEST(bitD->ptr);
383 |         return BIT_DStream_unfinished;
384 |     }
385 |     if (bitD->ptr == bitD->start) {
386 |         if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
387 |         return BIT_DStream_completed;
388 |     }
389 |     {   U32 nbBytes = bitD->bitsConsumed >> 3;
390 |         BIT_DStream_status result = BIT_DStream_unfinished;
391 |         if (bitD->ptr - nbBytes < bitD->start) {
392 |             nbBytes = (U32)(bitD->ptr - bitD->start);  /* ptr > start */
393 |             result = BIT_DStream_endOfBuffer;
394 |         }
395 |         bitD->ptr -= nbBytes;
396 |         bitD->bitsConsumed -= nbBytes*8;
397 |         bitD->bitContainer = MEM_readLEST(bitD->ptr);   /* reminder : srcSize > sizeof(bitD) */
398 |         return result;
399 |     }
400 | }
401 | 
402 | /*! BIT_endOfDStream() :
403 | *   @return Tells if DStream has exactly reached its end (all bits consumed).
404 | */
405 | MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
406 | {
407 |     return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
408 | }
409 | 
410 | #if defined (__cplusplus)
411 | }
412 | #endif
413 | 
414 | #endif /* BITSTREAM_H_MODULE */
415 | 


--------------------------------------------------------------------------------
/zstd/entropy_common.inc:
--------------------------------------------------------------------------------
  1 | /*
  2 |    Common functions of New Generation Entropy library
  3 |    Copyright (C) 2016, Yann Collet.
  4 | 
  5 |    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
  6 | 
  7 |    Redistribution and use in source and binary forms, with or without
  8 |    modification, are permitted provided that the following conditions are
  9 |    met:
 10 | 
 11 |        * Redistributions of source code must retain the above copyright
 12 |    notice, this list of conditions and the following disclaimer.
 13 |        * Redistributions in binary form must reproduce the above
 14 |    copyright notice, this list of conditions and the following disclaimer
 15 |    in the documentation and/or other materials provided with the
 16 |    distribution.
 17 | 
 18 |    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 19 |    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 20 |    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 21 |    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 22 |    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 23 |    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 24 |    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 25 |    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 26 |    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 27 |    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 28 |    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 29 | 
 30 |     You can contact the author at :
 31 |     - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
 32 |     - Public forum : https://groups.google.com/forum/#!forum/lz4c
 33 | *************************************************************************** */
 34 | 
 35 | /* *************************************
 36 | *  Dependencies
 37 | ***************************************/
 38 | #include "mem.h"
 39 | #include "error_private.h"       /* ERR_*, ERROR */
 40 | #define FSE_STATIC_LINKING_ONLY  /* FSE_MIN_TABLELOG */
 41 | #include "fse.h"
 42 | #define HUF_STATIC_LINKING_ONLY  /* HUF_TABLELOG_ABSOLUTEMAX */
 43 | #include "huf.h"
 44 | 
 45 | 
 46 | /*-****************************************
 47 | *  FSE Error Management
 48 | ******************************************/
 49 | unsigned FSE_isError(size_t code) { return ERR_isError(code); }
 50 | 
 51 | const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
 52 | 
 53 | 
 54 | /* **************************************************************
 55 | *  HUF Error Management
 56 | ****************************************************************/
 57 | unsigned HUF_isError(size_t code) { return ERR_isError(code); }
 58 | 
 59 | const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
 60 | 
 61 | 
 62 | /*-**************************************************************
 63 | *  FSE NCount encoding-decoding
 64 | ****************************************************************/
 65 | static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
 66 | 
 67 | size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
 68 |                  const void* headerBuffer, size_t hbSize)
 69 | {
 70 |     const BYTE* const istart = (const BYTE*) headerBuffer;
 71 |     const BYTE* const iend = istart + hbSize;
 72 |     const BYTE* ip = istart;
 73 |     int nbBits;
 74 |     int remaining;
 75 |     int threshold;
 76 |     U32 bitStream;
 77 |     int bitCount;
 78 |     unsigned charnum = 0;
 79 |     int previous0 = 0;
 80 | 
 81 |     if (hbSize < 4) return ERROR(srcSize_wrong);
 82 |     bitStream = MEM_readLE32(ip);
 83 |     nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG;   /* extract tableLog */
 84 |     if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
 85 |     bitStream >>= 4;
 86 |     bitCount = 4;
 87 |     *tableLogPtr = nbBits;
 88 |     remaining = (1<<nbBits)+1;
 89 |     threshold = 1<<nbBits;
 90 |     nbBits++;
 91 | 
 92 |     while ((remaining>1) & (charnum<=*maxSVPtr)) {
 93 |         if (previous0) {
 94 |             unsigned n0 = charnum;
 95 |             while ((bitStream & 0xFFFF) == 0xFFFF) {
 96 |                 n0 += 24;
 97 |                 if (ip < iend-5) {
 98 |                     ip += 2;
 99 |                     bitStream = MEM_readLE32(ip) >> bitCount;
100 |                 } else {
101 |                     bitStream >>= 16;
102 |                     bitCount   += 16;
103 |             }   }
104 |             while ((bitStream & 3) == 3) {
105 |                 n0 += 3;
106 |                 bitStream >>= 2;
107 |                 bitCount += 2;
108 |             }
109 |             n0 += bitStream & 3;
110 |             bitCount += 2;
111 |             if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
112 |             while (charnum < n0) normalizedCounter[charnum++] = 0;
113 |             if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
114 |                 ip += bitCount>>3;
115 |                 bitCount &= 7;
116 |                 bitStream = MEM_readLE32(ip) >> bitCount;
117 |             } else {
118 |                 bitStream >>= 2;
119 |         }   }
120 |         {   short const max = (short)((2*threshold-1)-remaining);
121 |             short count;
122 | 
123 |             if ((bitStream & (threshold-1)) < (U32)max) {
124 |                 count = (short)(bitStream & (threshold-1));
125 |                 bitCount   += nbBits-1;
126 |             } else {
127 |                 count = (short)(bitStream & (2*threshold-1));
128 |                 if (count >= threshold) count -= max;
129 |                 bitCount   += nbBits;
130 |             }
131 | 
132 |             count--;   /* extra accuracy */
133 |             remaining -= FSE_abs(count);
134 |             normalizedCounter[charnum++] = count;
135 |             previous0 = !count;
136 |             while (remaining < threshold) {
137 |                 nbBits--;
138 |                 threshold >>= 1;
139 |             }
140 | 
141 |             if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
142 |                 ip += bitCount>>3;
143 |                 bitCount &= 7;
144 |             } else {
145 |                 bitCount -= (int)(8 * (iend - 4 - ip));
146 |                 ip = iend - 4;
147 |             }
148 |             bitStream = MEM_readLE32(ip) >> (bitCount & 31);
149 |     }   }   /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
150 |     if (remaining != 1) return ERROR(corruption_detected);
151 |     if (bitCount > 32) return ERROR(corruption_detected);
152 |     *maxSVPtr = charnum-1;
153 | 
154 |     ip += (bitCount+7)>>3;
155 |     return ip-istart;
156 | }
157 | 
158 | 
159 | /*! HUF_readStats() :
160 |     Read compact Huffman tree, saved by HUF_writeCTable().
161 |     `huffWeight` is destination buffer.
162 |     @return : size read from `src` , or an error Code .
163 |     Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
164 | */
165 | size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
166 |                      U32* nbSymbolsPtr, U32* tableLogPtr,
167 |                      const void* src, size_t srcSize)
168 | {
169 |     U32 weightTotal;
170 |     const BYTE* ip = (const BYTE*) src;
171 |     size_t iSize;
172 |     size_t oSize;
173 | 
174 |     if (!srcSize) return ERROR(srcSize_wrong);
175 |     iSize = ip[0];
176 |     /* memset(huffWeight, 0, hwSize);   *//* is not necessary, even though some analyzer complain ... */
177 | 
178 |     if (iSize >= 128) {  /* special header */
179 |         oSize = iSize - 127;
180 |         iSize = ((oSize+1)/2);
181 |         if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
182 |         if (oSize >= hwSize) return ERROR(corruption_detected);
183 |         ip += 1;
184 |         {   U32 n;
185 |             for (n=0; n<oSize; n+=2) {
186 |                 huffWeight[n]   = ip[n/2] >> 4;
187 |                 huffWeight[n+1] = ip[n/2] & 15;
188 |     }   }   }
189 |     else  {   /* header compressed with FSE (normal case) */
190 |         if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
191 |         oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize);   /* max (hwSize-1) values decoded, as last one is implied */
192 |         if (FSE_isError(oSize)) return oSize;
193 |     }
194 | 
195 |     /* collect weight stats */
196 |     memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32));
197 |     weightTotal = 0;
198 |     {   U32 n; for (n=0; n<oSize; n++) {
199 |             if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
200 |             rankStats[huffWeight[n]]++;
201 |             weightTotal += (1 << huffWeight[n]) >> 1;
202 |     }   }
203 |     if (weightTotal == 0) return ERROR(corruption_detected);
204 | 
205 |     /* get last non-null symbol weight (implied, total must be 2^n) */
206 |     {   U32 const tableLog = BIT_highbit32(weightTotal) + 1;
207 |         if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
208 |         *tableLogPtr = tableLog;
209 |         /* determine last weight */
210 |         {   U32 const total = 1 << tableLog;
211 |             U32 const rest = total - weightTotal;
212 |             U32 const verif = 1 << BIT_highbit32(rest);
213 |             U32 const lastWeight = BIT_highbit32(rest) + 1;
214 |             if (verif != rest) return ERROR(corruption_detected);    /* last value must be a clean power of 2 */
215 |             huffWeight[oSize] = (BYTE)lastWeight;
216 |             rankStats[lastWeight]++;
217 |     }   }
218 | 
219 |     /* check tree construction validity */
220 |     if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected);   /* by construction : at least 2 elts of rank 1, must be even */
221 | 
222 |     /* results */
223 |     *nbSymbolsPtr = (U32)(oSize+1);
224 |     return iSize+1;
225 | }
226 | 


--------------------------------------------------------------------------------
/zstd/error_private.h:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
 3 |  * All rights reserved.
 4 |  *
 5 |  * This source code is licensed under the BSD-style license found in the
 6 |  * LICENSE file in the root directory of this source tree. An additional grant
 7 |  * of patent rights can be found in the PATENTS file in the same directory.
 8 |  */
 9 | 
10 | /* Note : this module is expected to remain private, do not expose it */
11 | 
12 | #ifndef ERROR_H_MODULE
13 | #define ERROR_H_MODULE
14 | 
15 | #if defined (__cplusplus)
16 | extern "C" {
17 | #endif
18 | 
19 | 
20 | /* ****************************************
21 | *  Dependencies
22 | ******************************************/
23 | #include <stddef.h>        /* size_t */
24 | #include "zstd_errors.h"  /* enum list */
25 | 
26 | 
27 | /* ****************************************
28 | *  Compiler-specific
29 | ******************************************/
30 | #if defined(__GNUC__)
31 | #  define ERR_STATIC static __attribute__((unused))
32 | #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
33 | #  define ERR_STATIC static inline
34 | #elif defined(_MSC_VER)
35 | #  define ERR_STATIC static __inline
36 | #else
37 | #  define ERR_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
38 | #endif
39 | 
40 | 
41 | /*-****************************************
42 | *  Customization (error_public.h)
43 | ******************************************/
44 | typedef ZSTD_ErrorCode ERR_enum;
45 | #define PREFIX(name) ZSTD_error_##name
46 | 
47 | 
48 | /*-****************************************
49 | *  Error codes handling
50 | ******************************************/
51 | #ifdef ERROR
52 | #  undef ERROR   /* reported already defined on VS 2015 (Rich Geldreich) */
53 | #endif
54 | #define ERROR(name) ((size_t)-PREFIX(name))
55 | 
56 | ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
57 | 
58 | ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
59 | 
60 | 
61 | /*-****************************************
62 | *  Error Strings
63 | ******************************************/
64 | 
65 | const char* ERR_getErrorString(ERR_enum code);   /* error_private.c */
66 | 
67 | ERR_STATIC const char* ERR_getErrorName(size_t code)
68 | {
69 |     return ERR_getErrorString(ERR_getErrorCode(code));
70 | }
71 | 
72 | #if defined (__cplusplus)
73 | }
74 | #endif
75 | 
76 | #endif /* ERROR_H_MODULE */
77 | 


--------------------------------------------------------------------------------
/zstd/error_private.inc:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
 3 |  * All rights reserved.
 4 |  *
 5 |  * This source code is licensed under the BSD-style license found in the
 6 |  * LICENSE file in the root directory of this source tree. An additional grant
 7 |  * of patent rights can be found in the PATENTS file in the same directory.
 8 |  */
 9 | 
10 | /* The purpose of this file is to have a single list of error strings embedded in binary */
11 | 
12 | #include "error_private.h"
13 | 
14 | const char* ERR_getErrorString(ERR_enum code)
15 | {
16 |     static const char* const notErrorCode = "Unspecified error code";
17 |     switch( code )
18 |     {
19 |     case PREFIX(no_error): return "No error detected";
20 |     case PREFIX(GENERIC):  return "Error (generic)";
21 |     case PREFIX(prefix_unknown): return "Unknown frame descriptor";
22 |     case PREFIX(version_unsupported): return "Version not supported";
23 |     case PREFIX(parameter_unknown): return "Unknown parameter type";
24 |     case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
25 |     case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
26 |     case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
27 |     case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
28 |     case PREFIX(init_missing): return "Context should be init first";
29 |     case PREFIX(memory_allocation): return "Allocation error : not enough memory";
30 |     case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
31 |     case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
32 |     case PREFIX(srcSize_wrong): return "Src size incorrect";
33 |     case PREFIX(corruption_detected): return "Corrupted block detected";
34 |     case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
35 |     case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
36 |     case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
37 |     case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
38 |     case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
39 |     case PREFIX(dictionary_wrong): return "Dictionary mismatch";
40 |     case PREFIX(maxCode):
41 |     default: return notErrorCode;
42 |     }
43 | }
44 | 


--------------------------------------------------------------------------------
/zstd/fse_decompress.inc:
--------------------------------------------------------------------------------
  1 | /* ******************************************************************
  2 |    FSE : Finite State Entropy decoder
  3 |    Copyright (C) 2013-2015, Yann Collet.
  4 | 
  5 |    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
  6 | 
  7 |    Redistribution and use in source and binary forms, with or without
  8 |    modification, are permitted provided that the following conditions are
  9 |    met:
 10 | 
 11 |        * Redistributions of source code must retain the above copyright
 12 |    notice, this list of conditions and the following disclaimer.
 13 |        * Redistributions in binary form must reproduce the above
 14 |    copyright notice, this list of conditions and the following disclaimer
 15 |    in the documentation and/or other materials provided with the
 16 |    distribution.
 17 | 
 18 |    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 19 |    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 20 |    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 21 |    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 22 |    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 23 |    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 24 |    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 25 |    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 26 |    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 27 |    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 28 |    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 29 | 
 30 |     You can contact the author at :
 31 |     - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
 32 |     - Public forum : https://groups.google.com/forum/#!forum/lz4c
 33 | ****************************************************************** */
 34 | 
 35 | 
 36 | /* **************************************************************
 37 | *  Compiler specifics
 38 | ****************************************************************/
 39 | #ifdef _MSC_VER    /* Visual Studio */
 40 | #  define FORCE_INLINE static __forceinline
 41 | #  include <intrin.h>                    /* For Visual 2005 */
 42 | #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 43 | #  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
 44 | #else
 45 | #  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
 46 | #    ifdef __GNUC__
 47 | #      define FORCE_INLINE static inline __attribute__((always_inline))
 48 | #    else
 49 | #      define FORCE_INLINE static inline
 50 | #    endif
 51 | #  else
 52 | #    define FORCE_INLINE static
 53 | #  endif /* __STDC_VERSION__ */
 54 | #endif
 55 | 
 56 | 
 57 | /* **************************************************************
 58 | *  Includes
 59 | ****************************************************************/
 60 | #include <stdlib.h>     /* malloc, free, qsort */
 61 | #include <string.h>     /* memcpy, memset */
 62 | #include <stdio.h>      /* printf (debug) */
 63 | #include "bitstream.h"
 64 | #define FSE_STATIC_LINKING_ONLY
 65 | #include "fse.h"
 66 | 
 67 | 
 68 | /* **************************************************************
 69 | *  Error Management
 70 | ****************************************************************/
 71 | #define FSE_isError ERR_isError
 72 | #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 73 | 
 74 | /* check and forward error code */
 75 | #define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
 76 | 
 77 | 
 78 | /* **************************************************************
 79 | *  Complex types
 80 | ****************************************************************/
 81 | typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
 82 | 
 83 | 
 84 | /* **************************************************************
 85 | *  Templates
 86 | ****************************************************************/
 87 | /*
 88 |   designed to be included
 89 |   for type-specific functions (template emulation in C)
 90 |   Objective is to write these functions only once, for improved maintenance
 91 | */
 92 | 
 93 | /* safety checks */
 94 | #ifndef FSE_FUNCTION_EXTENSION
 95 | #  error "FSE_FUNCTION_EXTENSION must be defined"
 96 | #endif
 97 | #ifndef FSE_FUNCTION_TYPE
 98 | #  error "FSE_FUNCTION_TYPE must be defined"
 99 | #endif
100 | 
101 | /* Function names */
102 | #define FSE_CAT(X,Y) X##Y
103 | #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
104 | #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
105 | 
106 | 
107 | /* Function templates */
108 | FSE_DTable* FSE_createDTable (unsigned tableLog)
109 | {
110 |     if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
111 |     return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
112 | }
113 | 
114 | void FSE_freeDTable (FSE_DTable* dt)
115 | {
116 |     free(dt);
117 | }
118 | 
119 | size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
120 | {
121 |     void* const tdPtr = dt+1;   /* because *dt is unsigned, 32-bits aligned on 32-bits */
122 |     FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
123 |     U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
124 | 
125 |     U32 const maxSV1 = maxSymbolValue + 1;
126 |     U32 const tableSize = 1 << tableLog;
127 |     U32 highThreshold = tableSize-1;
128 | 
129 |     /* Sanity Checks */
130 |     if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
131 |     if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
132 | 
133 |     /* Init, lay down lowprob symbols */
134 |     {   FSE_DTableHeader DTableH;
135 |         DTableH.tableLog = (U16)tableLog;
136 |         DTableH.fastMode = 1;
137 |         {   S16 const largeLimit= (S16)(1 << (tableLog-1));
138 |             U32 s;
139 |             for (s=0; s<maxSV1; s++) {
140 |                 if (normalizedCounter[s]==-1) {
141 |                     tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
142 |                     symbolNext[s] = 1;
143 |                 } else {
144 |                     if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
145 |                     symbolNext[s] = normalizedCounter[s];
146 |         }   }   }
147 |         memcpy(dt, &DTableH, sizeof(DTableH));
148 |     }
149 | 
150 |     /* Spread symbols */
151 |     {   U32 const tableMask = tableSize-1;
152 |         U32 const step = FSE_TABLESTEP(tableSize);
153 |         U32 s, position = 0;
154 |         for (s=0; s<maxSV1; s++) {
155 |             int i;
156 |             for (i=0; i<normalizedCounter[s]; i++) {
157 |                 tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
158 |                 position = (position + step) & tableMask;
159 |                 while (position > highThreshold) position = (position + step) & tableMask;   /* lowprob area */
160 |         }   }
161 |         if (position!=0) return ERROR(GENERIC);   /* position must reach all cells once, otherwise normalizedCounter is incorrect */
162 |     }
163 | 
164 |     /* Build Decoding table */
165 |     {   U32 u;
166 |         for (u=0; u<tableSize; u++) {
167 |             FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
168 |             U16 nextState = symbolNext[symbol]++;
169 |             tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
170 |             tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
171 |     }   }
172 | 
173 |     return 0;
174 | }
175 | 
176 | 
177 | #ifndef FSE_COMMONDEFS_ONLY
178 | 
179 | /*-*******************************************************
180 | *  Decompression (Byte symbols)
181 | *********************************************************/
182 | size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
183 | {
184 |     void* ptr = dt;
185 |     FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
186 |     void* dPtr = dt + 1;
187 |     FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
188 | 
189 |     DTableH->tableLog = 0;
190 |     DTableH->fastMode = 0;
191 | 
192 |     cell->newState = 0;
193 |     cell->symbol = symbolValue;
194 |     cell->nbBits = 0;
195 | 
196 |     return 0;
197 | }
198 | 
199 | 
200 | size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
201 | {
202 |     void* ptr = dt;
203 |     FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
204 |     void* dPtr = dt + 1;
205 |     FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
206 |     const unsigned tableSize = 1 << nbBits;
207 |     const unsigned tableMask = tableSize - 1;
208 |     const unsigned maxSV1 = tableMask+1;
209 |     unsigned s;
210 | 
211 |     /* Sanity checks */
212 |     if (nbBits < 1) return ERROR(GENERIC);         /* min size */
213 | 
214 |     /* Build Decoding Table */
215 |     DTableH->tableLog = (U16)nbBits;
216 |     DTableH->fastMode = 1;
217 |     for (s=0; s<maxSV1; s++) {
218 |         dinfo[s].newState = 0;
219 |         dinfo[s].symbol = (BYTE)s;
220 |         dinfo[s].nbBits = (BYTE)nbBits;
221 |     }
222 | 
223 |     return 0;
224 | }
225 | 
226 | FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
227 |           void* dst, size_t maxDstSize,
228 |     const void* cSrc, size_t cSrcSize,
229 |     const FSE_DTable* dt, const unsigned fast)
230 | {
231 |     BYTE* const ostart = (BYTE*) dst;
232 |     BYTE* op = ostart;
233 |     BYTE* const omax = op + maxDstSize;
234 |     BYTE* const olimit = omax-3;
235 | 
236 |     BIT_DStream_t bitD;
237 |     FSE_DState_t state1;
238 |     FSE_DState_t state2;
239 | 
240 |     /* Init */
241 |     CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
242 | 
243 |     FSE_initDState(&state1, &bitD, dt);
244 |     FSE_initDState(&state2, &bitD, dt);
245 | 
246 | #define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
247 | 
248 |     /* 4 symbols per loop */
249 |     for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) {
250 |         op[0] = FSE_GETSYMBOL(&state1);
251 | 
252 |         if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8)    /* This test must be static */
253 |             BIT_reloadDStream(&bitD);
254 | 
255 |         op[1] = FSE_GETSYMBOL(&state2);
256 | 
257 |         if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8)    /* This test must be static */
258 |             { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
259 | 
260 |         op[2] = FSE_GETSYMBOL(&state1);
261 | 
262 |         if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8)    /* This test must be static */
263 |             BIT_reloadDStream(&bitD);
264 | 
265 |         op[3] = FSE_GETSYMBOL(&state2);
266 |     }
267 | 
268 |     /* tail */
269 |     /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
270 |     while (1) {
271 |         if (op>(omax-2)) return ERROR(dstSize_tooSmall);
272 |         *op++ = FSE_GETSYMBOL(&state1);
273 |         if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
274 |             *op++ = FSE_GETSYMBOL(&state2);
275 |             break;
276 |         }
277 | 
278 |         if (op>(omax-2)) return ERROR(dstSize_tooSmall);
279 |         *op++ = FSE_GETSYMBOL(&state2);
280 |         if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
281 |             *op++ = FSE_GETSYMBOL(&state1);
282 |             break;
283 |     }   }
284 | 
285 |     return op-ostart;
286 | }
287 | 
288 | 
289 | size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
290 |                             const void* cSrc, size_t cSrcSize,
291 |                             const FSE_DTable* dt)
292 | {
293 |     const void* ptr = dt;
294 |     const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
295 |     const U32 fastMode = DTableH->fastMode;
296 | 
297 |     /* select fast mode (static) */
298 |     if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
299 |     return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
300 | }
301 | 
302 | 
303 | size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
304 | {
305 |     const BYTE* const istart = (const BYTE*)cSrc;
306 |     const BYTE* ip = istart;
307 |     short counting[FSE_MAX_SYMBOL_VALUE+1];
308 |     DTable_max_t dt;   /* Static analyzer seems unable to understand this table will be properly initialized later */
309 |     unsigned tableLog;
310 |     unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
311 | 
312 |     if (cSrcSize<2) return ERROR(srcSize_wrong);   /* too small input size */
313 | 
314 |     /* normal FSE decoding mode */
315 |     {   size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
316 |         if (FSE_isError(NCountLength)) return NCountLength;
317 |         if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong);   /* too small input size */
318 |         ip += NCountLength;
319 |         cSrcSize -= NCountLength;
320 |     }
321 | 
322 |     CHECK_F( FSE_buildDTable (dt, counting, maxSymbolValue, tableLog) );
323 | 
324 |     return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt);   /* always return, even if it is an error code */
325 | }
326 | 
327 | 
328 | 
329 | #endif   /* FSE_COMMONDEFS_ONLY */
330 | 


--------------------------------------------------------------------------------
/zstd/huf.h:
--------------------------------------------------------------------------------
  1 | /* ******************************************************************
  2 |    Huffman coder, part of New Generation Entropy library
  3 |    header file
  4 |    Copyright (C) 2013-2016, Yann Collet.
  5 | 
  6 |    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
  7 | 
  8 |    Redistribution and use in source and binary forms, with or without
  9 |    modification, are permitted provided that the following conditions are
 10 |    met:
 11 | 
 12 |        * Redistributions of source code must retain the above copyright
 13 |    notice, this list of conditions and the following disclaimer.
 14 |        * Redistributions in binary form must reproduce the above
 15 |    copyright notice, this list of conditions and the following disclaimer
 16 |    in the documentation and/or other materials provided with the
 17 |    distribution.
 18 | 
 19 |    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 20 |    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 21 |    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 22 |    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 23 |    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 24 |    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 25 |    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 26 |    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 27 |    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 28 |    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 29 |    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 30 | 
 31 |    You can contact the author at :
 32 |    - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
 33 | ****************************************************************** */
 34 | #ifndef HUF_H_298734234
 35 | #define HUF_H_298734234
 36 | 
 37 | #if defined (__cplusplus)
 38 | extern "C" {
 39 | #endif
 40 | 
 41 | 
 42 | /* *** Dependencies *** */
 43 | #include <stddef.h>    /* size_t */
 44 | 
 45 | 
 46 | /* *** simple functions *** */
 47 | /**
 48 | HUF_compress() :
 49 |     Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
 50 |     'dst' buffer must be already allocated.
 51 |     Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
 52 |     `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
 53 |     @return : size of compressed data (<= `dstCapacity`).
 54 |     Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
 55 |                      if return == 1, srcData is a single repeated byte symbol (RLE compression).
 56 |                      if HUF_isError(return), compression failed (more details using HUF_getErrorName())
 57 | */
 58 | size_t HUF_compress(void* dst, size_t dstCapacity,
 59 |               const void* src, size_t srcSize);
 60 | 
 61 | /**
 62 | HUF_decompress() :
 63 |     Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
 64 |     into already allocated buffer 'dst', of minimum size 'dstSize'.
 65 |     `dstSize` : **must** be the ***exact*** size of original (uncompressed) data.
 66 |     Note : in contrast with FSE, HUF_decompress can regenerate
 67 |            RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
 68 |            because it knows size to regenerate.
 69 |     @return : size of regenerated data (== dstSize),
 70 |               or an error code, which can be tested using HUF_isError()
 71 | */
 72 | size_t HUF_decompress(void* dst,  size_t dstSize,
 73 |                 const void* cSrc, size_t cSrcSize);
 74 | 
 75 | 
 76 | /* ****************************************
 77 | *  Tool functions
 78 | ******************************************/
 79 | #define HUF_BLOCKSIZE_MAX (128 * 1024)
 80 | size_t HUF_compressBound(size_t size);       /**< maximum compressed size (worst case) */
 81 | 
 82 | /* Error Management */
 83 | unsigned    HUF_isError(size_t code);        /**< tells if a return value is an error code */
 84 | const char* HUF_getErrorName(size_t code);   /**< provides error code string (useful for debugging) */
 85 | 
 86 | 
 87 | /* *** Advanced function *** */
 88 | 
 89 | /** HUF_compress2() :
 90 | *   Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` */
 91 | size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
 92 | 
 93 | 
 94 | #ifdef HUF_STATIC_LINKING_ONLY
 95 | 
 96 | /* *** Dependencies *** */
 97 | #include "mem.h"   /* U32 */
 98 | 
 99 | 
100 | /* *** Constants *** */
101 | #define HUF_TABLELOG_ABSOLUTEMAX  16   /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
102 | #define HUF_TABLELOG_MAX  12           /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
103 | #define HUF_TABLELOG_DEFAULT  11       /* tableLog by default, when not specified */
104 | #define HUF_SYMBOLVALUE_MAX 255
105 | #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
106 | #  error "HUF_TABLELOG_MAX is too large !"
107 | #endif
108 | 
109 | 
110 | /* ****************************************
111 | *  Static allocation
112 | ******************************************/
113 | /* HUF buffer bounds */
114 | #define HUF_CTABLEBOUND 129
115 | #define HUF_BLOCKBOUND(size) (size + (size>>8) + 8)   /* only true if incompressible pre-filtered with fast heuristic */
116 | #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size))   /* Macro version, useful for static allocation */
117 | 
118 | /* static allocation of HUF's Compression Table */
119 | #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
120 |     U32 name##hb[maxSymbolValue+1]; \
121 |     void* name##hv = &(name##hb); \
122 |     HUF_CElt* name = (HUF_CElt*)(name##hv)   /* no final ; */
123 | 
124 | /* static allocation of HUF's DTable */
125 | typedef U32 HUF_DTable;
126 | #define HUF_DTABLE_SIZE(maxTableLog)   (1 + (1<<(maxTableLog)))
127 | #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
128 |         HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) }
129 | #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
130 |         HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) }
131 | 
132 | 
133 | /* ****************************************
134 | *  Advanced decompression functions
135 | ******************************************/
136 | size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
137 | size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
138 | 
139 | size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< decodes RLE and uncompressed */
140 | size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
141 | size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
142 | size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
143 | 
144 | size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
145 | size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< single-symbol decoder */
146 | size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /**< double-symbols decoder */
147 | 
148 | 
149 | /* ****************************************
150 | *  HUF detailed API
151 | ******************************************/
152 | /*!
153 | HUF_compress() does the following:
154 | 1. count symbol occurrence from source[] into table count[] using FSE_count()
155 | 2. (optional) refine tableLog using HUF_optimalTableLog()
156 | 3. build Huffman table from count using HUF_buildCTable()
157 | 4. save Huffman table to memory buffer using HUF_writeCTable()
158 | 5. encode the data stream using HUF_compress4X_usingCTable()
159 | 
160 | The following API allows targeting specific sub-functions for advanced tasks.
161 | For example, it's possible to compress several blocks using the same 'CTable',
162 | or to save and regenerate 'CTable' using external methods.
163 | */
164 | /* FSE_count() : find it within "fse.h" */
165 | unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
166 | typedef struct HUF_CElt_s HUF_CElt;   /* incomplete type */
167 | size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
168 | size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
169 | size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
170 | 
171 | 
172 | /*! HUF_readStats() :
173 |     Read compact Huffman tree, saved by HUF_writeCTable().
174 |     `huffWeight` is destination buffer.
175 |     @return : size read from `src` , or an error Code .
176 |     Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
177 | size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
178 |                      U32* nbSymbolsPtr, U32* tableLogPtr,
179 |                      const void* src, size_t srcSize);
180 | 
181 | /** HUF_readCTable() :
182 | *   Loading a CTable saved with HUF_writeCTable() */
183 | size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
184 | 
185 | 
186 | /*
187 | HUF_decompress() does the following:
188 | 1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
189 | 2. build Huffman table from save, using HUF_readDTableXn()
190 | 3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
191 | */
192 | 
193 | /** HUF_selectDecoder() :
194 | *   Tells which decoder is likely to decode faster,
195 | *   based on a set of pre-determined metrics.
196 | *   @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
197 | *   Assumption : 0 < cSrcSize < dstSize <= 128 KB */
198 | U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
199 | 
200 | size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
201 | size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize);
202 | 
203 | size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
204 | size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
205 | size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
206 | 
207 | 
208 | /* single stream variants */
209 | 
210 | size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
211 | size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
212 | 
213 | size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* single-symbol decoder */
214 | size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);   /* double-symbol decoder */
215 | 
216 | size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
217 | size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
218 | size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
219 | 
220 | 
221 | #endif /* HUF_STATIC_LINKING_ONLY */
222 | 
223 | 
224 | #if defined (__cplusplus)
225 | }
226 | #endif
227 | 
228 | #endif   /* HUF_H_298734234 */
229 | 


--------------------------------------------------------------------------------
/zstd/huf_compress.inc:
--------------------------------------------------------------------------------
  1 | /* ******************************************************************
  2 |    Huffman encoder, part of New Generation Entropy library
  3 |    Copyright (C) 2013-2016, Yann Collet.
  4 | 
  5 |    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
  6 | 
  7 |    Redistribution and use in source and binary forms, with or without
  8 |    modification, are permitted provided that the following conditions are
  9 |    met:
 10 | 
 11 |        * Redistributions of source code must retain the above copyright
 12 |    notice, this list of conditions and the following disclaimer.
 13 |        * Redistributions in binary form must reproduce the above
 14 |    copyright notice, this list of conditions and the following disclaimer
 15 |    in the documentation and/or other materials provided with the
 16 |    distribution.
 17 | 
 18 |    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 19 |    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 20 |    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 21 |    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 22 |    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 23 |    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 24 |    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 25 |    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 26 |    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 27 |    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 28 |    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 29 | 
 30 |     You can contact the author at :
 31 |     - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
 32 |     - Public forum : https://groups.google.com/forum/#!forum/lz4c
 33 | ****************************************************************** */
 34 | 
 35 | /* **************************************************************
 36 | *  Compiler specifics
 37 | ****************************************************************/
 38 | #ifdef _MSC_VER    /* Visual Studio */
 39 | #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 40 | #endif
 41 | 
 42 | 
 43 | /* **************************************************************
 44 | *  Includes
 45 | ****************************************************************/
 46 | #include <string.h>     /* memcpy, memset */
 47 | #include <stdio.h>      /* printf (debug) */
 48 | #include "bitstream.h"
 49 | #define FSE_STATIC_LINKING_ONLY   /* FSE_optimalTableLog_internal */
 50 | #include "fse.h"        /* header compression */
 51 | #define HUF_STATIC_LINKING_ONLY
 52 | #include "huf.h"
 53 | 
 54 | 
 55 | /* **************************************************************
 56 | *  Error Management
 57 | ****************************************************************/
 58 | #define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
 59 | 
 60 | 
 61 | /* **************************************************************
 62 | *  Utils
 63 | ****************************************************************/
 64 | unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
 65 | {
 66 |     return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
 67 | }
 68 | 
 69 | 
 70 | /* *******************************************************
 71 | *  HUF : Huffman block compression
 72 | *********************************************************/
 73 | struct HUF_CElt_s {
 74 |   U16  val;
 75 |   BYTE nbBits;
 76 | };   /* typedef'd to HUF_CElt within "huf.h" */
 77 | 
 78 | typedef struct nodeElt_s {
 79 |     U32 count;
 80 |     U16 parent;
 81 |     BYTE byte;
 82 |     BYTE nbBits;
 83 | } nodeElt;
 84 | 
 85 | /*! HUF_writeCTable() :
 86 |     `CTable` : huffman tree to save, using huf representation.
 87 |     @return : size of saved CTable */
 88 | size_t HUF_writeCTable (void* dst, size_t maxDstSize,
 89 |                         const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
 90 | {
 91 |     BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
 92 |     BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
 93 |     BYTE* op = (BYTE*)dst;
 94 |     U32 n;
 95 | 
 96 |      /* check conditions */
 97 |     if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
 98 | 
 99 |     /* convert to weight */
100 |     bitsToWeight[0] = 0;
101 |     for (n=1; n<huffLog+1; n++)
102 |         bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
103 |     for (n=0; n<maxSymbolValue; n++)
104 |         huffWeight[n] = bitsToWeight[CTable[n].nbBits];
105 | 
106 |     {   size_t const size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue);
107 |         if (FSE_isError(size)) return size;
108 |         if ((size>1) & (size < maxSymbolValue/2)) {   /* FSE compressed */
109 |             op[0] = (BYTE)size;
110 |             return size+1;
111 |         }
112 |     }
113 | 
114 |     /* raw values */
115 |     if (maxSymbolValue > (256-128)) return ERROR(GENERIC);   /* should not happen */
116 |     if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall);   /* not enough space within dst buffer */
117 |     op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
118 |     huffWeight[maxSymbolValue] = 0;   /* to be sure it doesn't cause issue in final combination */
119 |     for (n=0; n<maxSymbolValue; n+=2)
120 |         op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
121 |     return ((maxSymbolValue+1)/2) + 1;
122 | 
123 | }
124 | 
125 | 
126 | size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
127 | {
128 |     BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
129 |     U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];   /* large enough for values from 0 to 16 */
130 |     U32 tableLog = 0;
131 |     size_t readSize;
132 |     U32 nbSymbols = 0;
133 |     /*memset(huffWeight, 0, sizeof(huffWeight));*/   /* is not necessary, even though some analyzer complain ... */
134 | 
135 |     /* get symbol weights */
136 |     readSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
137 |     if (HUF_isError(readSize)) return readSize;
138 | 
139 |     /* check result */
140 |     if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
141 |     if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
142 | 
143 |     /* Prepare base value per rank */
144 |     {   U32 n, nextRankStart = 0;
145 |         for (n=1; n<=tableLog; n++) {
146 |             U32 current = nextRankStart;
147 |             nextRankStart += (rankVal[n] << (n-1));
148 |             rankVal[n] = current;
149 |     }   }
150 | 
151 |     /* fill nbBits */
152 |     {   U32 n; for (n=0; n<nbSymbols; n++) {
153 |             const U32 w = huffWeight[n];
154 |             CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
155 |     }   }
156 | 
157 |     /* fill val */
158 |     {   U16 nbPerRank[HUF_TABLELOG_MAX+2]  = {0};  /* support w=0=>n=tableLog+1 */
159 |         U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
160 |         { U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
161 |         /* determine stating value per rank */
162 |         valPerRank[tableLog+1] = 0;   /* for w==0 */
163 |         {   U16 min = 0;
164 |             U32 n; for (n=tableLog; n>0; n--) {  /* start at n=tablelog <-> w=1 */
165 |                 valPerRank[n] = min;     /* get starting value within each rank */
166 |                 min += nbPerRank[n];
167 |                 min >>= 1;
168 |         }   }
169 |         /* assign value within rank, symbol order */
170 |         { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; }
171 |     }
172 | 
173 |     return readSize;
174 | }
175 | 
176 | 
177 | static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
178 | {
179 |     const U32 largestBits = huffNode[lastNonNull].nbBits;
180 |     if (largestBits <= maxNbBits) return largestBits;   /* early exit : no elt > maxNbBits */
181 | 
182 |     /* there are several too large elements (at least >= 2) */
183 |     {   int totalCost = 0;
184 |         const U32 baseCost = 1 << (largestBits - maxNbBits);
185 |         U32 n = lastNonNull;
186 | 
187 |         while (huffNode[n].nbBits > maxNbBits) {
188 |             totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
189 |             huffNode[n].nbBits = (BYTE)maxNbBits;
190 |             n --;
191 |         }  /* n stops at huffNode[n].nbBits <= maxNbBits */
192 |         while (huffNode[n].nbBits == maxNbBits) n--;   /* n end at index of smallest symbol using < maxNbBits */
193 | 
194 |         /* renorm totalCost */
195 |         totalCost >>= (largestBits - maxNbBits);  /* note : totalCost is necessarily a multiple of baseCost */
196 | 
197 |         /* repay normalized cost */
198 |         {   U32 const noSymbol = 0xF0F0F0F0;
199 |             U32 rankLast[HUF_TABLELOG_MAX+2];
200 |             int pos;
201 | 
202 |             /* Get pos of last (smallest) symbol per rank */
203 |             memset(rankLast, 0xF0, sizeof(rankLast));
204 |             {   U32 currentNbBits = maxNbBits;
205 |                 for (pos=n ; pos >= 0; pos--) {
206 |                     if (huffNode[pos].nbBits >= currentNbBits) continue;
207 |                     currentNbBits = huffNode[pos].nbBits;   /* < maxNbBits */
208 |                     rankLast[maxNbBits-currentNbBits] = pos;
209 |             }   }
210 | 
211 |             while (totalCost > 0) {
212 |                 U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
213 |                 for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
214 |                     U32 highPos = rankLast[nBitsToDecrease];
215 |                     U32 lowPos = rankLast[nBitsToDecrease-1];
216 |                     if (highPos == noSymbol) continue;
217 |                     if (lowPos == noSymbol) break;
218 |                     {   U32 const highTotal = huffNode[highPos].count;
219 |                         U32 const lowTotal = 2 * huffNode[lowPos].count;
220 |                         if (highTotal <= lowTotal) break;
221 |                 }   }
222 |                 /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
223 |                 while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol))  /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
224 |                     nBitsToDecrease ++;
225 |                 totalCost -= 1 << (nBitsToDecrease-1);
226 |                 if (rankLast[nBitsToDecrease-1] == noSymbol)
227 |                     rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease];   /* this rank is no longer empty */
228 |                 huffNode[rankLast[nBitsToDecrease]].nbBits ++;
229 |                 if (rankLast[nBitsToDecrease] == 0)    /* special case, reached largest symbol */
230 |                     rankLast[nBitsToDecrease] = noSymbol;
231 |                 else {
232 |                     rankLast[nBitsToDecrease]--;
233 |                     if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
234 |                         rankLast[nBitsToDecrease] = noSymbol;   /* this rank is now empty */
235 |             }   }   /* while (totalCost > 0) */
236 | 
237 |             while (totalCost < 0) {  /* Sometimes, cost correction overshoot */
238 |                 if (rankLast[1] == noSymbol) {  /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
239 |                     while (huffNode[n].nbBits == maxNbBits) n--;
240 |                     huffNode[n+1].nbBits--;
241 |                     rankLast[1] = n+1;
242 |                     totalCost++;
243 |                     continue;
244 |                 }
245 |                 huffNode[ rankLast[1] + 1 ].nbBits--;
246 |                 rankLast[1]++;
247 |                 totalCost ++;
248 |     }   }   }   /* there are several too large elements (at least >= 2) */
249 | 
250 |     return maxNbBits;
251 | }
252 | 
253 | 
254 | typedef struct {
255 |     U32 base;
256 |     U32 current;
257 | } rankPos;
258 | 
259 | static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
260 | {
261 |     rankPos rank[32];
262 |     U32 n;
263 | 
264 |     memset(rank, 0, sizeof(rank));
265 |     for (n=0; n<=maxSymbolValue; n++) {
266 |         U32 r = BIT_highbit32(count[n] + 1);
267 |         rank[r].base ++;
268 |     }
269 |     for (n=30; n>0; n--) rank[n-1].base += rank[n].base;
270 |     for (n=0; n<32; n++) rank[n].current = rank[n].base;
271 |     for (n=0; n<=maxSymbolValue; n++) {
272 |         U32 const c = count[n];
273 |         U32 const r = BIT_highbit32(c+1) + 1;
274 |         U32 pos = rank[r].current++;
275 |         while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--;
276 |         huffNode[pos].count = c;
277 |         huffNode[pos].byte  = (BYTE)n;
278 |     }
279 | }
280 | 
281 | 
282 | #define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
283 | size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
284 | {
285 |     nodeElt huffNode0[2*HUF_SYMBOLVALUE_MAX+1 +1];
286 |     nodeElt* huffNode = huffNode0 + 1;
287 |     U32 n, nonNullRank;
288 |     int lowS, lowN;
289 |     U16 nodeNb = STARTNODE;
290 |     U32 nodeRoot;
291 | 
292 |     /* safety checks */
293 |     if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
294 |     if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
295 |     memset(huffNode0, 0, sizeof(huffNode0));
296 | 
297 |     /* sort, decreasing order */
298 |     HUF_sort(huffNode, count, maxSymbolValue);
299 | 
300 |     /* init for parents */
301 |     nonNullRank = maxSymbolValue;
302 |     while(huffNode[nonNullRank].count == 0) nonNullRank--;
303 |     lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
304 |     huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
305 |     huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
306 |     nodeNb++; lowS-=2;
307 |     for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
308 |     huffNode0[0].count = (U32)(1U<<31);
309 | 
310 |     /* create parents */
311 |     while (nodeNb <= nodeRoot) {
312 |         U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
313 |         U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
314 |         huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
315 |         huffNode[n1].parent = huffNode[n2].parent = nodeNb;
316 |         nodeNb++;
317 |     }
318 | 
319 |     /* distribute weights (unlimited tree height) */
320 |     huffNode[nodeRoot].nbBits = 0;
321 |     for (n=nodeRoot-1; n>=STARTNODE; n--)
322 |         huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
323 |     for (n=0; n<=nonNullRank; n++)
324 |         huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
325 | 
326 |     /* enforce maxTableLog */
327 |     maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
328 | 
329 |     /* fill result into tree (val, nbBits) */
330 |     {   U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
331 |         U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
332 |         if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC);   /* check fit into table */
333 |         for (n=0; n<=nonNullRank; n++)
334 |             nbPerRank[huffNode[n].nbBits]++;
335 |         /* determine stating value per rank */
336 |         {   U16 min = 0;
337 |             for (n=maxNbBits; n>0; n--) {
338 |                 valPerRank[n] = min;      /* get starting value within each rank */
339 |                 min += nbPerRank[n];
340 |                 min >>= 1;
341 |         }   }
342 |         for (n=0; n<=maxSymbolValue; n++)
343 |             tree[huffNode[n].byte].nbBits = huffNode[n].nbBits;   /* push nbBits per symbol, symbol order */
344 |         for (n=0; n<=maxSymbolValue; n++)
345 |             tree[n].val = valPerRank[tree[n].nbBits]++;   /* assign value within rank, symbol order */
346 |     }
347 | 
348 |     return maxNbBits;
349 | }
350 | 
351 | static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
352 | {
353 |     BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
354 | }
355 | 
356 | size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
357 | 
358 | #define HUF_FLUSHBITS(s)  (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
359 | 
360 | #define HUF_FLUSHBITS_1(stream) \
361 |     if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream)
362 | 
363 | #define HUF_FLUSHBITS_2(stream) \
364 |     if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream)
365 | 
366 | size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
367 | {
368 |     const BYTE* ip = (const BYTE*) src;
369 |     BYTE* const ostart = (BYTE*)dst;
370 |     BYTE* const oend = ostart + dstSize;
371 |     BYTE* op = ostart;
372 |     size_t n;
373 |     const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
374 |     BIT_CStream_t bitC;
375 | 
376 |     /* init */
377 |     if (dstSize < 8) return 0;   /* not enough space to compress */
378 |     { size_t const errorCode = BIT_initCStream(&bitC, op, oend-op);
379 |       if (HUF_isError(errorCode)) return 0; }
380 | 
381 |     n = srcSize & ~3;  /* join to mod 4 */
382 |     switch (srcSize & 3)
383 |     {
384 |         case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
385 |                  HUF_FLUSHBITS_2(&bitC);
386 |         case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
387 |                  HUF_FLUSHBITS_1(&bitC);
388 |         case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
389 |                  HUF_FLUSHBITS(&bitC);
390 |         case 0 :
391 |         default: ;
392 |     }
393 | 
394 |     for (; n>0; n-=4) {  /* note : n&3==0 at this stage */
395 |         HUF_encodeSymbol(&bitC, ip[n- 1], CTable);
396 |         HUF_FLUSHBITS_1(&bitC);
397 |         HUF_encodeSymbol(&bitC, ip[n- 2], CTable);
398 |         HUF_FLUSHBITS_2(&bitC);
399 |         HUF_encodeSymbol(&bitC, ip[n- 3], CTable);
400 |         HUF_FLUSHBITS_1(&bitC);
401 |         HUF_encodeSymbol(&bitC, ip[n- 4], CTable);
402 |         HUF_FLUSHBITS(&bitC);
403 |     }
404 | 
405 |     return BIT_closeCStream(&bitC);
406 | }
407 | 
408 | 
409 | size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
410 | {
411 |     size_t const segmentSize = (srcSize+3)/4;   /* first 3 segments */
412 |     const BYTE* ip = (const BYTE*) src;
413 |     const BYTE* const iend = ip + srcSize;
414 |     BYTE* const ostart = (BYTE*) dst;
415 |     BYTE* const oend = ostart + dstSize;
416 |     BYTE* op = ostart;
417 | 
418 |     if (dstSize < 6 + 1 + 1 + 1 + 8) return 0;   /* minimum space to compress successfully */
419 |     if (srcSize < 12) return 0;   /* no saving possible : too small input */
420 |     op += 6;   /* jumpTable */
421 | 
422 |     {   size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
423 |         if (HUF_isError(cSize)) return cSize;
424 |         if (cSize==0) return 0;
425 |         MEM_writeLE16(ostart, (U16)cSize);
426 |         op += cSize;
427 |     }
428 | 
429 |     ip += segmentSize;
430 |     {   size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
431 |         if (HUF_isError(cSize)) return cSize;
432 |         if (cSize==0) return 0;
433 |         MEM_writeLE16(ostart+2, (U16)cSize);
434 |         op += cSize;
435 |     }
436 | 
437 |     ip += segmentSize;
438 |     {   size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
439 |         if (HUF_isError(cSize)) return cSize;
440 |         if (cSize==0) return 0;
441 |         MEM_writeLE16(ostart+4, (U16)cSize);
442 |         op += cSize;
443 |     }
444 | 
445 |     ip += segmentSize;
446 |     {   size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
447 |         if (HUF_isError(cSize)) return cSize;
448 |         if (cSize==0) return 0;
449 |         op += cSize;
450 |     }
451 | 
452 |     return op-ostart;
453 | }
454 | 
455 | 
456 | static size_t HUF_compress_internal (
457 |                 void* dst, size_t dstSize,
458 |                 const void* src, size_t srcSize,
459 |                 unsigned maxSymbolValue, unsigned huffLog,
460 |                 unsigned singleStream)
461 | {
462 |     BYTE* const ostart = (BYTE*)dst;
463 |     BYTE* const oend = ostart + dstSize;
464 |     BYTE* op = ostart;
465 | 
466 |     U32 count[HUF_SYMBOLVALUE_MAX+1];
467 |     HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
468 | 
469 |     /* checks & inits */
470 |     if (!srcSize) return 0;  /* Uncompressed (note : 1 means rle, so first byte must be correct) */
471 |     if (!dstSize) return 0;  /* cannot fit within dst budget */
472 |     if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong);   /* current block size limit */
473 |     if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
474 |     if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
475 |     if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
476 | 
477 |     /* Scan input and build symbol stats */
478 |     {   size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
479 |         if (HUF_isError(largest)) return largest;
480 |         if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; }   /* single symbol, rle */
481 |         if (largest <= (srcSize >> 7)+1) return 0;   /* Fast heuristic : not compressible enough */
482 |     }
483 | 
484 |     /* Build Huffman Tree */
485 |     huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
486 |     {   size_t const maxBits = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
487 |         if (HUF_isError(maxBits)) return maxBits;
488 |         huffLog = (U32)maxBits;
489 |     }
490 | 
491 |     /* Write table description header */
492 |     {   size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
493 |         if (HUF_isError(hSize)) return hSize;
494 |         if (hSize + 12 >= srcSize) return 0;   /* not useful to try compression */
495 |         op += hSize;
496 |     }
497 | 
498 |     /* Compress */
499 |     {   size_t const cSize = (singleStream) ?
500 |                             HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) :   /* single segment */
501 |                             HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
502 |         if (HUF_isError(cSize)) return cSize;
503 |         if (cSize==0) return 0;   /* uncompressible */
504 |         op += cSize;
505 |     }
506 | 
507 |     /* check compressibility */
508 |     if ((size_t)(op-ostart) >= srcSize-1)
509 |         return 0;
510 | 
511 |     return op-ostart;
512 | }
513 | 
514 | 
515 | size_t HUF_compress1X (void* dst, size_t dstSize,
516 |                  const void* src, size_t srcSize,
517 |                  unsigned maxSymbolValue, unsigned huffLog)
518 | {
519 |     return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1);
520 | }
521 | 
522 | size_t HUF_compress2 (void* dst, size_t dstSize,
523 |                 const void* src, size_t srcSize,
524 |                 unsigned maxSymbolValue, unsigned huffLog)
525 | {
526 |     return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
527 | }
528 | 
529 | 
530 | size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
531 | {
532 |     return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);
533 | }
534 | 


--------------------------------------------------------------------------------
/zstd/mem.h:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  3 |  * All rights reserved.
  4 |  *
  5 |  * This source code is licensed under the BSD-style license found in the
  6 |  * LICENSE file in the root directory of this source tree. An additional grant
  7 |  * of patent rights can be found in the PATENTS file in the same directory.
  8 |  */
  9 | 
 10 | #ifndef MEM_H_MODULE
 11 | #define MEM_H_MODULE
 12 | 
 13 | #if defined (__cplusplus)
 14 | extern "C" {
 15 | #endif
 16 | 
 17 | /*-****************************************
 18 | *  Dependencies
 19 | ******************************************/
 20 | #include <stddef.h>     /* size_t, ptrdiff_t */
 21 | #include <string.h>     /* memcpy */
 22 | 
 23 | 
 24 | /*-****************************************
 25 | *  Compiler specifics
 26 | ******************************************/
 27 | #if defined(_MSC_VER)   /* Visual Studio */
 28 | #   include <stdlib.h>  /* _byteswap_ulong */
 29 | #   include <intrin.h>  /* _byteswap_* */
 30 | #endif
 31 | #if defined(__GNUC__)
 32 | #  define MEM_STATIC static __inline __attribute__((unused))
 33 | #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 34 | #  define MEM_STATIC static inline
 35 | #elif defined(_MSC_VER)
 36 | #  define MEM_STATIC static __inline
 37 | #else
 38 | #  define MEM_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
 39 | #endif
 40 | 
 41 | /* code only tested on 32 and 64 bits systems */
 42 | #define MEM_STATIC_ASSERT(c)   { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
 43 | MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
 44 | 
 45 | 
 46 | /*-**************************************************************
 47 | *  Basic Types
 48 | *****************************************************************/
 49 | #if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
 50 | # include <stdint.h>
 51 |   typedef  uint8_t BYTE;
 52 |   typedef uint16_t U16;
 53 |   typedef  int16_t S16;
 54 |   typedef uint32_t U32;
 55 |   typedef  int32_t S32;
 56 |   typedef uint64_t U64;
 57 |   typedef  int64_t S64;
 58 | #else
 59 |   typedef unsigned char       BYTE;
 60 |   typedef unsigned short      U16;
 61 |   typedef   signed short      S16;
 62 |   typedef unsigned int        U32;
 63 |   typedef   signed int        S32;
 64 |   typedef unsigned long long  U64;
 65 |   typedef   signed long long  S64;
 66 | #endif
 67 | 
 68 | 
 69 | /*-**************************************************************
 70 | *  Memory I/O
 71 | *****************************************************************/
 72 | /* MEM_FORCE_MEMORY_ACCESS :
 73 |  * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
 74 |  * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
 75 |  * The below switch allow to select different access method for improved performance.
 76 |  * Method 0 (default) : use `memcpy()`. Safe and portable.
 77 |  * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
 78 |  *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
 79 |  * Method 2 : direct access. This method is portable but violate C standard.
 80 |  *            It can generate buggy code on targets depending on alignment.
 81 |  *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
 82 |  * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
 83 |  * Prefer these methods in priority order (0 > 1 > 2)
 84 |  */
 85 | #ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
 86 | #  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
 87 | #    define MEM_FORCE_MEMORY_ACCESS 2
 88 | #  elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \
 89 |   (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
 90 | #    define MEM_FORCE_MEMORY_ACCESS 1
 91 | #  endif
 92 | #endif
 93 | 
 94 | MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
 95 | MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
 96 | 
 97 | MEM_STATIC unsigned MEM_isLittleEndian(void)
 98 | {
 99 |     const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental  */
100 |     return one.c[0];
101 | }
102 | 
103 | #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
104 | 
105 | /* violates C standard, by lying on structure alignment.
106 | Only use if no other choice to achieve best performance on target platform */
107 | MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
108 | MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
109 | MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
110 | MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
111 | 
112 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
113 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
114 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
115 | 
116 | #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
117 | 
118 | /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
119 | /* currently only defined for gcc and icc */
120 | #if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
121 | 	__pragma( pack(push, 1) )
122 |     typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign;
123 |     __pragma( pack(pop) )
124 | #else
125 |     typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
126 | #endif
127 | 
128 | MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
129 | MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
130 | MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
131 | MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
132 | 
133 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
134 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
135 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
136 | 
137 | #else
138 | 
139 | /* default method, safe and standard.
140 |    can sometimes prove slower */
141 | 
142 | MEM_STATIC U16 MEM_read16(const void* memPtr)
143 | {
144 |     U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
145 | }
146 | 
147 | MEM_STATIC U32 MEM_read32(const void* memPtr)
148 | {
149 |     U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
150 | }
151 | 
152 | MEM_STATIC U64 MEM_read64(const void* memPtr)
153 | {
154 |     U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
155 | }
156 | 
157 | MEM_STATIC size_t MEM_readST(const void* memPtr)
158 | {
159 |     size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
160 | }
161 | 
162 | MEM_STATIC void MEM_write16(void* memPtr, U16 value)
163 | {
164 |     memcpy(memPtr, &value, sizeof(value));
165 | }
166 | 
167 | MEM_STATIC void MEM_write32(void* memPtr, U32 value)
168 | {
169 |     memcpy(memPtr, &value, sizeof(value));
170 | }
171 | 
172 | MEM_STATIC void MEM_write64(void* memPtr, U64 value)
173 | {
174 |     memcpy(memPtr, &value, sizeof(value));
175 | }
176 | 
177 | #endif /* MEM_FORCE_MEMORY_ACCESS */
178 | 
179 | MEM_STATIC U32 MEM_swap32(U32 in)
180 | {
181 | #if defined(_MSC_VER)     /* Visual Studio */
182 |     return _byteswap_ulong(in);
183 | #elif defined (__GNUC__)
184 |     return __builtin_bswap32(in);
185 | #else
186 |     return  ((in << 24) & 0xff000000 ) |
187 |             ((in <<  8) & 0x00ff0000 ) |
188 |             ((in >>  8) & 0x0000ff00 ) |
189 |             ((in >> 24) & 0x000000ff );
190 | #endif
191 | }
192 | 
193 | MEM_STATIC U64 MEM_swap64(U64 in)
194 | {
195 | #if defined(_MSC_VER)     /* Visual Studio */
196 |     return _byteswap_uint64(in);
197 | #elif defined (__GNUC__)
198 |     return __builtin_bswap64(in);
199 | #else
200 |     return  ((in << 56) & 0xff00000000000000ULL) |
201 |             ((in << 40) & 0x00ff000000000000ULL) |
202 |             ((in << 24) & 0x0000ff0000000000ULL) |
203 |             ((in << 8)  & 0x000000ff00000000ULL) |
204 |             ((in >> 8)  & 0x00000000ff000000ULL) |
205 |             ((in >> 24) & 0x0000000000ff0000ULL) |
206 |             ((in >> 40) & 0x000000000000ff00ULL) |
207 |             ((in >> 56) & 0x00000000000000ffULL);
208 | #endif
209 | }
210 | 
211 | MEM_STATIC size_t MEM_swapST(size_t in)
212 | {
213 |     if (MEM_32bits())
214 |         return (size_t)MEM_swap32((U32)in);
215 |     else
216 |         return (size_t)MEM_swap64((U64)in);
217 | }
218 | 
219 | /*=== Little endian r/w ===*/
220 | 
221 | MEM_STATIC U16 MEM_readLE16(const void* memPtr)
222 | {
223 |     if (MEM_isLittleEndian())
224 |         return MEM_read16(memPtr);
225 |     else {
226 |         const BYTE* p = (const BYTE*)memPtr;
227 |         return (U16)(p[0] + (p[1]<<8));
228 |     }
229 | }
230 | 
231 | MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
232 | {
233 |     if (MEM_isLittleEndian()) {
234 |         MEM_write16(memPtr, val);
235 |     } else {
236 |         BYTE* p = (BYTE*)memPtr;
237 |         p[0] = (BYTE)val;
238 |         p[1] = (BYTE)(val>>8);
239 |     }
240 | }
241 | 
242 | MEM_STATIC U32 MEM_readLE24(const void* memPtr)
243 | {
244 |     return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
245 | }
246 | 
247 | MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
248 | {
249 |     MEM_writeLE16(memPtr, (U16)val);
250 |     ((BYTE*)memPtr)[2] = (BYTE)(val>>16);
251 | }
252 | 
253 | MEM_STATIC U32 MEM_readLE32(const void* memPtr)
254 | {
255 |     if (MEM_isLittleEndian())
256 |         return MEM_read32(memPtr);
257 |     else
258 |         return MEM_swap32(MEM_read32(memPtr));
259 | }
260 | 
261 | MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
262 | {
263 |     if (MEM_isLittleEndian())
264 |         MEM_write32(memPtr, val32);
265 |     else
266 |         MEM_write32(memPtr, MEM_swap32(val32));
267 | }
268 | 
269 | MEM_STATIC U64 MEM_readLE64(const void* memPtr)
270 | {
271 |     if (MEM_isLittleEndian())
272 |         return MEM_read64(memPtr);
273 |     else
274 |         return MEM_swap64(MEM_read64(memPtr));
275 | }
276 | 
277 | MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
278 | {
279 |     if (MEM_isLittleEndian())
280 |         MEM_write64(memPtr, val64);
281 |     else
282 |         MEM_write64(memPtr, MEM_swap64(val64));
283 | }
284 | 
285 | MEM_STATIC size_t MEM_readLEST(const void* memPtr)
286 | {
287 |     if (MEM_32bits())
288 |         return (size_t)MEM_readLE32(memPtr);
289 |     else
290 |         return (size_t)MEM_readLE64(memPtr);
291 | }
292 | 
293 | MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
294 | {
295 |     if (MEM_32bits())
296 |         MEM_writeLE32(memPtr, (U32)val);
297 |     else
298 |         MEM_writeLE64(memPtr, (U64)val);
299 | }
300 | 
301 | /*=== Big endian r/w ===*/
302 | 
303 | MEM_STATIC U32 MEM_readBE32(const void* memPtr)
304 | {
305 |     if (MEM_isLittleEndian())
306 |         return MEM_swap32(MEM_read32(memPtr));
307 |     else
308 |         return MEM_read32(memPtr);
309 | }
310 | 
311 | MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
312 | {
313 |     if (MEM_isLittleEndian())
314 |         MEM_write32(memPtr, MEM_swap32(val32));
315 |     else
316 |         MEM_write32(memPtr, val32);
317 | }
318 | 
319 | MEM_STATIC U64 MEM_readBE64(const void* memPtr)
320 | {
321 |     if (MEM_isLittleEndian())
322 |         return MEM_swap64(MEM_read64(memPtr));
323 |     else
324 |         return MEM_read64(memPtr);
325 | }
326 | 
327 | MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
328 | {
329 |     if (MEM_isLittleEndian())
330 |         MEM_write64(memPtr, MEM_swap64(val64));
331 |     else
332 |         MEM_write64(memPtr, val64);
333 | }
334 | 
335 | MEM_STATIC size_t MEM_readBEST(const void* memPtr)
336 | {
337 |     if (MEM_32bits())
338 |         return (size_t)MEM_readBE32(memPtr);
339 |     else
340 |         return (size_t)MEM_readBE64(memPtr);
341 | }
342 | 
343 | MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
344 | {
345 |     if (MEM_32bits())
346 |         MEM_writeBE32(memPtr, (U32)val);
347 |     else
348 |         MEM_writeBE64(memPtr, (U64)val);
349 | }
350 | 
351 | 
352 | /* function safe only for comparisons */
353 | MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
354 | {
355 |     switch (length)
356 |     {
357 |     default :
358 |     case 4 : return MEM_read32(memPtr);
359 |     case 3 : if (MEM_isLittleEndian())
360 |                 return MEM_read32(memPtr)<<8;
361 |              else
362 |                 return MEM_read32(memPtr)>>8;
363 |     }
364 | }
365 | 
366 | #if defined (__cplusplus)
367 | }
368 | #endif
369 | 
370 | #endif /* MEM_H_MODULE */
371 | 


--------------------------------------------------------------------------------
/zstd/xxhash.h:
--------------------------------------------------------------------------------
  1 | /*
  2 |    xxHash - Extremely Fast Hash algorithm
  3 |    Header File
  4 |    Copyright (C) 2012-2016, Yann Collet.
  5 | 
  6 |    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
  7 | 
  8 |    Redistribution and use in source and binary forms, with or without
  9 |    modification, are permitted provided that the following conditions are
 10 |    met:
 11 | 
 12 |        * Redistributions of source code must retain the above copyright
 13 |    notice, this list of conditions and the following disclaimer.
 14 |        * Redistributions in binary form must reproduce the above
 15 |    copyright notice, this list of conditions and the following disclaimer
 16 |    in the documentation and/or other materials provided with the
 17 |    distribution.
 18 | 
 19 |    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 20 |    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 21 |    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 22 |    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 23 |    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 24 |    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 25 |    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 26 |    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 27 |    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 28 |    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 29 |    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 30 | 
 31 |    You can contact the author at :
 32 |    - xxHash source repository : https://github.com/Cyan4973/xxHash
 33 | */
 34 | 
 35 | /* Notice extracted from xxHash homepage :
 36 | 
 37 | xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
 38 | It also successfully passes all tests from the SMHasher suite.
 39 | 
 40 | Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
 41 | 
 42 | Name            Speed       Q.Score   Author
 43 | xxHash          5.4 GB/s     10
 44 | CrapWow         3.2 GB/s      2       Andrew
 45 | MumurHash 3a    2.7 GB/s     10       Austin Appleby
 46 | SpookyHash      2.0 GB/s     10       Bob Jenkins
 47 | SBox            1.4 GB/s      9       Bret Mulvey
 48 | Lookup3         1.2 GB/s      9       Bob Jenkins
 49 | SuperFastHash   1.2 GB/s      1       Paul Hsieh
 50 | CityHash64      1.05 GB/s    10       Pike & Alakuijala
 51 | FNV             0.55 GB/s     5       Fowler, Noll, Vo
 52 | CRC32           0.43 GB/s     9
 53 | MD5-32          0.33 GB/s    10       Ronald L. Rivest
 54 | SHA1-32         0.28 GB/s    10
 55 | 
 56 | Q.Score is a measure of quality of the hash function.
 57 | It depends on successfully passing SMHasher test set.
 58 | 10 is a perfect score.
 59 | 
 60 | A 64-bits version, named XXH64, is available since r35.
 61 | It offers much better speed, but for 64-bits applications only.
 62 | Name     Speed on 64 bits    Speed on 32 bits
 63 | XXH64       13.8 GB/s            1.9 GB/s
 64 | XXH32        6.8 GB/s            6.0 GB/s
 65 | */
 66 | 
 67 | #ifndef XXHASH_H_5627135585666179
 68 | #define XXHASH_H_5627135585666179 1
 69 | 
 70 | #if defined (__cplusplus)
 71 | extern "C" {
 72 | #endif
 73 | 
 74 | #ifndef XXH_NAMESPACE
 75 | #  define XXH_NAMESPACE ZSTD_  /* Zstandard specific */
 76 | #endif
 77 | 
 78 | 
 79 | /* ****************************
 80 | *  Definitions
 81 | ******************************/
 82 | #include <stddef.h>   /* size_t */
 83 | typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
 84 | 
 85 | 
 86 | /* ****************************
 87 | *  API modifier
 88 | ******************************/
 89 | /** XXH_PRIVATE_API
 90 | *   This is useful if you want to include xxhash functions in `static` mode
 91 | *   in order to inline them, and remove their symbol from the public list.
 92 | *   Methodology :
 93 | *     #define XXH_PRIVATE_API
 94 | *     #include "xxhash.h"
 95 | *   `xxhash.c` is automatically included.
 96 | *   It's not useful to compile and link it as a separate module anymore.
 97 | */
 98 | #ifdef XXH_PRIVATE_API
 99 | #  ifndef XXH_STATIC_LINKING_ONLY
100 | #    define XXH_STATIC_LINKING_ONLY
101 | #  endif
102 | #  if defined(__GNUC__)
103 | #    define XXH_PUBLIC_API static __inline __attribute__((unused))
104 | #  elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
105 | #    define XXH_PUBLIC_API static inline
106 | #  elif defined(_MSC_VER)
107 | #    define XXH_PUBLIC_API static __inline
108 | #  else
109 | #    define XXH_PUBLIC_API static   /* this version may generate warnings for unused static functions; disable the relevant warning */
110 | #  endif
111 | #else
112 | #  define XXH_PUBLIC_API   /* do nothing */
113 | #endif /* XXH_PRIVATE_API */
114 | 
115 | /*!XXH_NAMESPACE, aka Namespace Emulation :
116 | 
117 | If you want to include _and expose_ xxHash functions from within your own library,
118 | but also want to avoid symbol collisions with another library which also includes xxHash,
119 | 
120 | you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
121 | with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values).
122 | 
123 | Note that no change is required within the calling program as long as it includes `xxhash.h` :
124 | regular symbol name will be automatically translated by this header.
125 | */
126 | #ifdef XXH_NAMESPACE
127 | #  define XXH_CAT(A,B) A##B
128 | #  define XXH_NAME2(A,B) XXH_CAT(A,B)
129 | #  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
130 | #  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
131 | #  define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
132 | #  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
133 | #  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
134 | #  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
135 | #  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
136 | #  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
137 | #  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
138 | #  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
139 | #  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
140 | #  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
141 | #  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
142 | #  define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
143 | #  define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
144 | #  define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
145 | #  define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
146 | #  define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
147 | #  define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
148 | #endif
149 | 
150 | 
151 | /* *************************************
152 | *  Version
153 | ***************************************/
154 | #define XXH_VERSION_MAJOR    0
155 | #define XXH_VERSION_MINOR    6
156 | #define XXH_VERSION_RELEASE  2
157 | #define XXH_VERSION_NUMBER  (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
158 | XXH_PUBLIC_API unsigned XXH_versionNumber (void);
159 | 
160 | 
161 | /* ****************************
162 | *  Simple Hash Functions
163 | ******************************/
164 | typedef unsigned int       XXH32_hash_t;
165 | typedef unsigned long long XXH64_hash_t;
166 | 
167 | XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
168 | XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
169 | 
170 | /*!
171 | XXH32() :
172 |     Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
173 |     The memory between input & input+length must be valid (allocated and read-accessible).
174 |     "seed" can be used to alter the result predictably.
175 |     Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
176 | XXH64() :
177 |     Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
178 |     "seed" can be used to alter the result predictably.
179 |     This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
180 | */
181 | 
182 | 
183 | /* ****************************
184 | *  Streaming Hash Functions
185 | ******************************/
186 | typedef struct XXH32_state_s XXH32_state_t;   /* incomplete type */
187 | typedef struct XXH64_state_s XXH64_state_t;   /* incomplete type */
188 | 
189 | /*! State allocation, compatible with dynamic libraries */
190 | 
191 | XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
192 | XXH_PUBLIC_API XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
193 | 
194 | XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
195 | XXH_PUBLIC_API XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
196 | 
197 | 
198 | /* hash streaming */
199 | 
200 | XXH_PUBLIC_API XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, unsigned int seed);
201 | XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
202 | XXH_PUBLIC_API XXH32_hash_t  XXH32_digest (const XXH32_state_t* statePtr);
203 | 
204 | XXH_PUBLIC_API XXH_errorcode XXH64_reset  (XXH64_state_t* statePtr, unsigned long long seed);
205 | XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
206 | XXH_PUBLIC_API XXH64_hash_t  XXH64_digest (const XXH64_state_t* statePtr);
207 | 
208 | /*
209 | These functions generate the xxHash of an input provided in multiple segments.
210 | Note that, for small input, they are slower than single-call functions, due to state management.
211 | For small input, prefer `XXH32()` and `XXH64()` .
212 | 
213 | XXH state must first be allocated, using XXH*_createState() .
214 | 
215 | Start a new hash by initializing state with a seed, using XXH*_reset().
216 | 
217 | Then, feed the hash state by calling XXH*_update() as many times as necessary.
218 | Obviously, input must be allocated and read accessible.
219 | The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
220 | 
221 | Finally, a hash value can be produced anytime, by using XXH*_digest().
222 | This function returns the nn-bits hash as an int or long long.
223 | 
224 | It's still possible to continue inserting input into the hash state after a digest,
225 | and generate some new hashes later on, by calling again XXH*_digest().
226 | 
227 | When done, free XXH state space if it was allocated dynamically.
228 | */
229 | 
230 | 
231 | /* **************************
232 | *  Utils
233 | ****************************/
234 | #if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L))   /* ! C99 */
235 | #  define restrict   /* disable restrict */
236 | #endif
237 | 
238 | XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state);
239 | XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state);
240 | 
241 | 
242 | /* **************************
243 | *  Canonical representation
244 | ****************************/
245 | typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
246 | typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
247 | 
248 | XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
249 | XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
250 | 
251 | XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
252 | XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
253 | 
254 | /* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
255 | *  The canonical representation uses human-readable write convention, aka big-endian (large digits first).
256 | *  These functions allow transformation of hash result into and from its canonical format.
257 | *  This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
258 | */
259 | 
260 | 
261 | #ifdef XXH_STATIC_LINKING_ONLY
262 | 
263 | /* ================================================================================================
264 |    This section contains definitions which are not guaranteed to remain stable.
265 |    They may change in future versions, becoming incompatible with a different version of the library.
266 |    They shall only be used with static linking.
267 |    Never use these definitions in association with dynamic linking !
268 | =================================================================================================== */
269 | 
270 | /* These definitions are only meant to allow allocation of XXH state
271 |    statically, on stack, or in a struct for example.
272 |    Do not use members directly. */
273 | 
274 |    struct XXH32_state_s {
275 |        unsigned total_len_32;
276 |        unsigned large_len;
277 |        unsigned v1;
278 |        unsigned v2;
279 |        unsigned v3;
280 |        unsigned v4;
281 |        unsigned mem32[4];   /* buffer defined as U32 for alignment */
282 |        unsigned memsize;
283 |        unsigned reserved;   /* never read nor write, will be removed in a future version */
284 |    };   /* typedef'd to XXH32_state_t */
285 | 
286 |    struct XXH64_state_s {
287 |        unsigned long long total_len;
288 |        unsigned long long v1;
289 |        unsigned long long v2;
290 |        unsigned long long v3;
291 |        unsigned long long v4;
292 |        unsigned long long mem64[4];   /* buffer defined as U64 for alignment */
293 |        unsigned memsize;
294 |        unsigned reserved[2];          /* never read nor write, will be removed in a future version */
295 |    };   /* typedef'd to XXH64_state_t */
296 | 
297 | 
298 | #  ifdef XXH_PRIVATE_API
299 | #    include "xxhash.c"   /* include xxhash functions as `static`, for inlining */
300 | #  endif
301 | 
302 | #endif /* XXH_STATIC_LINKING_ONLY */
303 | 
304 | 
305 | #if defined (__cplusplus)
306 | }
307 | #endif
308 | 
309 | #endif /* XXHASH_H_5627135585666179 */
310 | 


--------------------------------------------------------------------------------
/zstd/zbuff.h:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  3 |  * All rights reserved.
  4 |  *
  5 |  * This source code is licensed under the BSD-style license found in the
  6 |  * LICENSE file in the root directory of this source tree. An additional grant
  7 |  * of patent rights can be found in the PATENTS file in the same directory.
  8 |  */
  9 | 
 10 | /* ***************************************************************
 11 | *  NOTES/WARNINGS
 12 | *****************************************************************/
 13 | /* The streaming API defined here will soon be deprecated by the
 14 | * new one in 'zstd.h'; consider migrating towards newer streaming
 15 | * API. See 'lib/README.md'.
 16 | *****************************************************************/
 17 | 
 18 | #ifndef ZSTD_BUFFERED_H_23987
 19 | #define ZSTD_BUFFERED_H_23987
 20 | 
 21 | //#if defined (__cplusplus)
 22 | //extern "C" {
 23 | //#endif
 24 | 
 25 | /* *************************************
 26 | *  Dependencies
 27 | ***************************************/
 28 | #include <stddef.h>      /* size_t */
 29 | 
 30 | 
 31 | /* ***************************************************************
 32 | *  Compiler specifics
 33 | *****************************************************************/
 34 | /* ZSTD_DLL_EXPORT :
 35 | *  Enable exporting of functions when building a Windows DLL */
 36 | #if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
 37 | #  define ZSTDLIB_API __declspec(dllexport)
 38 | #else
 39 | #  define ZSTDLIB_API
 40 | #endif
 41 | 
 42 | 
 43 | /* *************************************
 44 | *  Streaming functions
 45 | ***************************************/
 46 | /* This is the easier "buffered" streaming API,
 47 | *  using an internal buffer to lift all restrictions on user-provided buffers
 48 | *  which can be any size, any place, for both input and output.
 49 | *  ZBUFF and ZSTD are 100% interoperable,
 50 | *  frames created by one can be decoded by the other one */
 51 | 
 52 | typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
 53 | ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void);
 54 | ZSTDLIB_API size_t      ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
 55 | 
 56 | ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
 57 | ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
 58 | 
 59 | ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
 60 | ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
 61 | ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
 62 | 
 63 | /*-*************************************************
 64 | *  Streaming compression - howto
 65 | *
 66 | *  A ZBUFF_CCtx object is required to track streaming operation.
 67 | *  Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
 68 | *  ZBUFF_CCtx objects can be reused multiple times.
 69 | *
 70 | *  Start by initializing ZBUF_CCtx.
 71 | *  Use ZBUFF_compressInit() to start a new compression operation.
 72 | *  Use ZBUFF_compressInitDictionary() for a compression which requires a dictionary.
 73 | *
 74 | *  Use ZBUFF_compressContinue() repetitively to consume input stream.
 75 | *  *srcSizePtr and *dstCapacityPtr can be any size.
 76 | *  The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr.
 77 | *  Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
 78 | *  The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst .
 79 | *  @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
 80 | *            or an error code, which can be tested using ZBUFF_isError().
 81 | *
 82 | *  At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush().
 83 | *  The nb of bytes written into `dst` will be reported into *dstCapacityPtr.
 84 | *  Note that the function cannot output more than *dstCapacityPtr,
 85 | *  therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small.
 86 | *  @return : nb of bytes still present into internal buffer (0 if it's empty)
 87 | *            or an error code, which can be tested using ZBUFF_isError().
 88 | *
 89 | *  ZBUFF_compressEnd() instructs to finish a frame.
 90 | *  It will perform a flush and write frame epilogue.
 91 | *  The epilogue is required for decoders to consider a frame completed.
 92 | *  Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
 93 | *  In which case, call again ZBUFF_compressFlush() to complete the flush.
 94 | *  @return : nb of bytes still present into internal buffer (0 if it's empty)
 95 | *            or an error code, which can be tested using ZBUFF_isError().
 96 | *
 97 | *  Hint : _recommended buffer_ sizes (not compulsory) : ZBUFF_recommendedCInSize() / ZBUFF_recommendedCOutSize()
 98 | *  input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, use this value to reduce intermediate stages (better latency)
 99 | *  output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering.
100 | *  By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering.
101 | * **************************************************/
102 | 
103 | 
104 | typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
105 | ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void);
106 | ZSTDLIB_API size_t      ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
107 | 
108 | ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
109 | ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
110 | 
111 | ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
112 |                                             void* dst, size_t* dstCapacityPtr,
113 |                                       const void* src, size_t* srcSizePtr);
114 | 
115 | /*-***************************************************************************
116 | *  Streaming decompression howto
117 | *
118 | *  A ZBUFF_DCtx object is required to track streaming operations.
119 | *  Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
120 | *  Use ZBUFF_decompressInit() to start a new decompression operation,
121 | *   or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
122 | *  Note that ZBUFF_DCtx objects can be re-init multiple times.
123 | *
124 | *  Use ZBUFF_decompressContinue() repetitively to consume your input.
125 | *  *srcSizePtr and *dstCapacityPtr can be any size.
126 | *  The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
127 | *  Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
128 | *  The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
129 | *  @return : 0 when a frame is completely decoded and fully flushed,
130 | *            1 when there is still some data left within internal buffer to flush,
131 | *            >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency),
132 | *            or an error code, which can be tested using ZBUFF_isError().
133 | *
134 | *  Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
135 | *  output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
136 | *  input  : ZBUFF_recommendedDInSize == 128KB + 3;
137 | *           just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
138 | * *******************************************************************************/
139 | 
140 | 
141 | /* *************************************
142 | *  Tool functions
143 | ***************************************/
144 | ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode);
145 | ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode);
146 | 
147 | /** Functions below provide recommended buffer sizes for Compression or Decompression operations.
148 | *   These sizes are just hints, they tend to offer better latency */
149 | ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void);
150 | ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void);
151 | ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void);
152 | ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
153 | 
154 | 
155 | //#ifdef ZBUFF_STATIC_LINKING_ONLY
156 | 
157 | /* ====================================================================================
158 |  * The definitions in this section are considered experimental.
159 |  * They should never be used in association with a dynamic library, as they may change in the future.
160 |  * They are provided for advanced usages.
161 |  * Use them only in association with static linking.
162 |  * ==================================================================================== */
163 | 
164 | /*--- Dependency ---*/
165 | //#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_parameters, ZSTD_customMem */
166 | #include "zstd.h"
167 | 
168 | 
169 | /*--- Custom memory allocator ---*/
170 | /*! ZBUFF_createCCtx_advanced() :
171 |  *  Create a ZBUFF compression context using external alloc and free functions */
172 | ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
173 | 
174 | /*! ZBUFF_createDCtx_advanced() :
175 |  *  Create a ZBUFF decompression context using external alloc and free functions */
176 | ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
177 | 
178 | 
179 | /*--- Advanced Streaming Initialization ---*/
180 | ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
181 |                                                const void* dict, size_t dictSize,
182 |                                                ZSTD_parameters params, unsigned long long pledgedSrcSize);
183 | 
184 | //#endif /* ZBUFF_STATIC_LINKING_ONLY */
185 | 
186 | 
187 | //#if defined (__cplusplus)
188 | //}
189 | //#endif
190 | 
191 | #endif  /* ZSTD_BUFFERED_H_23987 */
192 | 


--------------------------------------------------------------------------------
/zstd/zbuff_compress.inc:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  3 |  * All rights reserved.
  4 |  *
  5 |  * This source code is licensed under the BSD-style license found in the
  6 |  * LICENSE file in the root directory of this source tree. An additional grant
  7 |  * of patent rights can be found in the PATENTS file in the same directory.
  8 |  */
  9 | 
 10 | 
 11 | 
 12 | /* *************************************
 13 | *  Dependencies
 14 | ***************************************/
 15 | #include <stdlib.h>
 16 | #include "error_private.h"
 17 | #include "zstd_internal.h"  /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */
 18 | #define ZBUFF_STATIC_LINKING_ONLY
 19 | #include "zbuff.h"
 20 | 
 21 | 
 22 | /* *************************************
 23 | *  Constants
 24 | ***************************************/
 25 | static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
 26 | 
 27 | 
 28 | /*-***********************************************************
 29 | *  Streaming compression
 30 | *
 31 | *  A ZBUFF_CCtx object is required to track streaming operation.
 32 | *  Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
 33 | *  Use ZBUFF_compressInit() to start a new compression operation.
 34 | *  ZBUFF_CCtx objects can be reused multiple times.
 35 | *
 36 | *  Use ZBUFF_compressContinue() repetitively to consume your input.
 37 | *  *srcSizePtr and *dstCapacityPtr can be any size.
 38 | *  The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
 39 | *  Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
 40 | *  The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst .
 41 | *  @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
 42 | *            or an error code, which can be tested using ZBUFF_isError().
 43 | *
 44 | *  ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
 45 | *  Note that it will not output more than *dstCapacityPtr.
 46 | *  Therefore, some content might still be left into its internal buffer if dst buffer is too small.
 47 | *  @return : nb of bytes still present into internal buffer (0 if it's empty)
 48 | *            or an error code, which can be tested using ZBUFF_isError().
 49 | *
 50 | *  ZBUFF_compressEnd() instructs to finish a frame.
 51 | *  It will perform a flush and write frame epilogue.
 52 | *  Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
 53 | *  @return : nb of bytes still present into internal buffer (0 if it's empty)
 54 | *            or an error code, which can be tested using ZBUFF_isError().
 55 | *
 56 | *  Hint : recommended buffer sizes (not compulsory)
 57 | *  input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value.
 58 | *  output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
 59 | * ***********************************************************/
 60 | 
 61 | typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage;
 62 | 
 63 | /* *** Resources *** */
 64 | struct ZBUFF_CCtx_s {
 65 |     ZSTD_CCtx* zc;
 66 |     char*  inBuff;
 67 |     size_t inBuffSize;
 68 |     size_t inToCompress;
 69 |     size_t inBuffPos;
 70 |     size_t inBuffTarget;
 71 |     size_t blockSize;
 72 |     char*  outBuff;
 73 |     size_t outBuffSize;
 74 |     size_t outBuffContentSize;
 75 |     size_t outBuffFlushedSize;
 76 |     ZBUFF_cStage stage;
 77 |     U32    checksum;
 78 |     U32    frameEnded;
 79 |     ZSTD_customMem customMem;
 80 | };   /* typedef'd tp ZBUFF_CCtx within "zbuff.h" */
 81 | 
 82 | ZBUFF_CCtx* ZBUFF_createCCtx(void)
 83 | {
 84 |     return ZBUFF_createCCtx_advanced(defaultCustomMem);
 85 | }
 86 | 
 87 | ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
 88 | {
 89 |     ZBUFF_CCtx* zbc;
 90 | 
 91 |     if (!customMem.customAlloc && !customMem.customFree)
 92 |         customMem = defaultCustomMem;
 93 | 
 94 |     if (!customMem.customAlloc || !customMem.customFree)
 95 |         return NULL;
 96 | 
 97 |     zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx));
 98 |     if (zbc==NULL) return NULL;
 99 |     memset(zbc, 0, sizeof(ZBUFF_CCtx));
100 |     memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem));
101 |     zbc->zc = ZSTD_createCCtx_advanced(customMem);
102 |     if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; }
103 |     return zbc;
104 | }
105 | 
106 | size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
107 | {
108 |     if (zbc==NULL) return 0;   /* support free on NULL */
109 |     ZSTD_freeCCtx(zbc->zc);
110 |     if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff);
111 |     if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff);
112 |     zbc->customMem.customFree(zbc->customMem.opaque, zbc);
113 |     return 0;
114 | }
115 | 
116 | 
117 | /* ======   Initialization   ====== */
118 | 
119 | size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
120 |                                    const void* dict, size_t dictSize,
121 |                                    ZSTD_parameters params, unsigned long long pledgedSrcSize)
122 | {
123 |     /* allocate buffers */
124 |     {   size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
125 |         if (zbc->inBuffSize < neededInBuffSize) {
126 |             zbc->inBuffSize = neededInBuffSize;
127 |             zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff);   /* should not be necessary */
128 |             zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize);
129 |             if (zbc->inBuff == NULL) return ERROR(memory_allocation);
130 |         }
131 |         zbc->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
132 |     }
133 |     if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
134 |         zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
135 |         zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff);   /* should not be necessary */
136 |         zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize);
137 |         if (zbc->outBuff == NULL) return ERROR(memory_allocation);
138 |     }
139 | 
140 |     { size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize);
141 |       if (ZSTD_isError(errorCode)) return errorCode; }
142 | 
143 |     zbc->inToCompress = 0;
144 |     zbc->inBuffPos = 0;
145 |     zbc->inBuffTarget = zbc->blockSize;
146 |     zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
147 |     zbc->stage = ZBUFFcs_load;
148 |     zbc->checksum = params.fParams.checksumFlag > 0;
149 |     zbc->frameEnded = 0;
150 |     return 0;   /* ready to go */
151 | }
152 | 
153 | 
154 | size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
155 | {
156 |     ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
157 |     return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
158 | }
159 | 
160 | size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
161 | {
162 |     return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel);
163 | }
164 | 
165 | 
166 | /* internal util function */
167 | MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
168 | {
169 |     size_t const length = MIN(dstCapacity, srcSize);
170 |     memcpy(dst, src, length);
171 |     return length;
172 | }
173 | 
174 | 
175 | /* ======   Compression   ====== */
176 | 
177 | typedef enum { zbf_gather, zbf_flush, zbf_end } ZBUFF_flush_e;
178 | 
179 | static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
180 |                               void* dst, size_t* dstCapacityPtr,
181 |                         const void* src, size_t* srcSizePtr,
182 |                               ZBUFF_flush_e const flush)
183 | {
184 |     U32 someMoreWork = 1;
185 |     const char* const istart = (const char*)src;
186 |     const char* const iend = istart + *srcSizePtr;
187 |     const char* ip = istart;
188 |     char* const ostart = (char*)dst;
189 |     char* const oend = ostart + *dstCapacityPtr;
190 |     char* op = ostart;
191 | 
192 |     while (someMoreWork) {
193 |         switch(zbc->stage)
194 |         {
195 |         case ZBUFFcs_init: return ERROR(init_missing);   /* call ZBUFF_compressInit() first ! */
196 | 
197 |         case ZBUFFcs_load:
198 |             /* complete inBuffer */
199 |             {   size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos;
200 |                 size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
201 |                 zbc->inBuffPos += loaded;
202 |                 ip += loaded;
203 |                 if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) {
204 |                     someMoreWork = 0; break;  /* not enough input to get a full block : stop there, wait for more */
205 |             }   }
206 |             /* compress current block (note : this stage cannot be stopped in the middle) */
207 |             {   void* cDst;
208 |                 size_t cSize;
209 |                 size_t const iSize = zbc->inBuffPos - zbc->inToCompress;
210 |                 size_t oSize = oend-op;
211 |                 if (oSize >= ZSTD_compressBound(iSize))
212 |                     cDst = op;   /* compress directly into output buffer (avoid flush stage) */
213 |                 else
214 |                     cDst = zbc->outBuff, oSize = zbc->outBuffSize;
215 |                 cSize = (flush == zbf_end) ?
216 |                         ZSTD_compressEnd(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize) :
217 |                         ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize);
218 |                 if (ZSTD_isError(cSize)) return cSize;
219 |                 if (flush == zbf_end) zbc->frameEnded = 1;
220 |                 /* prepare next block */
221 |                 zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
222 |                 if (zbc->inBuffTarget > zbc->inBuffSize)
223 |                     zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize;   /* note : inBuffSize >= blockSize */
224 |                 zbc->inToCompress = zbc->inBuffPos;
225 |                 if (cDst == op) { op += cSize; break; }   /* no need to flush */
226 |                 zbc->outBuffContentSize = cSize;
227 |                 zbc->outBuffFlushedSize = 0;
228 |                 zbc->stage = ZBUFFcs_flush;   /* continue to flush stage */
229 |             }
230 | 
231 |         case ZBUFFcs_flush:
232 |             {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
233 |                 size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
234 |                 op += flushed;
235 |                 zbc->outBuffFlushedSize += flushed;
236 |                 if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */
237 |                 zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
238 |                 zbc->stage = ZBUFFcs_load;
239 |                 break;
240 |             }
241 | 
242 |         case ZBUFFcs_final:
243 |             someMoreWork = 0;   /* do nothing */
244 |             break;
245 | 
246 |         default:
247 |             return ERROR(GENERIC);   /* impossible */
248 |         }
249 |     }
250 | 
251 |     *srcSizePtr = ip - istart;
252 |     *dstCapacityPtr = op - ostart;
253 |     if (zbc->frameEnded) return 0;
254 |     {   size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
255 |         if (hintInSize==0) hintInSize = zbc->blockSize;
256 |         return hintInSize;
257 |     }
258 | }
259 | 
260 | size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
261 |                               void* dst, size_t* dstCapacityPtr,
262 |                         const void* src, size_t* srcSizePtr)
263 | {
264 |     return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, zbf_gather);
265 | }
266 | 
267 | 
268 | 
269 | /* ======   Finalize   ====== */
270 | 
271 | size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
272 | {
273 |     size_t srcSize = 0;
274 |     ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, zbf_flush);  /* use a valid src address instead of NULL */
275 |     return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
276 | }
277 | 
278 | 
279 | size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
280 | {
281 |     BYTE* const ostart = (BYTE*)dst;
282 |     BYTE* const oend = ostart + *dstCapacityPtr;
283 |     BYTE* op = ostart;
284 | 
285 |     if (zbc->stage != ZBUFFcs_final) {
286 |         /* flush whatever remains */
287 |         size_t outSize = *dstCapacityPtr;
288 |         size_t srcSize = 0;
289 |         size_t const notEnded = ZBUFF_compressContinue_generic(zbc, dst, &outSize, &srcSize, &srcSize, zbf_end);  /* use a valid address instead of NULL */
290 |         size_t const remainingToFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
291 |         op += outSize;
292 |         if (remainingToFlush) {
293 |             *dstCapacityPtr = op-ostart;
294 |             return remainingToFlush + ZBUFF_endFrameSize + (zbc->checksum * 4);
295 |         }
296 |         /* create epilogue */
297 |         zbc->stage = ZBUFFcs_final;
298 |         zbc->outBuffContentSize = !notEnded ? 0 :
299 |             ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize, NULL, 0);  /* write epilogue into outBuff */
300 |     }
301 | 
302 |     /* flush epilogue */
303 |     {   size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
304 |         size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
305 |         op += flushed;
306 |         zbc->outBuffFlushedSize += flushed;
307 |         *dstCapacityPtr = op-ostart;
308 |         if (toFlush==flushed) zbc->stage = ZBUFFcs_init;  /* end reached */
309 |         return toFlush - flushed;
310 |     }
311 | }
312 | 
313 | 
314 | 
315 | /* *************************************
316 | *  Tool functions
317 | ***************************************/
318 | size_t ZBUFF_recommendedCInSize(void)  { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
319 | size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; }
320 | 


--------------------------------------------------------------------------------
/zstd/zbuff_decompress.inc:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  3 |  * All rights reserved.
  4 |  *
  5 |  * This source code is licensed under the BSD-style license found in the
  6 |  * LICENSE file in the root directory of this source tree. An additional grant
  7 |  * of patent rights can be found in the PATENTS file in the same directory.
  8 |  */
  9 | 
 10 | 
 11 | 
 12 | /* *************************************
 13 | *  Dependencies
 14 | ***************************************/
 15 | #include <stdlib.h>
 16 | #include "error_private.h"
 17 | #include "zstd_internal.h"  /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */
 18 | #define ZBUFF_STATIC_LINKING_ONLY
 19 | #include "zbuff.h"
 20 | 
 21 | 
 22 | typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
 23 |                ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
 24 | 
 25 | /* *** Resource management *** */
 26 | struct ZBUFF_DCtx_s {
 27 |     ZSTD_DCtx* zd;
 28 |     ZSTD_frameParams fParams;
 29 |     ZBUFF_dStage stage;
 30 |     char*  inBuff;
 31 |     size_t inBuffSize;
 32 |     size_t inPos;
 33 |     char*  outBuff;
 34 |     size_t outBuffSize;
 35 |     size_t outStart;
 36 |     size_t outEnd;
 37 |     size_t blockSize;
 38 |     BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
 39 |     size_t lhSize;
 40 |     ZSTD_customMem customMem;
 41 | };   /* typedef'd to ZBUFF_DCtx within "zbuff.h" */
 42 | 
 43 | 
 44 | ZBUFF_DCtx* ZBUFF_createDCtx(void)
 45 | {
 46 |     return ZBUFF_createDCtx_advanced(defaultCustomMem);
 47 | }
 48 | 
 49 | ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
 50 | {
 51 |     ZBUFF_DCtx* zbd;
 52 | 
 53 |     if (!customMem.customAlloc && !customMem.customFree)
 54 |         customMem = defaultCustomMem;
 55 | 
 56 |     if (!customMem.customAlloc || !customMem.customFree)
 57 |         return NULL;
 58 | 
 59 |     zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx));
 60 |     if (zbd==NULL) return NULL;
 61 |     memset(zbd, 0, sizeof(ZBUFF_DCtx));
 62 |     memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem));
 63 |     zbd->zd = ZSTD_createDCtx_advanced(customMem);
 64 |     if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; }
 65 |     zbd->stage = ZBUFFds_init;
 66 |     return zbd;
 67 | }
 68 | 
 69 | size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
 70 | {
 71 |     if (zbd==NULL) return 0;   /* support free on null */
 72 |     ZSTD_freeDCtx(zbd->zd);
 73 |     if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
 74 |     if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
 75 |     zbd->customMem.customFree(zbd->customMem.opaque, zbd);
 76 |     return 0;
 77 | }
 78 | 
 79 | 
 80 | /* *** Initialization *** */
 81 | 
 82 | size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
 83 | {
 84 |     zbd->stage = ZBUFFds_loadHeader;
 85 |     zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
 86 |     return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
 87 | }
 88 | 
 89 | size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd)
 90 | {
 91 |     return ZBUFF_decompressInitDictionary(zbd, NULL, 0);
 92 | }
 93 | 
 94 | 
 95 | /* internal util function */
 96 | /*
 97 | MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
 98 | {
 99 |     size_t const length = MIN(dstCapacity, srcSize);
100 |     memcpy(dst, src, length);
101 |     return length;
102 | }
103 | */
104 | 
105 | /* *** Decompression *** */
106 | 
107 | size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
108 |                                 void* dst, size_t* dstCapacityPtr,
109 |                           const void* src, size_t* srcSizePtr)
110 | {
111 |     const char* const istart = (const char*)src;
112 |     const char* const iend = istart + *srcSizePtr;
113 |     const char* ip = istart;
114 |     char* const ostart = (char*)dst;
115 |     char* const oend = ostart + *dstCapacityPtr;
116 |     char* op = ostart;
117 |     U32 someMoreWork = 1;
118 | 
119 |     while (someMoreWork) {
120 |         switch(zbd->stage)
121 |         {
122 |         case ZBUFFds_init :
123 |             return ERROR(init_missing);
124 | 
125 |         case ZBUFFds_loadHeader :
126 |             {   size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
127 |                 if (ZSTD_isError(hSize)) return hSize;
128 |                 if (hSize != 0) {   /* need more input */
129 |                     size_t const toLoad = hSize - zbd->lhSize;   /* if hSize!=0, hSize > zbd->lhSize */
130 |                     if (toLoad > (size_t)(iend-ip)) {   /* not enough input to load full header */
131 |                         memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
132 |                         zbd->lhSize += iend-ip;
133 |                         *dstCapacityPtr = 0;
134 |                         return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize;   /* remaining header bytes + next block header */
135 |                     }
136 |                     memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
137 |                     break;
138 |             }   }
139 | 
140 |             /* Consume header */
141 |             {   size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);  /* == ZSTD_frameHeaderSize_min */
142 |                 size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
143 |                 if (ZSTD_isError(h1Result)) return h1Result;   /* should not happen : already checked */
144 |                 if (h1Size < zbd->lhSize) {   /* long header */
145 |                     size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);
146 |                     size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
147 |                     if (ZSTD_isError(h2Result)) return h2Result;
148 |             }   }
149 | 
150 |             zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
151 | 
152 |             /* Frame header instruct buffer sizes */
153 |             {   size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
154 |                 size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
155 |                 zbd->blockSize = blockSize;
156 |                 if (zbd->inBuffSize < blockSize) {
157 |                     zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
158 |                     zbd->inBuffSize = blockSize;
159 |                     zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
160 |                     if (zbd->inBuff == NULL) return ERROR(memory_allocation);
161 |                 }
162 |                 if (zbd->outBuffSize < neededOutSize) {
163 |                     zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
164 |                     zbd->outBuffSize = neededOutSize;
165 |                     zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize);
166 |                     if (zbd->outBuff == NULL) return ERROR(memory_allocation);
167 |             }   }
168 |             zbd->stage = ZBUFFds_read;
169 |             /* pass-through */
170 | 
171 |         case ZBUFFds_read:
172 |             {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
173 |                 if (neededInSize==0) {  /* end of frame */
174 |                     zbd->stage = ZBUFFds_init;
175 |                     someMoreWork = 0;
176 |                     break;
177 |                 }
178 |                 if ((size_t)(iend-ip) >= neededInSize) {  /* decode directly from src */
179 |                     const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
180 |                     size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
181 |                         zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
182 |                         ip, neededInSize);
183 |                     if (ZSTD_isError(decodedSize)) return decodedSize;
184 |                     ip += neededInSize;
185 |                     if (!decodedSize && !isSkipFrame) break;   /* this was just a header */
186 |                     zbd->outEnd = zbd->outStart +  decodedSize;
187 |                     zbd->stage = ZBUFFds_flush;
188 |                     break;
189 |                 }
190 |                 if (ip==iend) { someMoreWork = 0; break; }   /* no more input */
191 |                 zbd->stage = ZBUFFds_load;
192 |                 /* pass-through */
193 |             }
194 | 
195 |         case ZBUFFds_load:
196 |             {   size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
197 |                 size_t const toLoad = neededInSize - zbd->inPos;   /* should always be <= remaining space within inBuff */
198 |                 size_t loadedSize;
199 |                 if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected);   /* should never happen */
200 |                 loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip);
201 |                 ip += loadedSize;
202 |                 zbd->inPos += loadedSize;
203 |                 if (loadedSize < toLoad) { someMoreWork = 0; break; }   /* not enough input, wait for more */
204 | 
205 |                 /* decode loaded input */
206 |                 {  const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
207 |                    size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
208 |                         zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
209 |                         zbd->inBuff, neededInSize);
210 |                     if (ZSTD_isError(decodedSize)) return decodedSize;
211 |                     zbd->inPos = 0;   /* input is consumed */
212 |                     if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; }   /* this was just a header */
213 |                     zbd->outEnd = zbd->outStart +  decodedSize;
214 |                     zbd->stage = ZBUFFds_flush;
215 |                     /* pass-through */
216 |             }   }
217 | 
218 |         case ZBUFFds_flush:
219 |             {   size_t const toFlushSize = zbd->outEnd - zbd->outStart;
220 |                 size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
221 |                 op += flushedSize;
222 |                 zbd->outStart += flushedSize;
223 |                 if (flushedSize == toFlushSize) {  /* flush completed */
224 |                     zbd->stage = ZBUFFds_read;
225 |                     if (zbd->outStart + zbd->blockSize > zbd->outBuffSize)
226 |                         zbd->outStart = zbd->outEnd = 0;
227 |                     break;
228 |                 }
229 |                 /* cannot flush everything */
230 |                 someMoreWork = 0;
231 |                 break;
232 |             }
233 |         default: return ERROR(GENERIC);   /* impossible */
234 |     }   }
235 | 
236 |     /* result */
237 |     *srcSizePtr = ip-istart;
238 |     *dstCapacityPtr = op-ostart;
239 |     {   size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
240 |         if (!nextSrcSizeHint) return (zbd->outEnd != zbd->outStart);   /* return 0 only if fully flushed too */
241 |         nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zbd->zd) == ZSTDnit_block);
242 |         if (zbd->inPos > nextSrcSizeHint) return ERROR(GENERIC);   /* should never happen */
243 |         nextSrcSizeHint -= zbd->inPos;   /* already loaded*/
244 |         return nextSrcSizeHint;
245 |     }
246 | }
247 | 
248 | 
249 | /* *************************************
250 | *  Tool functions
251 | ***************************************/
252 | size_t ZBUFF_recommendedDInSize(void)  { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize /* block header size*/ ; }
253 | size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
254 | 


--------------------------------------------------------------------------------
/zstd/zstd.cpp:
--------------------------------------------------------------------------------
 1 | 
 2 | #include "error_private.inc"
 3 | #include "entropy_common.inc"
 4 | 
 5 | #include "fse_compress.inc"
 6 | #include "fse_decompress.inc"
 7 | 
 8 | #include "huf_compress.inc"
 9 | #include "huf_decompress.inc"
10 | 
11 | #include "zstd_common.inc"
12 | 
13 | #include "zbuff_compress.inc"
14 | #include "zbuff_decompress.inc"
15 | 
16 | #include "zstd_compress.inc"
17 | #include "zstd_decompress.inc"
18 | 
19 | 


--------------------------------------------------------------------------------
/zstd/zstd.inc:
--------------------------------------------------------------------------------
  1 | 
  2 | #if 1
  3 | 
  4 | #define ZSTD_STATIC_LINKING_ONLY
  5 | #include "zstd.h"
  6 | 
  7 | #else
  8 | 
  9 | #include "error_private.inc"
 10 | #include "entropy_common.inc"
 11 | 
 12 | #include "fse_compress.inc"
 13 | #include "fse_decompress.inc"
 14 | 
 15 | #include "huf_compress.inc"
 16 | #include "huf_decompress.inc"
 17 | 
 18 | #include "zstd_common.inc"
 19 | 
 20 | #include "zbuff_compress.inc"
 21 | #include "zbuff_decompress.inc"
 22 | 
 23 | #include "zstd_compress.inc"
 24 | #include "zstd_decompress.inc"
 25 | 
 26 | #endif
 27 | 
 28 | template< int f_DEC > 
 29 | struct zstd_Codec : Coroutine {
 30 | 
 31 |   enum{ ID='Z' };
 32 | 
 33 |   ZSTD_CStream* cstream;
 34 |   ZSTD_DStream* dstream;
 35 | 
 36 |   uint Init( uint cLevel=0, uint winlog=-1 ) {
 37 |     coro_init(); 
 38 | //    D.C=this; D.total=0;
 39 |     if( f_DEC==0 ) return cInit(cLevel,winlog); else return dInit();
 40 |   }
 41 | 
 42 |   void Quit( void ) {
 43 |     if( f_DEC==0 ) cQuit(); else cQuit();
 44 |   }
 45 | 
 46 |   void do_process( void ) {
 47 |     if( f_DEC==0 ) compressFile(); else decompressFile();
 48 |     yield(this,0);
 49 |   }
 50 | 
 51 | 
 52 |   uint _initCStream( ZSTD_CStream* zcs, int compressionLevel, uint winlog ) {
 53 |     ZSTD_parameters params = ZSTD_getParams( compressionLevel, 0, 0 );
 54 |     if( winlog!=-1 ) {
 55 |       params.cParams.windowLog=winlog;
 56 |       params.cParams.chainLog = Min<uint>(winlog,params.cParams.chainLog);
 57 |       params.cParams.hashLog = Min<uint>(winlog,params.cParams.hashLog);
 58 |     }
 59 |     return ZSTD_initCStream_advanced(zcs, 0, 0, params, 0);
 60 |   }
 61 | 
 62 |   // r0 on success
 63 |   uint cInit( uint cLevel, uint winlog ) {
 64 |     uint r;
 65 |     //ZSTD_customMem dummy = { dum_malloc, dum_free, &D }; cstream = ZSTD_createCStream_advanced( dummy ); 
 66 |     cstream = ZSTD_createCStream(); 
 67 |     if( cstream==0 ) r=10; else r = _initCStream(cstream, cLevel, winlog); 
 68 |     return r;
 69 |   }
 70 | 
 71 |   void cQuit( void ) {
 72 |     ZSTD_freeCStream(cstream);
 73 |   }
 74 | 
 75 |   uint dInit( void ) {
 76 |     uint r;
 77 |     dstream = ZSTD_createDStream(); 
 78 |     if( dstream==0 ) r=10; else {
 79 |       r = ZSTD_initDStream(dstream); if( r==5 ) r=0;
 80 |     }
 81 |     return r;
 82 |   }
 83 | 
 84 |   void dQuit( void ) {
 85 |     ZSTD_freeDStream(dstream);
 86 |   }
 87 | 
 88 | 
 89 | 
 90 |   void compressFile(void) {
 91 | 
 92 |     size_t read, toRead;
 93 |     byte* buffIn;
 94 | 
 95 |     while(1) {
 96 |       chkinp(); read=getinplen(); buffIn=(byte*)inpptr; inpptr+=read;
 97 |       if( read==0 ) break;
 98 | 
 99 |       ZSTD_inBuffer input = { buffIn, read, 0 };
100 |       while( input.pos < input.size ) {
101 |         //ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
102 |         chkout(); ZSTD_outBuffer output = { (byte*)outptr, getoutlen(), 0 };
103 |         toRead = ZSTD_compressStream( cstream, &output, &input );
104 | //printf( "%i/%i %i/%i %i\n", input.pos,input.size, output.pos,output.size, toRead );
105 |         if( ZSTD_isError(toRead) ) yield(this,3);
106 |         outptr+=output.pos;
107 |       }
108 |     }
109 | 
110 |     while(1) {
111 |       chkout(); ZSTD_outBuffer output = { (byte*)outptr, getoutlen(), 0 };
112 |       toRead = ZSTD_endStream( cstream, &output );
113 |       outptr+=output.pos;
114 |       if( toRead==0 ) break;
115 |     }
116 | 
117 |   }
118 | 
119 |   void decompressFile(void) {
120 | 
121 |     uint read, toRead=-1;
122 |     byte* buffIn;
123 | 
124 |     while(1) {
125 |       chkinp(); read=Min<uint>(toRead,getinplen()); buffIn=(byte*)inpptr; inpptr+=read;
126 |       if( read==0 ) break;
127 | 
128 |       ZSTD_inBuffer input = { buffIn, read, 0 };
129 |       while( input.pos<input.size ) {
130 |         chkout(); ZSTD_outBuffer output = { (byte*)outptr, getoutlen(), 0 };
131 | 
132 |         // toRead : size of next compressed block
133 |         toRead = ZSTD_decompressStream( dstream, &output , &input );  
134 |         if( ZSTD_isError(toRead) ) yield(this,3);
135 | 
136 |         outptr+=output.pos;
137 |       }
138 |     }
139 | 
140 |   }
141 | 
142 | };
143 | 


--------------------------------------------------------------------------------
/zstd/zstd_common.inc:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
 3 |  * All rights reserved.
 4 |  *
 5 |  * This source code is licensed under the BSD-style license found in the
 6 |  * LICENSE file in the root directory of this source tree. An additional grant
 7 |  * of patent rights can be found in the PATENTS file in the same directory.
 8 |  */
 9 | 
10 | 
11 | 
12 | /*-*************************************
13 | *  Dependencies
14 | ***************************************/
15 | #include <stdlib.h>         /* malloc */
16 | #include "error_private.h"
17 | #define ZSTD_STATIC_LINKING_ONLY
18 | #include "zstd.h"           /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
19 | #include "zbuff.h"          /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
20 | 
21 | 
22 | /*-****************************************
23 | *  Version
24 | ******************************************/
25 | unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; }
26 | 
27 | 
28 | /*-****************************************
29 | *  ZSTD Error Management
30 | ******************************************/
31 | /*! ZSTD_isError() :
32 | *   tells if a return value is an error code */
33 | unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
34 | 
35 | /*! ZSTD_getErrorName() :
36 | *   provides error code string from function result (useful for debugging) */
37 | const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); }
38 | 
39 | /*! ZSTD_getError() :
40 | *   convert a `size_t` function result into a proper ZSTD_errorCode enum */
41 | ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
42 | 
43 | /*! ZSTD_getErrorString() :
44 | *   provides error code string from enum */
45 | const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
46 | 
47 | 
48 | /* **************************************************************
49 | *  ZBUFF Error Management
50 | ****************************************************************/
51 | unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
52 | 
53 | const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
54 | 
55 | 
56 | 
57 | /*=**************************************************************
58 | *  Custom allocator
59 | ****************************************************************/
60 | /* default uses stdlib */
61 | void* ZSTD_defaultAllocFunction(void* opaque, size_t size)
62 | {
63 |     void* address = malloc(size);
64 |     (void)opaque;
65 |     return address;
66 | }
67 | 
68 | void ZSTD_defaultFreeFunction(void* opaque, void* address)
69 | {
70 |     (void)opaque;
71 |     free(address);
72 | }
73 | 
74 | void* ZSTD_malloc(size_t size, ZSTD_customMem customMem)
75 | {
76 |     return customMem.customAlloc(customMem.opaque, size);
77 | }
78 | 
79 | void ZSTD_free(void* ptr, ZSTD_customMem customMem)
80 | {
81 |     if (ptr!=NULL)
82 |         customMem.customFree(customMem.opaque, ptr);
83 | }
84 | 


--------------------------------------------------------------------------------
/zstd/zstd_errors.h:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
 3 |  * All rights reserved.
 4 |  *
 5 |  * This source code is licensed under the BSD-style license found in the
 6 |  * LICENSE file in the root directory of this source tree. An additional grant
 7 |  * of patent rights can be found in the PATENTS file in the same directory.
 8 |  */
 9 | 
10 | #ifndef ZSTD_ERRORS_H_398273423
11 | #define ZSTD_ERRORS_H_398273423
12 | 
13 | #if defined (__cplusplus)
14 | extern "C" {
15 | #endif
16 | 
17 | /*===== dependency =====*/
18 | #include <stddef.h>   /* size_t */
19 | 
20 | 
21 | /*-****************************************
22 | *  error codes list
23 | ******************************************/
24 | typedef enum {
25 |   ZSTD_error_no_error,
26 |   ZSTD_error_GENERIC,
27 |   ZSTD_error_prefix_unknown,
28 |   ZSTD_error_version_unsupported,
29 |   ZSTD_error_parameter_unknown,
30 |   ZSTD_error_frameParameter_unsupported,
31 |   ZSTD_error_frameParameter_unsupportedBy32bits,
32 |   ZSTD_error_frameParameter_windowTooLarge,
33 |   ZSTD_error_compressionParameter_unsupported,
34 |   ZSTD_error_init_missing,
35 |   ZSTD_error_memory_allocation,
36 |   ZSTD_error_stage_wrong,
37 |   ZSTD_error_dstSize_tooSmall,
38 |   ZSTD_error_srcSize_wrong,
39 |   ZSTD_error_corruption_detected,
40 |   ZSTD_error_checksum_wrong,
41 |   ZSTD_error_tableLog_tooLarge,
42 |   ZSTD_error_maxSymbolValue_tooLarge,
43 |   ZSTD_error_maxSymbolValue_tooSmall,
44 |   ZSTD_error_dictionary_corrupted,
45 |   ZSTD_error_dictionary_wrong,
46 |   ZSTD_error_maxCode
47 | } ZSTD_ErrorCode;
48 | 
49 | /*! ZSTD_getErrorCode() :
50 |     convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
51 |     which can be used to compare directly with enum list published into "error_public.h" */
52 | ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
53 | const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
54 | 
55 | 
56 | #if defined (__cplusplus)
57 | }
58 | #endif
59 | 
60 | #endif /* ZSTD_ERRORS_H_398273423 */
61 | 


--------------------------------------------------------------------------------
/zstd/zstd_internal.h:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  3 |  * All rights reserved.
  4 |  *
  5 |  * This source code is licensed under the BSD-style license found in the
  6 |  * LICENSE file in the root directory of this source tree. An additional grant
  7 |  * of patent rights can be found in the PATENTS file in the same directory.
  8 |  */
  9 | 
 10 | #ifndef ZSTD_CCOMMON_H_MODULE
 11 | #define ZSTD_CCOMMON_H_MODULE
 12 | 
 13 | /*-*******************************************************
 14 | *  Compiler specifics
 15 | *********************************************************/
 16 | #ifdef _MSC_VER    /* Visual Studio */
 17 | #  define FORCE_INLINE static __forceinline
 18 | #  include <intrin.h>                    /* For Visual 2005 */
 19 | #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
 20 | #  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
 21 | #  pragma warning(disable : 4100)        /* disable: C4100: unreferenced formal parameter */
 22 | #else
 23 | #  if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
 24 | #    ifdef __GNUC__
 25 | #      define FORCE_INLINE static inline __attribute__((always_inline))
 26 | #    else
 27 | #      define FORCE_INLINE static inline
 28 | #    endif
 29 | #  else
 30 | #    define FORCE_INLINE static
 31 | #  endif /* __STDC_VERSION__ */
 32 | #endif
 33 | 
 34 | #ifdef _MSC_VER
 35 | #  define FORCE_NOINLINE static __declspec(noinline)
 36 | #else
 37 | #  ifdef __GNUC__
 38 | #    define FORCE_NOINLINE static __attribute__((__noinline__))
 39 | #  else
 40 | #    define FORCE_NOINLINE static
 41 | #  endif
 42 | #endif
 43 | 
 44 | 
 45 | /*-*************************************
 46 | *  Dependencies
 47 | ***************************************/
 48 | #include "mem.h"
 49 | #include "error_private.h"
 50 | #define ZSTD_STATIC_LINKING_ONLY
 51 | #include "zstd.h"
 52 | 
 53 | 
 54 | /*-*************************************
 55 | *  shared macros
 56 | ***************************************/
 57 | #define MIN(a,b) ((a)<(b) ? (a) : (b))
 58 | #define MAX(a,b) ((a)>(b) ? (a) : (b))
 59 | #define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; }  /* check and Forward error code */
 60 | #define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); }  /* check and send Error code */
 61 | 
 62 | 
 63 | /*-*************************************
 64 | *  Common constants
 65 | ***************************************/
 66 | #define ZSTD_OPT_NUM    (1<<12)
 67 | #define ZSTD_DICT_MAGIC  0xEC30A437   /* v0.7+ */
 68 | 
 69 | #define ZSTD_REP_NUM      3                 /* number of repcodes */
 70 | #define ZSTD_REP_CHECK    (ZSTD_REP_NUM)    /* number of repcodes to check by the optimal parser */
 71 | #define ZSTD_REP_MOVE     (ZSTD_REP_NUM-1)
 72 | #define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM)
 73 | static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
 74 | 
 75 | #define KB *(1 <<10)
 76 | #define MB *(1 <<20)
 77 | #define GB *(1U<<30)
 78 | 
 79 | #define BIT7 128
 80 | #define BIT6  64
 81 | #define BIT5  32
 82 | #define BIT4  16
 83 | #define BIT1   2
 84 | #define BIT0   1
 85 | 
 86 | #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
 87 | static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
 88 | static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
 89 | 
 90 | #define ZSTD_BLOCKHEADERSIZE 3   /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
 91 | static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
 92 | typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e;
 93 | 
 94 | #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */
 95 | #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */)   /* for a non-null block */
 96 | 
 97 | #define HufLog 12
 98 | typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e;
 99 | 
100 | #define LONGNBSEQ 0x7F00
101 | 
102 | #define MINMATCH 3
103 | #define EQUAL_READ32 4
104 | 
105 | #define Litbits  8
106 | #define MaxLit ((1<<Litbits) - 1)
107 | #define MaxML  52
108 | #define MaxLL  35
109 | #define MaxOff 28
110 | #define MaxSeq MAX(MaxLL, MaxML)   /* Assumption : MaxOff < MaxLL,MaxML */
111 | #define MLFSELog    9
112 | #define LLFSELog    9
113 | #define OffFSELog   8
114 | 
115 | static const U32 LL_bits[MaxLL+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
116 |                                       1, 1, 1, 1, 2, 2, 3, 3, 4, 6, 7, 8, 9,10,11,12,
117 |                                      13,14,15,16 };
118 | static const S16 LL_defaultNorm[MaxLL+1] = { 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1,
119 |                                              2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1,
120 |                                             -1,-1,-1,-1 };
121 | #define LL_DEFAULTNORMLOG 6  /* for static allocation */
122 | static const U32 LL_defaultNormLog = LL_DEFAULTNORMLOG;
123 | 
124 | static const U32 ML_bits[MaxML+1] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
125 |                                       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
126 |                                       1, 1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 7, 8, 9,10,11,
127 |                                      12,13,14,15,16 };
128 | static const S16 ML_defaultNorm[MaxML+1] = { 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
129 |                                              1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
130 |                                              1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,-1,-1,
131 |                                             -1,-1,-1,-1,-1 };
132 | #define ML_DEFAULTNORMLOG 6  /* for static allocation */
133 | static const U32 ML_defaultNormLog = ML_DEFAULTNORMLOG;
134 | 
135 | static const S16 OF_defaultNorm[MaxOff+1] = { 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1,
136 |                                               1, 1, 1, 1, 1, 1, 1, 1,-1,-1,-1,-1,-1 };
137 | #define OF_DEFAULTNORMLOG 5  /* for static allocation */
138 | static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG;
139 | 
140 | 
141 | /*-*******************************************
142 | *  Shared functions to include for inlining
143 | *********************************************/
144 | static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
145 | #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
146 | 
147 | /*! ZSTD_wildcopy() :
148 | *   custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
149 | #define WILDCOPY_OVERLENGTH 8
150 | MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length)
151 | {
152 |     const BYTE* ip = (const BYTE*)src;
153 |     BYTE* op = (BYTE*)dst;
154 |     BYTE* const oend = op + length;
155 |     do
156 |         COPY8(op, ip)
157 |     while (op < oend);
158 | }
159 | 
160 | MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd)   /* should be faster for decoding, but strangely, not verified on all platform */
161 | {
162 |     const BYTE* ip = (const BYTE*)src;
163 |     BYTE* op = (BYTE*)dst;
164 |     BYTE* const oend = (BYTE*)dstEnd;
165 |     do
166 |         COPY8(op, ip)
167 |     while (op < oend);
168 | }
169 | 
170 | 
171 | /*-*******************************************
172 | *  Private interfaces
173 | *********************************************/
174 | typedef struct ZSTD_stats_s ZSTD_stats_t;
175 | 
176 | typedef struct {
177 |     U32 off;
178 |     U32 len;
179 | } ZSTD_match_t;
180 | 
181 | typedef struct {
182 |     U32 price;
183 |     U32 off;
184 |     U32 mlen;
185 |     U32 litlen;
186 |     U32 rep[ZSTD_REP_NUM];
187 | } ZSTD_optimal_t;
188 | 
189 | 
190 | typedef struct seqDef_s {
191 |     U32 offset;
192 |     U16 litLength;
193 |     U16 matchLength;
194 | } seqDef;
195 | 
196 | 
197 | typedef struct {
198 |     seqDef* sequencesStart;
199 |     seqDef* sequences;
200 |     BYTE* litStart;
201 |     BYTE* lit;
202 |     BYTE* llCode;
203 |     BYTE* mlCode;
204 |     BYTE* ofCode;
205 |     U32   longLengthID;   /* 0 == no longLength; 1 == Lit.longLength; 2 == Match.longLength; */
206 |     U32   longLengthPos;
207 |     /* opt */
208 |     ZSTD_optimal_t* priceTable;
209 |     ZSTD_match_t* matchTable;
210 |     U32* matchLengthFreq;
211 |     U32* litLengthFreq;
212 |     U32* litFreq;
213 |     U32* offCodeFreq;
214 |     U32  matchLengthSum;
215 |     U32  matchSum;
216 |     U32  litLengthSum;
217 |     U32  litSum;
218 |     U32  offCodeSum;
219 |     U32  log2matchLengthSum;
220 |     U32  log2matchSum;
221 |     U32  log2litLengthSum;
222 |     U32  log2litSum;
223 |     U32  log2offCodeSum;
224 |     U32  factor;
225 |     U32  cachedPrice;
226 |     U32  cachedLitLength;
227 |     const BYTE* cachedLiterals;
228 | } seqStore_t;
229 | 
230 | const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx);
231 | void ZSTD_seqToCodes(const seqStore_t* seqStorePtr);
232 | int ZSTD_isSkipFrame(ZSTD_DCtx* dctx);
233 | 
234 | /* custom memory allocation functions */
235 | void* ZSTD_defaultAllocFunction(void* opaque, size_t size);
236 | void ZSTD_defaultFreeFunction(void* opaque, void* address);
237 | static const ZSTD_customMem defaultCustomMem = { ZSTD_defaultAllocFunction, ZSTD_defaultFreeFunction, NULL };
238 | void* ZSTD_malloc(size_t size, ZSTD_customMem customMem);
239 | void ZSTD_free(void* ptr, ZSTD_customMem customMem);
240 | 
241 | 
242 | /*======  common function  ======*/
243 | 
244 | MEM_STATIC U32 ZSTD_highbit32(U32 val)
245 | {
246 | #   if defined(_MSC_VER)   /* Visual */
247 |     unsigned long r=0;
248 |     _BitScanReverse(&r, val);
249 |     return (unsigned)r;
250 | #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* GCC Intrinsic */
251 |     return 31 - __builtin_clz(val);
252 | #   else   /* Software version */
253 |     static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
254 |     U32 v = val;
255 |     int r;
256 |     v |= v >> 1;
257 |     v |= v >> 2;
258 |     v |= v >> 4;
259 |     v |= v >> 8;
260 |     v |= v >> 16;
261 |     r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
262 |     return r;
263 | #   endif
264 | }
265 | 
266 | 
267 | #endif   /* ZSTD_CCOMMON_H_MODULE */
268 | 


--------------------------------------------------------------------------------