├── range_coder.hpp
└── t
    ├── bench.cpp
    └── build_table.pl


/range_coder.hpp:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyrgght (c) 2006, Daisuke Okanohara
  3 |  * Copyright (c) 2008-2010, Cybozu Labs, Inc.
  4 |  * All rights reserved.
  5 |  * 
  6 |  * Redistribution and use in source and binary forms, with or without
  7 |  * modification, are permitted provided that the following conditions are met:
  8 |  * 
  9 |  * * Redistributions of source code must retain the above copyright notice,
 10 |  *   this list of conditions and the following disclaimer.
 11 |  * * Redistributions in binary form must reproduce the above copyright notice,
 12 |  *   this list of conditions and the following disclaimer in the documentation
 13 |  *   and/or other materials provided with the distribution.
 14 |  * * Neither the name of the copyright holders nor the names of its
 15 |  *   contributors may be used to endorse or promote products derived from this
 16 |  *   software without specific prior written permission.
 17 |  * 
 18 |  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 19 |  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 20 |  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 21 |  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 22 |  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 23 |  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 24 |  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 25 |  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 26 |  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 27 |  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 28 |  * POSSIBILITY OF SUCH DAMAGE.
 29 |  */
 30 | 
 31 | #ifndef __RANGE_CODER_HPP__
 32 | #define __RANGE_CODER_HPP__
 33 | 
 34 | #ifdef RANGE_CODER_USE_SSE
 35 | #include <xmmintrin.h>
 36 | #endif
 37 | 
 38 | struct rc_type_t {
 39 |   enum {
 40 |     TOP = 1U << 24,
 41 |     TOPMASK = TOP - 1,
 42 |   };
 43 |   typedef unsigned int uint;
 44 |   typedef unsigned char byte;
 45 | };
 46 | 
 47 | template <class Iter> class rc_encoder_t : public rc_type_t {
 48 | public:
 49 |   rc_encoder_t(const Iter &i) : iter(i) {
 50 |     L      = 0;
 51 |     R      = 0xFFFFFFFF;
 52 |     buffer = 0;
 53 |     carryN = 0;
 54 |     counter = 0;
 55 |     start  = true;
 56 |   }
 57 |   void encode(const uint low, const uint high, const uint total) {
 58 |     uint r    = R / total;
 59 |     if (high < total) {
 60 |       R = r * (high-low);
 61 |     } else {
 62 |       R -= r * low;
 63 |     }
 64 |     uint newL = L + r*low;
 65 |     if (newL < L) {
 66 |       //overflow occured (newL >= 2^32)
 67 |       //buffer FF FF .. FF -> buffer+1 00 00 .. 00
 68 |       buffer++;
 69 |       for (;carryN > 0; carryN--) {
 70 | 	*iter++ = buffer;
 71 | 	buffer = 0;
 72 |       }
 73 |     }
 74 |     L = newL;
 75 |     while (R < TOP) {
 76 |       const byte newBuffer = (L >> 24);
 77 |       if (start) {
 78 | 	buffer = newBuffer;
 79 | 	start  = false;
 80 |       } else if (newBuffer == 0xFF) {
 81 | 	carryN++;
 82 |       } else {
 83 | 	*iter++ = buffer;
 84 | 	for (; carryN != 0; carryN--) {
 85 | 	  *iter++ = 0xFF;
 86 | 	}
 87 | 	buffer = newBuffer;
 88 |       }
 89 |       L <<= 8;
 90 |       R <<= 8;
 91 |     }
 92 |     counter++;
 93 |   }
 94 |   void final() {
 95 |     *iter++ = buffer;
 96 |     for (; carryN != 0; carryN--) {
 97 |       *iter++ = 0xFF;
 98 |     }
 99 |     uint t = L + R;
100 |     while (1) {
101 |       uint t8 = t >> 24, l8 = L >> 24;
102 |       *iter++ = l8;
103 |       if (t8 != l8) {
104 | 	break;
105 |       }
106 |       t <<= 8;
107 |       L <<= 8;
108 |     }
109 |   }
110 | private:
111 |   uint R;
112 |   uint L;
113 |   bool start;
114 |   byte buffer;
115 |   uint carryN;
116 |   Iter iter;
117 |   uint counter;
118 | };
119 | 
120 | template <typename FreqType, unsigned _N, int _BASE> struct rc_decoder_search_traits_t : public rc_type_t {
121 |   typedef FreqType freq_type;
122 |   enum {
123 |     N = _N,
124 |     BASE = _BASE
125 |   };
126 | };
127 | 
128 | template <typename FreqType, unsigned _N, int _BASE = 0> struct rc_decoder_search_t : public rc_decoder_search_traits_t<FreqType, _N, _BASE> {
129 |   static rc_type_t::uint get_index(const FreqType *freq, FreqType pos) {
130 |     rc_type_t::uint left  = 0;
131 |     rc_type_t::uint right = _N - 1;
132 |     while(left < right) {
133 |       rc_type_t::uint mid = (left+right)/2;
134 |       if (freq[mid+1] <= pos) left = mid+1;
135 |       else                    right = mid;
136 |     }
137 |     return left;
138 |   }
139 | };
140 | 
141 | #ifdef RANGE_CODER_USE_SSE
142 | 
143 | template<int _BASE> struct rc_decoder_search_t<short, 256, _BASE> : public rc_decoder_search_traits_t<short, 256, _BASE> {
144 |   static rc_type_t::uint get_index(const short *freq, short pos) {
145 |     __m128i v = _mm_set1_epi16(pos);
146 |     unsigned i, mask = 0;
147 |     for (i = 0; i < 256; i += 16) {
148 |       __m128i x = *reinterpret_cast<const __m128i*>(freq + i);
149 |       __m128i y = *reinterpret_cast<const __m128i*>(freq + i + 8);
150 |       __m128i a = _mm_cmplt_epi16(v, x);
151 |       __m128i b = _mm_cmplt_epi16(v, y);
152 |       mask = (_mm_movemask_epi8(b) << 16) | _mm_movemask_epi8(a);
153 |       if (mask) {
154 | 	return i + (__builtin_ctz(mask) >> 1) - 1;
155 |       }
156 |     }
157 |     return 255;
158 |   }
159 | };
160 | 
161 | #endif
162 | 
163 | template <class Iterator, class SearchType> class rc_decoder_t : public rc_type_t {
164 | public:
165 |   typedef SearchType search_type;
166 |   typedef typename search_type::freq_type freq_type;
167 |   static const unsigned N = search_type::N;
168 |   rc_decoder_t(const Iterator& _i, const Iterator _e) : iter(_i), iter_end(_e) {
169 |     R = 0xFFFFFFFF;
170 |     D = 0;
171 |     for (int i = 0; i < 4; i++) {
172 |       D = (D << 8) | next();
173 |     }
174 |   }
175 |   uint decode(const uint total, const freq_type* cumFreq) {
176 |     const uint r = R / total;
177 |     const int targetPos = std::min(total-1, D / r);
178 |     
179 |     //find target s.t. cumFreq[target] <= targetPos < cumFreq[target+1]
180 |     const uint target =
181 |       search_type::get_index(cumFreq, targetPos + search_type::BASE);
182 |     const uint low  = cumFreq[target] - search_type::BASE;
183 |     const uint high = cumFreq[target+1] - search_type::BASE;
184 |     
185 |     D -= r * low;
186 |     if (high != total) {
187 |       R = r * (high-low);
188 |     } else {
189 |       R -= r * low;
190 |     }
191 |     
192 |     while (R < TOP) {
193 |       R <<= 8;
194 |       D = (D << 8) | next();
195 |     }
196 |     
197 |     return target;
198 |   }
199 |   byte next() {
200 |     return iter != iter_end ? (byte)*iter++ : 0xff;
201 |   }
202 | private:
203 |   uint R;
204 |   uint D;
205 |   Iterator iter, iter_end;
206 | };
207 | 
208 | #endif
209 | 


--------------------------------------------------------------------------------
/t/bench.cpp:
--------------------------------------------------------------------------------
 1 | extern "C" {
 2 | #include <assert.h>
 3 | #include <limits.h>
 4 | #include <stdio.h>
 5 | #include <stdlib.h>
 6 | #include <string.h>
 7 | }
 8 | #include <algorithm>
 9 | #include "range_coder.hpp"
10 | 
11 | #include "table.c"
12 | 
13 | class writer_t {
14 |   char **p, *max;
15 | public:
16 |   struct overrun_t {
17 |   };
18 |   writer_t(char **_p, char *_max) : p(_p), max(_max) {}
19 |   writer_t &operator=(char c) {
20 |     if (*p == max) {
21 |       throw overrun_t();
22 |     }
23 |     *(*p)++ = c;
24 |     return *this;
25 |   }
26 |   writer_t &operator*() { return *this; }
27 |   writer_t &operator++() { return *this; }
28 |   writer_t &operator++(int) { return *this; }
29 | };
30 | 
31 | #define FREQ_BASE SHRT_MIN
32 | #define LOOP_CNT 1024
33 | 
34 | int main(int argc, char **argv)
35 | {
36 |   char buf[1024 * 1024], cbuf[1024 * 1024], rbuf[1024 * 1024];
37 |   size_t buflen, cbuflen;
38 |   unsigned long long start;
39 |   int i;
40 |   
41 |   /* read */
42 |   buflen = fread(buf, 1, sizeof(buf) - 1, stdin);
43 |   /* compress */
44 |   start = rdtsc();
45 |   for (i = 0; i < LOOP_CNT; i++) {
46 |     char *cbufpt = cbuf;
47 |     rc_encoder_t<writer_t> enc(writer_t(&cbufpt, cbuf + sizeof(cbuf)));
48 |     for (const char *p = buf, *e = buf + buflen; p != e; p++) {
49 |       unsigned ch = (unsigned char)*p;
50 | #ifdef USE_ORDERED_TABLE
51 |       ch = to_ordered[ch];
52 | #endif
53 |       assert(freq[ch] != freq[ch + 1]);
54 |       enc.encode(freq[ch] - FREQ_BASE, freq[ch + 1] - FREQ_BASE, MAX_FREQ);
55 |     }
56 |     enc.final();
57 |     cbuflen = cbufpt - cbuf;
58 |   }
59 |   printf("compression: %lu Mticks\n", (long)((rdtsc() - start) / 1024 / 1024));
60 |   /* decompress */
61 |   start = rdtsc();
62 |   for (i = 0; i < LOOP_CNT; i++) {
63 |     rc_decoder_t<const char*, rc_decoder_search_t<short, 256, FREQ_BASE> >
64 |       dec(cbuf, cbuf + cbuflen);
65 |     for (char *p = rbuf, *e = rbuf + buflen; p != e; p++) {
66 |       unsigned ch = dec.decode(MAX_FREQ, freq);
67 | #ifdef USE_ORDERED_TABLE
68 |       ch = from_ordered[ch];
69 | #endif
70 |       *p = ch;
71 |     }
72 |   }
73 |   printf("decompression: %lu Mticks\n",
74 | 	 (long)((rdtsc() - start) / 1024 / 1024));
75 |   /* check result */
76 |   if (memcmp(buf, rbuf, buflen) != 0) {
77 |     fprintf(stderr, "original data and decompressed data does not match.\n");
78 |     exit(99);
79 |   }
80 |   
81 |   return 0;
82 | }
83 | 


--------------------------------------------------------------------------------
/t/build_table.pl:
--------------------------------------------------------------------------------
 1 | #! /usr/bin/perl
 2 | 
 3 | use strict;
 4 | use warnings;
 5 | 
 6 | use Getopt::Long;
 7 | use List::Util qw/sum max/;
 8 | 
 9 | my ($do_ordered);
10 | 
11 | GetOptions(
12 |     'ordered' => \$do_ordered,
13 | );
14 | 
15 | 
16 | my @cnt = map { 0 } 0..255;
17 | 
18 | while (<>) {
19 |     foreach my $c (split '', $_) {
20 |         $cnt[ord $c]++;
21 |     }
22 | }
23 | 
24 | if ($do_ordered) {
25 |     my @order = sort { $cnt[$b] <=> $cnt[$a] } 0..255;
26 |     print "#define USE_ORDERED_TABLE 1\n";
27 |     print "static unsigned char from_ordered[] = {", join(',', @order), "};\n";
28 |     my %r = map { $order[$_] => $_ } @order;
29 |     print "static unsigned char to_ordered[] = {", join(',', map { $r{$_} } 0..255), "};\n";
30 |     @cnt = map { $cnt[$order[$_]] } 0..255;
31 | }
32 | 
33 | my @freq;
34 | my $cc = sum @cnt;
35 | my $mult = 0x8000;
36 | while (1) {
37 |     print STDERR "$mult\n";
38 |     my $acc = 0;
39 |     for (my $i = 0; $i < 256; $i++) {
40 | 	push @freq, $acc;
41 | 	$acc += $cnt[$i] != 0 ? max(int($cnt[$i] / $cc * $mult + 0.5), 1) : 0;
42 |     }
43 |     last if $acc <= 0x8000;
44 |     @freq = ();
45 |     $mult--;
46 | }
47 | push @freq, 0x8000;
48 | 
49 | print "#define MAX_FREQ 0x8000\n";
50 | print "static short freq[] __attribute__((aligned(16))) = {", join(',', map { $_ - 0x8000 } @freq), "};\n";
51 | 


--------------------------------------------------------------------------------