├── .gitignore ├── .travis.yml ├── CHANGES ├── README.md ├── c_src ├── lz4.c ├── lz4.h ├── lz4_nif.c ├── lz4hc.c ├── lz4hc.h └── lz4opt.h ├── doc └── overview.edoc ├── mix.exs ├── mix.lock ├── rebar.config ├── rebar.config.script ├── src ├── lz4.app.src └── lz4.erl └── test └── lz4_tests.erl /.gitignore: -------------------------------------------------------------------------------- 1 | .eunit 2 | ebin 3 | deps 4 | priv 5 | *.o 6 | *.beam 7 | *.plt 8 | *.swp 9 | *.html 10 | *.png 11 | edoc-info 12 | stylesheet.css 13 | /_build 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | notifications: 3 | disabled: true 4 | branches: 5 | only: 6 | - master 7 | - develop 8 | - 0.1.0 9 | - 0.1.1 10 | - 0.2 11 | - 0.2.1 12 | - 0.2.2 13 | otp_release: 14 | - 17.0 15 | - R16B03-1 16 | - R16B03 17 | - R16B02 18 | - R16B01 19 | - R16B 20 | - R15B03 21 | - R15B02 22 | - R15B01 23 | - R15B 24 | 25 | -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | 0.2.4 (2017/4/22) 2 | ================= 3 | 4 | - upgrade LZ4 1.7.5 (by @lpgauth) 5 | 6 | 0.2.3 (2017/2/5) 7 | ================= 8 | 9 | - rebar3 support (by @lpgauth) 10 | - deprecate rebar 11 | 12 | 0.2.2 (2014/7/18) 13 | ================= 14 | 15 | - NIF: replaces obsolute function LZ4_uncompress() with LZ4_decompress_safe() 16 | 17 | 0.2.1 (2014/7/9) 18 | ================= 19 | 20 | - update lz4 implementation 21 | 22 | 0.1.1 (2012/10/3) 23 | ================= 24 | 25 | - add travis.yml 26 | 27 | 0.1.0 (2012/10/2) 28 | ================= 29 | 30 | - first release 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | erlang-lz4 2 | ========== 3 | 4 | branch: master [![Build Status](https://secure.travis-ci.org/szktty/erlang-lz4.png?branch=master)](http://travis-ci.org/szktty/erlang-lz4) 5 | develop [![Build Status](https://secure.travis-ci.org/szktty/erlang-lz4.png?branch=develop)](http://travis-ci.org/szktty/erlang-lz4) 6 | 7 | LZ4 bindings for Erlang 8 | 9 | This library uses source code of LZ4 from https://github.com/Cyan4973/lz4. 10 | 11 | 12 | LZ4 implementation version 13 | -------------------------- 14 | 15 | https://github.com/Cyan4973/lz4/releases/tag/r119 16 | 17 | 18 | Licenses 19 | -------- 20 | 21 | This program is distributed under Apache License 2.0. 22 | 23 | LZ4 library is distributed under New BSD License. 24 | 25 | 26 | Author 27 | ------ 28 | 29 | SUZUKI Tetsuya 30 | 31 | -------------------------------------------------------------------------------- /c_src/lz4.c: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 - Fast LZ compression algorithm 3 | Copyright (C) 2011-2016, Yann Collet. 4 | 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - LZ4 homepage : http://www.lz4.org 32 | - LZ4 source repository : https://github.com/lz4/lz4 33 | */ 34 | 35 | 36 | /*-************************************ 37 | * Tuning parameters 38 | **************************************/ 39 | /* 40 | * HEAPMODE : 41 | * Select how default compression functions will allocate memory for their hash table, 42 | * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). 43 | */ 44 | #ifndef HEAPMODE 45 | # define HEAPMODE 0 46 | #endif 47 | 48 | /* 49 | * ACCELERATION_DEFAULT : 50 | * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 51 | */ 52 | #define ACCELERATION_DEFAULT 1 53 | 54 | 55 | /*-************************************ 56 | * CPU Feature Detection 57 | **************************************/ 58 | /* LZ4_FORCE_MEMORY_ACCESS 59 | * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. 60 | * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. 61 | * The below switch allow to select different access method for improved performance. 62 | * Method 0 (default) : use `memcpy()`. Safe and portable. 63 | * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). 64 | * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. 65 | * Method 2 : direct access. This method is portable but violate C standard. 66 | * It can generate buggy code on targets which generate assembly depending on alignment. 67 | * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) 68 | * See https://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. 69 | * Prefer these methods in priority order (0 > 1 > 2) 70 | */ 71 | #ifndef LZ4_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ 72 | # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) 73 | # define LZ4_FORCE_MEMORY_ACCESS 2 74 | # elif defined(__INTEL_COMPILER) || \ 75 | (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) 76 | # define LZ4_FORCE_MEMORY_ACCESS 1 77 | # endif 78 | #endif 79 | 80 | /* 81 | * LZ4_FORCE_SW_BITCOUNT 82 | * Define this parameter if your target system or compiler does not support hardware bit count 83 | */ 84 | #if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ 85 | # define LZ4_FORCE_SW_BITCOUNT 86 | #endif 87 | 88 | 89 | /*-************************************ 90 | * Dependency 91 | **************************************/ 92 | #include "lz4.h" 93 | /* see also "memory routines" below */ 94 | 95 | 96 | /*-************************************ 97 | * Compiler Options 98 | **************************************/ 99 | #ifdef _MSC_VER /* Visual Studio */ 100 | # define FORCE_INLINE static __forceinline 101 | # include 102 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 103 | # pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ 104 | #else 105 | # if defined(__GNUC__) || defined(__clang__) 106 | # define FORCE_INLINE static inline __attribute__((always_inline)) 107 | # elif defined(__cplusplus) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 108 | # define FORCE_INLINE static inline 109 | # else 110 | # define FORCE_INLINE static 111 | # endif 112 | #endif /* _MSC_VER */ 113 | 114 | #if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) 115 | # define expect(expr,value) (__builtin_expect ((expr),(value)) ) 116 | #else 117 | # define expect(expr,value) (expr) 118 | #endif 119 | 120 | #define likely(expr) expect((expr) != 0, 1) 121 | #define unlikely(expr) expect((expr) != 0, 0) 122 | 123 | 124 | /*-************************************ 125 | * Memory routines 126 | **************************************/ 127 | #include /* malloc, calloc, free */ 128 | #define ALLOCATOR(n,s) calloc(n,s) 129 | #define FREEMEM free 130 | #include /* memset, memcpy */ 131 | #define MEM_INIT memset 132 | 133 | 134 | /*-************************************ 135 | * Basic Types 136 | **************************************/ 137 | #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 138 | # include 139 | typedef uint8_t BYTE; 140 | typedef uint16_t U16; 141 | typedef uint32_t U32; 142 | typedef int32_t S32; 143 | typedef uint64_t U64; 144 | typedef uintptr_t uptrval; 145 | #else 146 | typedef unsigned char BYTE; 147 | typedef unsigned short U16; 148 | typedef unsigned int U32; 149 | typedef signed int S32; 150 | typedef unsigned long long U64; 151 | typedef size_t uptrval; /* generally true, except OpenVMS-64 */ 152 | #endif 153 | 154 | #if defined(__x86_64__) 155 | typedef U64 reg_t; /* 64-bits in x32 mode */ 156 | #else 157 | typedef size_t reg_t; /* 32-bits in x32 mode */ 158 | #endif 159 | 160 | /*-************************************ 161 | * Reading and writing into memory 162 | **************************************/ 163 | static unsigned LZ4_isLittleEndian(void) 164 | { 165 | const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ 166 | return one.c[0]; 167 | } 168 | 169 | 170 | #if defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==2) 171 | /* lie to the compiler about data alignment; use with caution */ 172 | 173 | static U16 LZ4_read16(const void* memPtr) { return *(const U16*) memPtr; } 174 | static U32 LZ4_read32(const void* memPtr) { return *(const U32*) memPtr; } 175 | static reg_t LZ4_read_ARCH(const void* memPtr) { return *(const reg_t*) memPtr; } 176 | 177 | static void LZ4_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } 178 | static void LZ4_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } 179 | 180 | #elif defined(LZ4_FORCE_MEMORY_ACCESS) && (LZ4_FORCE_MEMORY_ACCESS==1) 181 | 182 | /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ 183 | /* currently only defined for gcc and icc */ 184 | typedef union { U16 u16; U32 u32; reg_t uArch; } __attribute__((packed)) unalign; 185 | 186 | static U16 LZ4_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } 187 | static U32 LZ4_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } 188 | static reg_t LZ4_read_ARCH(const void* ptr) { return ((const unalign*)ptr)->uArch; } 189 | 190 | static void LZ4_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } 191 | static void LZ4_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } 192 | 193 | #else /* safe and portable access through memcpy() */ 194 | 195 | static U16 LZ4_read16(const void* memPtr) 196 | { 197 | U16 val; memcpy(&val, memPtr, sizeof(val)); return val; 198 | } 199 | 200 | static U32 LZ4_read32(const void* memPtr) 201 | { 202 | U32 val; memcpy(&val, memPtr, sizeof(val)); return val; 203 | } 204 | 205 | static reg_t LZ4_read_ARCH(const void* memPtr) 206 | { 207 | reg_t val; memcpy(&val, memPtr, sizeof(val)); return val; 208 | } 209 | 210 | static void LZ4_write16(void* memPtr, U16 value) 211 | { 212 | memcpy(memPtr, &value, sizeof(value)); 213 | } 214 | 215 | static void LZ4_write32(void* memPtr, U32 value) 216 | { 217 | memcpy(memPtr, &value, sizeof(value)); 218 | } 219 | 220 | #endif /* LZ4_FORCE_MEMORY_ACCESS */ 221 | 222 | 223 | static U16 LZ4_readLE16(const void* memPtr) 224 | { 225 | if (LZ4_isLittleEndian()) { 226 | return LZ4_read16(memPtr); 227 | } else { 228 | const BYTE* p = (const BYTE*)memPtr; 229 | return (U16)((U16)p[0] + (p[1]<<8)); 230 | } 231 | } 232 | 233 | static void LZ4_writeLE16(void* memPtr, U16 value) 234 | { 235 | if (LZ4_isLittleEndian()) { 236 | LZ4_write16(memPtr, value); 237 | } else { 238 | BYTE* p = (BYTE*)memPtr; 239 | p[0] = (BYTE) value; 240 | p[1] = (BYTE)(value>>8); 241 | } 242 | } 243 | 244 | static void LZ4_copy8(void* dst, const void* src) 245 | { 246 | memcpy(dst,src,8); 247 | } 248 | 249 | /* customized variant of memcpy, which can overwrite up to 8 bytes beyond dstEnd */ 250 | static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) 251 | { 252 | BYTE* d = (BYTE*)dstPtr; 253 | const BYTE* s = (const BYTE*)srcPtr; 254 | BYTE* const e = (BYTE*)dstEnd; 255 | 256 | do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); 300 | # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 301 | return (__builtin_ctzll((U64)val) >> 3); 302 | # else 303 | static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; 304 | return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; 305 | # endif 306 | } else /* 32 bits */ { 307 | # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) 308 | unsigned long r; 309 | _BitScanForward( &r, (U32)val ); 310 | return (int)(r>>3); 311 | # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 312 | return (__builtin_ctz((U32)val) >> 3); 313 | # else 314 | static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; 315 | return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; 316 | # endif 317 | } 318 | } else /* Big Endian CPU */ { 319 | if (sizeof(val)==8) { 320 | # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) 321 | unsigned long r = 0; 322 | _BitScanReverse64( &r, val ); 323 | return (unsigned)(r>>3); 324 | # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 325 | return (__builtin_clzll((U64)val) >> 3); 326 | # else 327 | unsigned r; 328 | if (!(val>>32)) { r=4; } else { r=0; val>>=32; } 329 | if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } 330 | r += (!val); 331 | return r; 332 | # endif 333 | } else /* 32 bits */ { 334 | # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) 335 | unsigned long r = 0; 336 | _BitScanReverse( &r, (unsigned long)val ); 337 | return (unsigned)(r>>3); 338 | # elif (defined(__clang__) || (defined(__GNUC__) && (__GNUC__>=3))) && !defined(LZ4_FORCE_SW_BITCOUNT) 339 | return (__builtin_clz((U32)val) >> 3); 340 | # else 341 | unsigned r; 342 | if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } 343 | r += (!val); 344 | return r; 345 | # endif 346 | } 347 | } 348 | } 349 | 350 | #define STEPSIZE sizeof(reg_t) 351 | static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) 352 | { 353 | const BYTE* const pStart = pIn; 354 | 355 | while (likely(pIn compression run slower on incompressible data */ 375 | 376 | 377 | /*-************************************ 378 | * Local Structures and types 379 | **************************************/ 380 | typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; 381 | typedef enum { byPtr, byU32, byU16 } tableType_t; 382 | 383 | typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; 384 | typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; 385 | 386 | typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; 387 | typedef enum { full = 0, partial = 1 } earlyEnd_directive; 388 | 389 | 390 | /*-************************************ 391 | * Local Utils 392 | **************************************/ 393 | int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } 394 | const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; } 395 | int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } 396 | int LZ4_sizeofState() { return LZ4_STREAMSIZE; } 397 | 398 | 399 | /*-****************************** 400 | * Compression functions 401 | ********************************/ 402 | static U32 LZ4_hash4(U32 sequence, tableType_t const tableType) 403 | { 404 | if (tableType == byU16) 405 | return ((sequence * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); 406 | else 407 | return ((sequence * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); 408 | } 409 | 410 | static U32 LZ4_hash5(U64 sequence, tableType_t const tableType) 411 | { 412 | static const U64 prime5bytes = 889523592379ULL; 413 | static const U64 prime8bytes = 11400714785074694791ULL; 414 | const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; 415 | if (LZ4_isLittleEndian()) 416 | return (U32)(((sequence << 24) * prime5bytes) >> (64 - hashLog)); 417 | else 418 | return (U32)(((sequence >> 24) * prime8bytes) >> (64 - hashLog)); 419 | } 420 | 421 | FORCE_INLINE U32 LZ4_hashPosition(const void* const p, tableType_t const tableType) 422 | { 423 | if ((sizeof(reg_t)==8) && (tableType != byU16)) return LZ4_hash5(LZ4_read_ARCH(p), tableType); 424 | return LZ4_hash4(LZ4_read32(p), tableType); 425 | } 426 | 427 | static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) 428 | { 429 | switch (tableType) 430 | { 431 | case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } 432 | case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } 433 | case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } 434 | } 435 | } 436 | 437 | FORCE_INLINE void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) 438 | { 439 | U32 const h = LZ4_hashPosition(p, tableType); 440 | LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); 441 | } 442 | 443 | static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) 444 | { 445 | if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } 446 | if (tableType == byU32) { const U32* const hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } 447 | { const U16* const hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ 448 | } 449 | 450 | FORCE_INLINE const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) 451 | { 452 | U32 const h = LZ4_hashPosition(p, tableType); 453 | return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); 454 | } 455 | 456 | 457 | /** LZ4_compress_generic() : 458 | inlined, to ensure branches are decided at compilation time */ 459 | FORCE_INLINE int LZ4_compress_generic( 460 | LZ4_stream_t_internal* const cctx, 461 | const char* const source, 462 | char* const dest, 463 | const int inputSize, 464 | const int maxOutputSize, 465 | const limitedOutput_directive outputLimited, 466 | const tableType_t tableType, 467 | const dict_directive dict, 468 | const dictIssue_directive dictIssue, 469 | const U32 acceleration) 470 | { 471 | const BYTE* ip = (const BYTE*) source; 472 | const BYTE* base; 473 | const BYTE* lowLimit; 474 | const BYTE* const lowRefLimit = ip - cctx->dictSize; 475 | const BYTE* const dictionary = cctx->dictionary; 476 | const BYTE* const dictEnd = dictionary + cctx->dictSize; 477 | const ptrdiff_t dictDelta = dictEnd - (const BYTE*)source; 478 | const BYTE* anchor = (const BYTE*) source; 479 | const BYTE* const iend = ip + inputSize; 480 | const BYTE* const mflimit = iend - MFLIMIT; 481 | const BYTE* const matchlimit = iend - LASTLITERALS; 482 | 483 | BYTE* op = (BYTE*) dest; 484 | BYTE* const olimit = op + maxOutputSize; 485 | 486 | U32 forwardH; 487 | 488 | /* Init conditions */ 489 | if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported inputSize, too large (or negative) */ 490 | switch(dict) 491 | { 492 | case noDict: 493 | default: 494 | base = (const BYTE*)source; 495 | lowLimit = (const BYTE*)source; 496 | break; 497 | case withPrefix64k: 498 | base = (const BYTE*)source - cctx->currentOffset; 499 | lowLimit = (const BYTE*)source - cctx->dictSize; 500 | break; 501 | case usingExtDict: 502 | base = (const BYTE*)source - cctx->currentOffset; 503 | lowLimit = (const BYTE*)source; 504 | break; 505 | } 506 | if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ 507 | if (inputSizehashTable, tableType, base); 511 | ip++; forwardH = LZ4_hashPosition(ip, tableType); 512 | 513 | /* Main Loop */ 514 | for ( ; ; ) { 515 | ptrdiff_t refDelta = 0; 516 | const BYTE* match; 517 | BYTE* token; 518 | 519 | /* Find a match */ 520 | { const BYTE* forwardIp = ip; 521 | unsigned step = 1; 522 | unsigned searchMatchNb = acceleration << LZ4_skipTrigger; 523 | do { 524 | U32 const h = forwardH; 525 | ip = forwardIp; 526 | forwardIp += step; 527 | step = (searchMatchNb++ >> LZ4_skipTrigger); 528 | 529 | if (unlikely(forwardIp > mflimit)) goto _last_literals; 530 | 531 | match = LZ4_getPositionOnHash(h, cctx->hashTable, tableType, base); 532 | if (dict==usingExtDict) { 533 | if (match < (const BYTE*)source) { 534 | refDelta = dictDelta; 535 | lowLimit = dictionary; 536 | } else { 537 | refDelta = 0; 538 | lowLimit = (const BYTE*)source; 539 | } } 540 | forwardH = LZ4_hashPosition(forwardIp, tableType); 541 | LZ4_putPositionOnHash(ip, h, cctx->hashTable, tableType, base); 542 | 543 | } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) 544 | || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) 545 | || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); 546 | } 547 | 548 | /* Catch up */ 549 | while (((ip>anchor) & (match+refDelta > lowLimit)) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } 550 | 551 | /* Encode Literals */ 552 | { unsigned const litLength = (unsigned)(ip - anchor); 553 | token = op++; 554 | if ((outputLimited) && /* Check output buffer overflow */ 555 | (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) 556 | return 0; 557 | if (litLength >= RUN_MASK) { 558 | int len = (int)litLength-RUN_MASK; 559 | *token = (RUN_MASK<= 255 ; len-=255) *op++ = 255; 561 | *op++ = (BYTE)len; 562 | } 563 | else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; 582 | matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); 583 | ip += MINMATCH + matchCode; 584 | if (ip==limit) { 585 | unsigned const more = LZ4_count(ip, (const BYTE*)source, matchlimit); 586 | matchCode += more; 587 | ip += more; 588 | } 589 | } else { 590 | matchCode = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); 591 | ip += MINMATCH + matchCode; 592 | } 593 | 594 | if ( outputLimited && /* Check output buffer overflow */ 595 | (unlikely(op + (1 + LASTLITERALS) + (matchCode>>8) > olimit)) ) 596 | return 0; 597 | if (matchCode >= ML_MASK) { 598 | *token += ML_MASK; 599 | matchCode -= ML_MASK; 600 | LZ4_write32(op, 0xFFFFFFFF); 601 | while (matchCode >= 4*255) op+=4, LZ4_write32(op, 0xFFFFFFFF), matchCode -= 4*255; 602 | op += matchCode / 255; 603 | *op++ = (BYTE)(matchCode % 255); 604 | } else 605 | *token += (BYTE)(matchCode); 606 | } 607 | 608 | anchor = ip; 609 | 610 | /* Test end of chunk */ 611 | if (ip > mflimit) break; 612 | 613 | /* Fill table */ 614 | LZ4_putPosition(ip-2, cctx->hashTable, tableType, base); 615 | 616 | /* Test next position */ 617 | match = LZ4_getPosition(ip, cctx->hashTable, tableType, base); 618 | if (dict==usingExtDict) { 619 | if (match < (const BYTE*)source) { 620 | refDelta = dictDelta; 621 | lowLimit = dictionary; 622 | } else { 623 | refDelta = 0; 624 | lowLimit = (const BYTE*)source; 625 | } } 626 | LZ4_putPosition(ip, cctx->hashTable, tableType, base); 627 | if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) 628 | && (match+MAX_DISTANCE>=ip) 629 | && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) 630 | { token=op++; *token=0; goto _next_match; } 631 | 632 | /* Prepare next loop */ 633 | forwardH = LZ4_hashPosition(++ip, tableType); 634 | } 635 | 636 | _last_literals: 637 | /* Encode Last Literals */ 638 | { size_t const lastRun = (size_t)(iend - anchor); 639 | if ( (outputLimited) && /* Check output buffer overflow */ 640 | ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) ) 641 | return 0; 642 | if (lastRun >= RUN_MASK) { 643 | size_t accumulator = lastRun - RUN_MASK; 644 | *op++ = RUN_MASK << ML_BITS; 645 | for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; 646 | *op++ = (BYTE) accumulator; 647 | } else { 648 | *op++ = (BYTE)(lastRun<internal_donotuse; 662 | LZ4_resetStream((LZ4_stream_t*)state); 663 | if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; 664 | 665 | if (maxOutputSize >= LZ4_compressBound(inputSize)) { 666 | if (inputSize < LZ4_64Klimit) 667 | return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); 668 | else 669 | return LZ4_compress_generic(ctx, source, dest, inputSize, 0, notLimited, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); 670 | } else { 671 | if (inputSize < LZ4_64Klimit) 672 | return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); 673 | else 674 | return LZ4_compress_generic(ctx, source, dest, inputSize, maxOutputSize, limitedOutput, (sizeof(void*)==8) ? byU32 : byPtr, noDict, noDictIssue, acceleration); 675 | } 676 | } 677 | 678 | 679 | int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) 680 | { 681 | #if (HEAPMODE) 682 | void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ 683 | #else 684 | LZ4_stream_t ctx; 685 | void* const ctxPtr = &ctx; 686 | #endif 687 | 688 | int const result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); 689 | 690 | #if (HEAPMODE) 691 | FREEMEM(ctxPtr); 692 | #endif 693 | return result; 694 | } 695 | 696 | 697 | int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) 698 | { 699 | return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); 700 | } 701 | 702 | 703 | /* hidden debug function */ 704 | /* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ 705 | int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) 706 | { 707 | LZ4_stream_t ctx; 708 | LZ4_resetStream(&ctx); 709 | 710 | if (inputSize < LZ4_64Klimit) 711 | return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); 712 | else 713 | return LZ4_compress_generic(&ctx.internal_donotuse, source, dest, inputSize, maxOutputSize, limitedOutput, sizeof(void*)==8 ? byU32 : byPtr, noDict, noDictIssue, acceleration); 714 | } 715 | 716 | 717 | /*-****************************** 718 | * *_destSize() variant 719 | ********************************/ 720 | 721 | static int LZ4_compress_destSize_generic( 722 | LZ4_stream_t_internal* const ctx, 723 | const char* const src, 724 | char* const dst, 725 | int* const srcSizePtr, 726 | const int targetDstSize, 727 | const tableType_t tableType) 728 | { 729 | const BYTE* ip = (const BYTE*) src; 730 | const BYTE* base = (const BYTE*) src; 731 | const BYTE* lowLimit = (const BYTE*) src; 732 | const BYTE* anchor = ip; 733 | const BYTE* const iend = ip + *srcSizePtr; 734 | const BYTE* const mflimit = iend - MFLIMIT; 735 | const BYTE* const matchlimit = iend - LASTLITERALS; 736 | 737 | BYTE* op = (BYTE*) dst; 738 | BYTE* const oend = op + targetDstSize; 739 | BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; 740 | BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); 741 | BYTE* const oMaxSeq = oMaxLit - 1 /* token */; 742 | 743 | U32 forwardH; 744 | 745 | 746 | /* Init conditions */ 747 | if (targetDstSize < 1) return 0; /* Impossible to store anything */ 748 | if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ 749 | if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ 750 | if (*srcSizePtrhashTable, tableType, base); 755 | ip++; forwardH = LZ4_hashPosition(ip, tableType); 756 | 757 | /* Main Loop */ 758 | for ( ; ; ) { 759 | const BYTE* match; 760 | BYTE* token; 761 | 762 | /* Find a match */ 763 | { const BYTE* forwardIp = ip; 764 | unsigned step = 1; 765 | unsigned searchMatchNb = 1 << LZ4_skipTrigger; 766 | 767 | do { 768 | U32 h = forwardH; 769 | ip = forwardIp; 770 | forwardIp += step; 771 | step = (searchMatchNb++ >> LZ4_skipTrigger); 772 | 773 | if (unlikely(forwardIp > mflimit)) goto _last_literals; 774 | 775 | match = LZ4_getPositionOnHash(h, ctx->hashTable, tableType, base); 776 | forwardH = LZ4_hashPosition(forwardIp, tableType); 777 | LZ4_putPositionOnHash(ip, h, ctx->hashTable, tableType, base); 778 | 779 | } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) 780 | || (LZ4_read32(match) != LZ4_read32(ip)) ); 781 | } 782 | 783 | /* Catch up */ 784 | while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } 785 | 786 | /* Encode Literal length */ 787 | { unsigned litLength = (unsigned)(ip - anchor); 788 | token = op++; 789 | if (op + ((litLength+240)/255) + litLength > oMaxLit) { 790 | /* Not enough space for a last match */ 791 | op--; 792 | goto _last_literals; 793 | } 794 | if (litLength>=RUN_MASK) { 795 | unsigned len = litLength - RUN_MASK; 796 | *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; 798 | *op++ = (BYTE)len; 799 | } 800 | else *token = (BYTE)(litLength< oMaxMatch) { 815 | /* Match description too long : reduce it */ 816 | matchLength = (15-1) + (oMaxMatch-op) * 255; 817 | } 818 | ip += MINMATCH + matchLength; 819 | 820 | if (matchLength>=ML_MASK) { 821 | *token += ML_MASK; 822 | matchLength -= ML_MASK; 823 | while (matchLength >= 255) { matchLength-=255; *op++ = 255; } 824 | *op++ = (BYTE)matchLength; 825 | } 826 | else *token += (BYTE)(matchLength); 827 | } 828 | 829 | anchor = ip; 830 | 831 | /* Test end of block */ 832 | if (ip > mflimit) break; 833 | if (op > oMaxSeq) break; 834 | 835 | /* Fill table */ 836 | LZ4_putPosition(ip-2, ctx->hashTable, tableType, base); 837 | 838 | /* Test next position */ 839 | match = LZ4_getPosition(ip, ctx->hashTable, tableType, base); 840 | LZ4_putPosition(ip, ctx->hashTable, tableType, base); 841 | if ( (match+MAX_DISTANCE>=ip) 842 | && (LZ4_read32(match)==LZ4_read32(ip)) ) 843 | { token=op++; *token=0; goto _next_match; } 844 | 845 | /* Prepare next loop */ 846 | forwardH = LZ4_hashPosition(++ip, tableType); 847 | } 848 | 849 | _last_literals: 850 | /* Encode Last Literals */ 851 | { size_t lastRunSize = (size_t)(iend - anchor); 852 | if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { 853 | /* adapt lastRunSize to fill 'dst' */ 854 | lastRunSize = (oend-op) - 1; 855 | lastRunSize -= (lastRunSize+240)/255; 856 | } 857 | ip = anchor + lastRunSize; 858 | 859 | if (lastRunSize >= RUN_MASK) { 860 | size_t accumulator = lastRunSize - RUN_MASK; 861 | *op++ = RUN_MASK << ML_BITS; 862 | for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; 863 | *op++ = (BYTE) accumulator; 864 | } else { 865 | *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */ 882 | return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); 883 | } else { 884 | if (*srcSizePtr < LZ4_64Klimit) 885 | return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, byU16); 886 | else 887 | return LZ4_compress_destSize_generic(&state->internal_donotuse, src, dst, srcSizePtr, targetDstSize, sizeof(void*)==8 ? byU32 : byPtr); 888 | } 889 | } 890 | 891 | 892 | int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) 893 | { 894 | #if (HEAPMODE) 895 | LZ4_stream_t* ctx = (LZ4_stream_t*)ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ 896 | #else 897 | LZ4_stream_t ctxBody; 898 | LZ4_stream_t* ctx = &ctxBody; 899 | #endif 900 | 901 | int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); 902 | 903 | #if (HEAPMODE) 904 | FREEMEM(ctx); 905 | #endif 906 | return result; 907 | } 908 | 909 | 910 | 911 | /*-****************************** 912 | * Streaming functions 913 | ********************************/ 914 | 915 | LZ4_stream_t* LZ4_createStream(void) 916 | { 917 | LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); 918 | LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ 919 | LZ4_resetStream(lz4s); 920 | return lz4s; 921 | } 922 | 923 | void LZ4_resetStream (LZ4_stream_t* LZ4_stream) 924 | { 925 | MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); 926 | } 927 | 928 | int LZ4_freeStream (LZ4_stream_t* LZ4_stream) 929 | { 930 | FREEMEM(LZ4_stream); 931 | return (0); 932 | } 933 | 934 | 935 | #define HASH_UNIT sizeof(reg_t) 936 | int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) 937 | { 938 | LZ4_stream_t_internal* dict = &LZ4_dict->internal_donotuse; 939 | const BYTE* p = (const BYTE*)dictionary; 940 | const BYTE* const dictEnd = p + dictSize; 941 | const BYTE* base; 942 | 943 | if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ 944 | LZ4_resetStream(LZ4_dict); 945 | 946 | if (dictSize < (int)HASH_UNIT) { 947 | dict->dictionary = NULL; 948 | dict->dictSize = 0; 949 | return 0; 950 | } 951 | 952 | if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; 953 | dict->currentOffset += 64 KB; 954 | base = p - dict->currentOffset; 955 | dict->dictionary = p; 956 | dict->dictSize = (U32)(dictEnd - p); 957 | dict->currentOffset += dict->dictSize; 958 | 959 | while (p <= dictEnd-HASH_UNIT) { 960 | LZ4_putPosition(p, dict->hashTable, byU32, base); 961 | p+=3; 962 | } 963 | 964 | return dict->dictSize; 965 | } 966 | 967 | 968 | static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) 969 | { 970 | if ((LZ4_dict->currentOffset > 0x80000000) || 971 | ((uptrval)LZ4_dict->currentOffset > (uptrval)src)) { /* address space overflow */ 972 | /* rescale hash table */ 973 | U32 const delta = LZ4_dict->currentOffset - 64 KB; 974 | const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; 975 | int i; 976 | for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; 978 | else LZ4_dict->hashTable[i] -= delta; 979 | } 980 | LZ4_dict->currentOffset = 64 KB; 981 | if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; 982 | LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; 983 | } 984 | } 985 | 986 | 987 | int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) 988 | { 989 | LZ4_stream_t_internal* streamPtr = &LZ4_stream->internal_donotuse; 990 | const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; 991 | 992 | const BYTE* smallest = (const BYTE*) source; 993 | if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ 994 | if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; 995 | LZ4_renormDictT(streamPtr, smallest); 996 | if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; 997 | 998 | /* Check overlapping input/dictionary space */ 999 | { const BYTE* sourceEnd = (const BYTE*) source + inputSize; 1000 | if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { 1001 | streamPtr->dictSize = (U32)(dictEnd - sourceEnd); 1002 | if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; 1003 | if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; 1004 | streamPtr->dictionary = dictEnd - streamPtr->dictSize; 1005 | } 1006 | } 1007 | 1008 | /* prefix mode : source data follows dictionary */ 1009 | if (dictEnd == (const BYTE*)source) { 1010 | int result; 1011 | if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) 1012 | result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); 1013 | else 1014 | result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); 1015 | streamPtr->dictSize += (U32)inputSize; 1016 | streamPtr->currentOffset += (U32)inputSize; 1017 | return result; 1018 | } 1019 | 1020 | /* external dictionary mode */ 1021 | { int result; 1022 | if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) 1023 | result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); 1024 | else 1025 | result = LZ4_compress_generic(streamPtr, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); 1026 | streamPtr->dictionary = (const BYTE*)source; 1027 | streamPtr->dictSize = (U32)inputSize; 1028 | streamPtr->currentOffset += (U32)inputSize; 1029 | return result; 1030 | } 1031 | } 1032 | 1033 | 1034 | /* Hidden debug function, to force external dictionary mode */ 1035 | int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) 1036 | { 1037 | LZ4_stream_t_internal* streamPtr = &LZ4_dict->internal_donotuse; 1038 | int result; 1039 | const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; 1040 | 1041 | const BYTE* smallest = dictEnd; 1042 | if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; 1043 | LZ4_renormDictT(streamPtr, smallest); 1044 | 1045 | result = LZ4_compress_generic(streamPtr, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); 1046 | 1047 | streamPtr->dictionary = (const BYTE*)source; 1048 | streamPtr->dictSize = (U32)inputSize; 1049 | streamPtr->currentOffset += (U32)inputSize; 1050 | 1051 | return result; 1052 | } 1053 | 1054 | 1055 | /*! LZ4_saveDict() : 1056 | * If previously compressed data block is not guaranteed to remain available at its memory location, 1057 | * save it into a safer place (char* safeBuffer). 1058 | * Note : you don't need to call LZ4_loadDict() afterwards, 1059 | * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). 1060 | * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. 1061 | */ 1062 | int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) 1063 | { 1064 | LZ4_stream_t_internal* const dict = &LZ4_dict->internal_donotuse; 1065 | const BYTE* const previousDictEnd = dict->dictionary + dict->dictSize; 1066 | 1067 | if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ 1068 | if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; 1069 | 1070 | memmove(safeBuffer, previousDictEnd - dictSize, dictSize); 1071 | 1072 | dict->dictionary = (const BYTE*)safeBuffer; 1073 | dict->dictSize = (U32)dictSize; 1074 | 1075 | return dictSize; 1076 | } 1077 | 1078 | 1079 | 1080 | /*-***************************** 1081 | * Decompression functions 1082 | *******************************/ 1083 | /*! LZ4_decompress_generic() : 1084 | * This generic decompression function cover all use cases. 1085 | * It shall be instantiated several times, using different sets of directives 1086 | * Note that it is important this generic function is really inlined, 1087 | * in order to remove useless branches during compilation optimization. 1088 | */ 1089 | FORCE_INLINE int LZ4_decompress_generic( 1090 | const char* const source, 1091 | char* const dest, 1092 | int inputSize, 1093 | int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ 1094 | 1095 | int endOnInput, /* endOnOutputSize, endOnInputSize */ 1096 | int partialDecoding, /* full, partial */ 1097 | int targetOutputSize, /* only used if partialDecoding==partial */ 1098 | int dict, /* noDict, withPrefix64k, usingExtDict */ 1099 | const BYTE* const lowPrefix, /* == dest when no prefix */ 1100 | const BYTE* const dictStart, /* only if dict==usingExtDict */ 1101 | const size_t dictSize /* note : = 0 if noDict */ 1102 | ) 1103 | { 1104 | /* Local Variables */ 1105 | const BYTE* ip = (const BYTE*) source; 1106 | const BYTE* const iend = ip + inputSize; 1107 | 1108 | BYTE* op = (BYTE*) dest; 1109 | BYTE* const oend = op + outputSize; 1110 | BYTE* cpy; 1111 | BYTE* oexit = op + targetOutputSize; 1112 | const BYTE* const lowLimit = lowPrefix - dictSize; 1113 | 1114 | const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; 1115 | const unsigned dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; 1116 | const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; 1117 | 1118 | const int safeDecode = (endOnInput==endOnInputSize); 1119 | const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); 1120 | 1121 | 1122 | /* Special cases */ 1123 | if ((partialDecoding) && (oexit > oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ 1124 | if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ 1125 | if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); 1126 | 1127 | /* Main Loop : decode sequences */ 1128 | while (1) { 1129 | size_t length; 1130 | const BYTE* match; 1131 | size_t offset; 1132 | 1133 | /* get literal length */ 1134 | unsigned const token = *ip++; 1135 | if ((length=(token>>ML_BITS)) == RUN_MASK) { 1136 | unsigned s; 1137 | do { 1138 | s = *ip++; 1139 | length += s; 1140 | } while ( likely(endOnInput ? ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) 1148 | || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)) ) 1149 | { 1150 | if (partialDecoding) { 1151 | if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ 1152 | if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ 1153 | } else { 1154 | if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ 1155 | if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ 1156 | } 1157 | memcpy(op, ip, length); 1158 | ip += length; 1159 | op += length; 1160 | break; /* Necessarily EOF, due to parsing restrictions */ 1161 | } 1162 | LZ4_wildCopy(op, ip, cpy); 1163 | ip += length; op = cpy; 1164 | 1165 | /* get offset */ 1166 | offset = LZ4_readLE16(ip); ip+=2; 1167 | match = op - offset; 1168 | if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ 1169 | LZ4_write32(op, (U32)offset); /* costs ~1%; silence an msan warning when offset==0 */ 1170 | 1171 | /* get matchlength */ 1172 | length = token & ML_MASK; 1173 | if (length == ML_MASK) { 1174 | unsigned s; 1175 | do { 1176 | s = *ip++; 1177 | if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; 1178 | length += s; 1179 | } while (s==255); 1180 | if ((safeDecode) && unlikely((uptrval)(op)+length<(uptrval)op)) goto _output_error; /* overflow detection */ 1181 | } 1182 | length += MINMATCH; 1183 | 1184 | /* check external dictionary */ 1185 | if ((dict==usingExtDict) && (match < lowPrefix)) { 1186 | if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ 1187 | 1188 | if (length <= (size_t)(lowPrefix-match)) { 1189 | /* match can be copied as a single segment from external dictionary */ 1190 | memmove(op, dictEnd - (lowPrefix-match), length); 1191 | op += length; 1192 | } else { 1193 | /* match encompass external dictionary and current block */ 1194 | size_t const copySize = (size_t)(lowPrefix-match); 1195 | size_t const restSize = length - copySize; 1196 | memcpy(op, dictEnd - copySize, copySize); 1197 | op += copySize; 1198 | if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */ 1199 | BYTE* const endOfMatch = op + restSize; 1200 | const BYTE* copyFrom = lowPrefix; 1201 | while (op < endOfMatch) *op++ = *copyFrom++; 1202 | } else { 1203 | memcpy(op, lowPrefix, restSize); 1204 | op += restSize; 1205 | } } 1206 | continue; 1207 | } 1208 | 1209 | /* copy match within block */ 1210 | cpy = op + length; 1211 | if (unlikely(offset<8)) { 1212 | const int dec64 = dec64table[offset]; 1213 | op[0] = match[0]; 1214 | op[1] = match[1]; 1215 | op[2] = match[2]; 1216 | op[3] = match[3]; 1217 | match += dec32table[offset]; 1218 | memcpy(op+4, match, 4); 1219 | match -= dec64; 1220 | } else { LZ4_copy8(op, match); match+=8; } 1221 | op += 8; 1222 | 1223 | if (unlikely(cpy>oend-12)) { 1224 | BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); 1225 | if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ 1226 | if (op < oCopyLimit) { 1227 | LZ4_wildCopy(op, match, oCopyLimit); 1228 | match += oCopyLimit - op; 1229 | op = oCopyLimit; 1230 | } 1231 | while (op16) LZ4_wildCopy(op+8, match+8, cpy); 1235 | } 1236 | op=cpy; /* correction */ 1237 | } 1238 | 1239 | /* end of decoding */ 1240 | if (endOnInput) 1241 | return (int) (((char*)op)-dest); /* Nb of output bytes decoded */ 1242 | else 1243 | return (int) (((const char*)ip)-source); /* Nb of input bytes read */ 1244 | 1245 | /* Overflow error detected */ 1246 | _output_error: 1247 | return (int) (-(((const char*)ip)-source))-1; 1248 | } 1249 | 1250 | 1251 | int LZ4_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize) 1252 | { 1253 | return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, full, 0, noDict, (BYTE*)dest, NULL, 0); 1254 | } 1255 | 1256 | int LZ4_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize) 1257 | { 1258 | return LZ4_decompress_generic(source, dest, compressedSize, maxDecompressedSize, endOnInputSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0); 1259 | } 1260 | 1261 | int LZ4_decompress_fast(const char* source, char* dest, int originalSize) 1262 | { 1263 | return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)(dest - 64 KB), NULL, 64 KB); 1264 | } 1265 | 1266 | 1267 | /*===== streaming decompression functions =====*/ 1268 | 1269 | /* 1270 | * If you prefer dynamic allocation methods, 1271 | * LZ4_createStreamDecode() 1272 | * provides a pointer (void*) towards an initialized LZ4_streamDecode_t structure. 1273 | */ 1274 | LZ4_streamDecode_t* LZ4_createStreamDecode(void) 1275 | { 1276 | LZ4_streamDecode_t* lz4s = (LZ4_streamDecode_t*) ALLOCATOR(1, sizeof(LZ4_streamDecode_t)); 1277 | return lz4s; 1278 | } 1279 | 1280 | int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream) 1281 | { 1282 | FREEMEM(LZ4_stream); 1283 | return 0; 1284 | } 1285 | 1286 | /*! 1287 | * LZ4_setStreamDecode() : 1288 | * Use this function to instruct where to find the dictionary. 1289 | * This function is not necessary if previous data is still available where it was decoded. 1290 | * Loading a size of 0 is allowed (same effect as no dictionary). 1291 | * Return : 1 if OK, 0 if error 1292 | */ 1293 | int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize) 1294 | { 1295 | LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; 1296 | lz4sd->prefixSize = (size_t) dictSize; 1297 | lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; 1298 | lz4sd->externalDict = NULL; 1299 | lz4sd->extDictSize = 0; 1300 | return 1; 1301 | } 1302 | 1303 | /* 1304 | *_continue() : 1305 | These decoding functions allow decompression of multiple blocks in "streaming" mode. 1306 | Previously decoded blocks must still be available at the memory position where they were decoded. 1307 | If it's not possible, save the relevant part of decoded data into a safe buffer, 1308 | and indicate where it stands using LZ4_setStreamDecode() 1309 | */ 1310 | int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) 1311 | { 1312 | LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; 1313 | int result; 1314 | 1315 | if (lz4sd->prefixEnd == (BYTE*)dest) { 1316 | result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, 1317 | endOnInputSize, full, 0, 1318 | usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); 1319 | if (result <= 0) return result; 1320 | lz4sd->prefixSize += result; 1321 | lz4sd->prefixEnd += result; 1322 | } else { 1323 | lz4sd->extDictSize = lz4sd->prefixSize; 1324 | lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; 1325 | result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, 1326 | endOnInputSize, full, 0, 1327 | usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); 1328 | if (result <= 0) return result; 1329 | lz4sd->prefixSize = result; 1330 | lz4sd->prefixEnd = (BYTE*)dest + result; 1331 | } 1332 | 1333 | return result; 1334 | } 1335 | 1336 | int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) 1337 | { 1338 | LZ4_streamDecode_t_internal* lz4sd = &LZ4_streamDecode->internal_donotuse; 1339 | int result; 1340 | 1341 | if (lz4sd->prefixEnd == (BYTE*)dest) { 1342 | result = LZ4_decompress_generic(source, dest, 0, originalSize, 1343 | endOnOutputSize, full, 0, 1344 | usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); 1345 | if (result <= 0) return result; 1346 | lz4sd->prefixSize += originalSize; 1347 | lz4sd->prefixEnd += originalSize; 1348 | } else { 1349 | lz4sd->extDictSize = lz4sd->prefixSize; 1350 | lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; 1351 | result = LZ4_decompress_generic(source, dest, 0, originalSize, 1352 | endOnOutputSize, full, 0, 1353 | usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); 1354 | if (result <= 0) return result; 1355 | lz4sd->prefixSize = originalSize; 1356 | lz4sd->prefixEnd = (BYTE*)dest + originalSize; 1357 | } 1358 | 1359 | return result; 1360 | } 1361 | 1362 | 1363 | /* 1364 | Advanced decoding functions : 1365 | *_usingDict() : 1366 | These decoding functions work the same as "_continue" ones, 1367 | the dictionary must be explicitly provided within parameters 1368 | */ 1369 | 1370 | FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) 1371 | { 1372 | if (dictSize==0) 1373 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); 1374 | if (dictStart+dictSize == dest) { 1375 | if (dictSize >= (int)(64 KB - 1)) 1376 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); 1377 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); 1378 | } 1379 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); 1380 | } 1381 | 1382 | int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) 1383 | { 1384 | return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); 1385 | } 1386 | 1387 | int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) 1388 | { 1389 | return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); 1390 | } 1391 | 1392 | /* debug function */ 1393 | int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) 1394 | { 1395 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); 1396 | } 1397 | 1398 | 1399 | /*=************************************************* 1400 | * Obsolete Functions 1401 | ***************************************************/ 1402 | /* obsolete compression functions */ 1403 | int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } 1404 | int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } 1405 | int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } 1406 | int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } 1407 | int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } 1408 | int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } 1409 | 1410 | /* 1411 | These function names are deprecated and should no longer be used. 1412 | They are only provided here for compatibility with older user programs. 1413 | - LZ4_uncompress is totally equivalent to LZ4_decompress_fast 1414 | - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe 1415 | */ 1416 | int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } 1417 | int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } 1418 | 1419 | 1420 | /* Obsolete Streaming functions */ 1421 | 1422 | int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } 1423 | 1424 | static void LZ4_init(LZ4_stream_t* lz4ds, BYTE* base) 1425 | { 1426 | MEM_INIT(lz4ds, 0, sizeof(LZ4_stream_t)); 1427 | lz4ds->internal_donotuse.bufferStart = base; 1428 | } 1429 | 1430 | int LZ4_resetStreamState(void* state, char* inputBuffer) 1431 | { 1432 | if ((((uptrval)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ 1433 | LZ4_init((LZ4_stream_t*)state, (BYTE*)inputBuffer); 1434 | return 0; 1435 | } 1436 | 1437 | void* LZ4_create (char* inputBuffer) 1438 | { 1439 | LZ4_stream_t* lz4ds = (LZ4_stream_t*)ALLOCATOR(8, sizeof(LZ4_stream_t)); 1440 | LZ4_init (lz4ds, (BYTE*)inputBuffer); 1441 | return lz4ds; 1442 | } 1443 | 1444 | char* LZ4_slideInputBuffer (void* LZ4_Data) 1445 | { 1446 | LZ4_stream_t_internal* ctx = &((LZ4_stream_t*)LZ4_Data)->internal_donotuse; 1447 | int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); 1448 | return (char*)(ctx->bufferStart + dictSize); 1449 | } 1450 | 1451 | /* Obsolete streaming decompression functions */ 1452 | 1453 | int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) 1454 | { 1455 | return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); 1456 | } 1457 | 1458 | int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) 1459 | { 1460 | return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); 1461 | } 1462 | 1463 | #endif /* LZ4_COMMONDEFS_ONLY */ 1464 | -------------------------------------------------------------------------------- /c_src/lz4.h: -------------------------------------------------------------------------------- 1 | /* 2 | * LZ4 - Fast LZ compression algorithm 3 | * Header File 4 | * Copyright (C) 2011-2016, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - LZ4 homepage : http://www.lz4.org 33 | - LZ4 source repository : https://github.com/lz4/lz4 34 | */ 35 | #ifndef LZ4_H_2983827168210 36 | #define LZ4_H_2983827168210 37 | 38 | #if defined (__cplusplus) 39 | extern "C" { 40 | #endif 41 | 42 | /* --- Dependency --- */ 43 | #include /* size_t */ 44 | 45 | 46 | /** 47 | Introduction 48 | 49 | LZ4 is lossless compression algorithm, providing compression speed at 400 MB/s per core, 50 | scalable with multi-cores CPU. It features an extremely fast decoder, with speed in 51 | multiple GB/s per core, typically reaching RAM speed limits on multi-core systems. 52 | 53 | The LZ4 compression library provides in-memory compression and decompression functions. 54 | Compression can be done in: 55 | - a single step (described as Simple Functions) 56 | - a single step, reusing a context (described in Advanced Functions) 57 | - unbounded multiple steps (described as Streaming compression) 58 | 59 | lz4.h provides block compression functions. It gives full buffer control to user. 60 | Decompressing an lz4-compressed block also requires metadata (such as compressed size). 61 | Each application is free to encode such metadata in whichever way it wants. 62 | 63 | An additional format, called LZ4 frame specification (doc/lz4_Frame_format.md), 64 | take care of encoding standard metadata alongside LZ4-compressed blocks. 65 | If your application requires interoperability, it's recommended to use it. 66 | A library is provided to take care of it, see lz4frame.h. 67 | */ 68 | 69 | /*^*************************************************************** 70 | * Export parameters 71 | *****************************************************************/ 72 | /* 73 | * LZ4_DLL_EXPORT : 74 | * Enable exporting of functions when building a Windows DLL 75 | */ 76 | #if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT==1) 77 | # define LZ4LIB_API __declspec(dllexport) 78 | #elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT==1) 79 | # define LZ4LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ 80 | #else 81 | # define LZ4LIB_API 82 | #endif 83 | 84 | 85 | /*========== Version =========== */ 86 | #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ 87 | #define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ 88 | #define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */ 89 | 90 | #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) 91 | 92 | #define LZ4_LIB_VERSION LZ4_VERSION_MAJOR.LZ4_VERSION_MINOR.LZ4_VERSION_RELEASE 93 | #define LZ4_QUOTE(str) #str 94 | #define LZ4_EXPAND_AND_QUOTE(str) LZ4_QUOTE(str) 95 | #define LZ4_VERSION_STRING LZ4_EXPAND_AND_QUOTE(LZ4_LIB_VERSION) 96 | 97 | LZ4LIB_API int LZ4_versionNumber (void); 98 | LZ4LIB_API const char* LZ4_versionString (void); 99 | 100 | 101 | /*-************************************ 102 | * Tuning parameter 103 | **************************************/ 104 | /*! 105 | * LZ4_MEMORY_USAGE : 106 | * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) 107 | * Increasing memory usage improves compression ratio 108 | * Reduced memory usage can improve speed, due to cache effect 109 | * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache 110 | */ 111 | #define LZ4_MEMORY_USAGE 14 112 | 113 | 114 | /*-************************************ 115 | * Simple Functions 116 | **************************************/ 117 | /*! LZ4_compress_default() : 118 | Compresses 'sourceSize' bytes from buffer 'source' 119 | into already allocated 'dest' buffer of size 'maxDestSize'. 120 | Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). 121 | It also runs faster, so it's a recommended setting. 122 | If the function cannot compress 'source' into a more limited 'dest' budget, 123 | compression stops *immediately*, and the function result is zero. 124 | As a consequence, 'dest' content is not valid. 125 | This function never writes outside 'dest' buffer, nor read outside 'source' buffer. 126 | sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE 127 | maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) 128 | return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) 129 | or 0 if compression fails */ 130 | LZ4LIB_API int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); 131 | 132 | /*! LZ4_decompress_safe() : 133 | compressedSize : is the precise full size of the compressed block. 134 | maxDecompressedSize : is the size of destination buffer, which must be already allocated. 135 | return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) 136 | If destination buffer is not large enough, decoding will stop and output an error code (<0). 137 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 138 | This function is protected against buffer overflow exploits, including malicious data packets. 139 | It never writes outside output buffer, nor reads outside input buffer. 140 | */ 141 | LZ4LIB_API int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); 142 | 143 | 144 | /*-************************************ 145 | * Advanced Functions 146 | **************************************/ 147 | #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ 148 | #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) 149 | 150 | /*! 151 | LZ4_compressBound() : 152 | Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) 153 | This function is primarily useful for memory allocation purposes (destination buffer size). 154 | Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). 155 | Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) 156 | inputSize : max supported value is LZ4_MAX_INPUT_SIZE 157 | return : maximum output size in a "worst case" scenario 158 | or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) 159 | */ 160 | LZ4LIB_API int LZ4_compressBound(int inputSize); 161 | 162 | /*! 163 | LZ4_compress_fast() : 164 | Same as LZ4_compress_default(), but allows to select an "acceleration" factor. 165 | The larger the acceleration value, the faster the algorithm, but also the lesser the compression. 166 | It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. 167 | An acceleration value of "1" is the same as regular LZ4_compress_default() 168 | Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. 169 | */ 170 | LZ4LIB_API int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); 171 | 172 | 173 | /*! 174 | LZ4_compress_fast_extState() : 175 | Same compression function, just using an externally allocated memory space to store compression state. 176 | Use LZ4_sizeofState() to know how much memory must be allocated, 177 | and allocate it on 8-bytes boundaries (using malloc() typically). 178 | Then, provide it as 'void* state' to compression function. 179 | */ 180 | LZ4LIB_API int LZ4_sizeofState(void); 181 | LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); 182 | 183 | 184 | /*! 185 | LZ4_compress_destSize() : 186 | Reverse the logic, by compressing as much data as possible from 'source' buffer 187 | into already allocated buffer 'dest' of size 'targetDestSize'. 188 | This function either compresses the entire 'source' content into 'dest' if it's large enough, 189 | or fill 'dest' buffer completely with as much data as possible from 'source'. 190 | *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. 191 | New value is necessarily <= old value. 192 | return : Nb bytes written into 'dest' (necessarily <= targetDestSize) 193 | or 0 if compression fails 194 | */ 195 | LZ4LIB_API int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); 196 | 197 | 198 | /*! 199 | LZ4_decompress_fast() : 200 | originalSize : is the original and therefore uncompressed size 201 | return : the number of bytes read from the source buffer (in other words, the compressed size) 202 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 203 | Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. 204 | note : This function fully respect memory boundaries for properly formed compressed data. 205 | It is a bit faster than LZ4_decompress_safe(). 206 | However, it does not provide any protection against intentionally modified data stream (malicious input). 207 | Use this function in trusted environment only (data to decode comes from a trusted source). 208 | */ 209 | LZ4LIB_API int LZ4_decompress_fast (const char* source, char* dest, int originalSize); 210 | 211 | /*! 212 | LZ4_decompress_safe_partial() : 213 | This function decompress a compressed block of size 'compressedSize' at position 'source' 214 | into destination buffer 'dest' of size 'maxDecompressedSize'. 215 | The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, 216 | reducing decompression time. 217 | return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) 218 | Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. 219 | Always control how many bytes were decoded. 220 | If the source stream is detected malformed, the function will stop decoding and return a negative result. 221 | This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets 222 | */ 223 | LZ4LIB_API int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); 224 | 225 | 226 | /*-********************************************* 227 | * Streaming Compression Functions 228 | ***********************************************/ 229 | typedef union LZ4_stream_u LZ4_stream_t; /* incomplete type (defined later) */ 230 | 231 | /*! LZ4_createStream() and LZ4_freeStream() : 232 | * LZ4_createStream() will allocate and initialize an `LZ4_stream_t` structure. 233 | * LZ4_freeStream() releases its memory. 234 | */ 235 | LZ4LIB_API LZ4_stream_t* LZ4_createStream(void); 236 | LZ4LIB_API int LZ4_freeStream (LZ4_stream_t* streamPtr); 237 | 238 | /*! LZ4_resetStream() : 239 | * An LZ4_stream_t structure can be allocated once and re-used multiple times. 240 | * Use this function to init an allocated `LZ4_stream_t` structure and start a new compression. 241 | */ 242 | LZ4LIB_API void LZ4_resetStream (LZ4_stream_t* streamPtr); 243 | 244 | /*! LZ4_loadDict() : 245 | * Use this function to load a static dictionary into LZ4_stream. 246 | * Any previous data will be forgotten, only 'dictionary' will remain in memory. 247 | * Loading a size of 0 is allowed. 248 | * Return : dictionary size, in bytes (necessarily <= 64 KB) 249 | */ 250 | LZ4LIB_API int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); 251 | 252 | /*! LZ4_compress_fast_continue() : 253 | * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. 254 | * Important : Previous data blocks are assumed to still be present and unmodified ! 255 | * 'dst' buffer must be already allocated. 256 | * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. 257 | * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. 258 | */ 259 | LZ4LIB_API int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); 260 | 261 | /*! LZ4_saveDict() : 262 | * If previously compressed data block is not guaranteed to remain available at its memory location, 263 | * save it into a safer place (char* safeBuffer). 264 | * Note : you don't need to call LZ4_loadDict() afterwards, 265 | * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue(). 266 | * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error. 267 | */ 268 | LZ4LIB_API int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); 269 | 270 | 271 | /*-********************************************** 272 | * Streaming Decompression Functions 273 | * Bufferless synchronous API 274 | ************************************************/ 275 | typedef union LZ4_streamDecode_u LZ4_streamDecode_t; /* incomplete type (defined later) */ 276 | 277 | /* creation / destruction of streaming decompression tracking structure */ 278 | LZ4LIB_API LZ4_streamDecode_t* LZ4_createStreamDecode(void); 279 | LZ4LIB_API int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); 280 | 281 | /*! LZ4_setStreamDecode() : 282 | * Use this function to instruct where to find the dictionary. 283 | * Setting a size of 0 is allowed (same effect as reset). 284 | * @return : 1 if OK, 0 if error 285 | */ 286 | LZ4LIB_API int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); 287 | 288 | /*! 289 | LZ4_decompress_*_continue() : 290 | These decoding functions allow decompression of multiple blocks in "streaming" mode. 291 | Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) 292 | In the case of a ring buffers, decoding buffer must be either : 293 | - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) 294 | In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). 295 | - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. 296 | maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. 297 | In which case, encoding and decoding buffers do not need to be synchronized, 298 | and encoding ring buffer can have any size, including small ones ( < 64 KB). 299 | - _At least_ 64 KB + 8 bytes + maxBlockSize. 300 | In which case, encoding and decoding buffers do not need to be synchronized, 301 | and encoding ring buffer can have any size, including larger than decoding buffer. 302 | Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, 303 | and indicate where it is saved using LZ4_setStreamDecode() 304 | */ 305 | LZ4LIB_API int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); 306 | LZ4LIB_API int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); 307 | 308 | 309 | /*! LZ4_decompress_*_usingDict() : 310 | * These decoding functions work the same as 311 | * a combination of LZ4_setStreamDecode() followed by LZ4_decompress_*_continue() 312 | * They are stand-alone, and don't need an LZ4_streamDecode_t structure. 313 | */ 314 | LZ4LIB_API int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); 315 | LZ4LIB_API int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); 316 | 317 | 318 | /*^********************************************** 319 | * !!!!!! STATIC LINKING ONLY !!!!!! 320 | ***********************************************/ 321 | /*-************************************ 322 | * Private definitions 323 | ************************************** 324 | * Do not use these definitions. 325 | * They are exposed to allow static allocation of `LZ4_stream_t` and `LZ4_streamDecode_t`. 326 | * Using these definitions will expose code to API and/or ABI break in future versions of the library. 327 | **************************************/ 328 | #define LZ4_HASHLOG (LZ4_MEMORY_USAGE-2) 329 | #define LZ4_HASHTABLESIZE (1 << LZ4_MEMORY_USAGE) 330 | #define LZ4_HASH_SIZE_U32 (1 << LZ4_HASHLOG) /* required as macro for static allocation */ 331 | 332 | #if defined(__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 333 | #include 334 | 335 | typedef struct { 336 | uint32_t hashTable[LZ4_HASH_SIZE_U32]; 337 | uint32_t currentOffset; 338 | uint32_t initCheck; 339 | const uint8_t* dictionary; 340 | uint8_t* bufferStart; /* obsolete, used for slideInputBuffer */ 341 | uint32_t dictSize; 342 | } LZ4_stream_t_internal; 343 | 344 | typedef struct { 345 | const uint8_t* externalDict; 346 | size_t extDictSize; 347 | const uint8_t* prefixEnd; 348 | size_t prefixSize; 349 | } LZ4_streamDecode_t_internal; 350 | 351 | #else 352 | 353 | typedef struct { 354 | unsigned int hashTable[LZ4_HASH_SIZE_U32]; 355 | unsigned int currentOffset; 356 | unsigned int initCheck; 357 | const unsigned char* dictionary; 358 | unsigned char* bufferStart; /* obsolete, used for slideInputBuffer */ 359 | unsigned int dictSize; 360 | } LZ4_stream_t_internal; 361 | 362 | typedef struct { 363 | const unsigned char* externalDict; 364 | size_t extDictSize; 365 | const unsigned char* prefixEnd; 366 | size_t prefixSize; 367 | } LZ4_streamDecode_t_internal; 368 | 369 | #endif 370 | 371 | /*! 372 | * LZ4_stream_t : 373 | * information structure to track an LZ4 stream. 374 | * init this structure before first use. 375 | * note : only use in association with static linking ! 376 | * this definition is not API/ABI safe, 377 | * and may change in a future version ! 378 | */ 379 | #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) 380 | #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(unsigned long long)) 381 | union LZ4_stream_u { 382 | unsigned long long table[LZ4_STREAMSIZE_U64]; 383 | LZ4_stream_t_internal internal_donotuse; 384 | } ; /* previously typedef'd to LZ4_stream_t */ 385 | 386 | 387 | /*! 388 | * LZ4_streamDecode_t : 389 | * information structure to track an LZ4 stream during decompression. 390 | * init this structure using LZ4_setStreamDecode (or memset()) before first use 391 | * note : only use in association with static linking ! 392 | * this definition is not API/ABI safe, 393 | * and may change in a future version ! 394 | */ 395 | #define LZ4_STREAMDECODESIZE_U64 4 396 | #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) 397 | union LZ4_streamDecode_u { 398 | unsigned long long table[LZ4_STREAMDECODESIZE_U64]; 399 | LZ4_streamDecode_t_internal internal_donotuse; 400 | } ; /* previously typedef'd to LZ4_streamDecode_t */ 401 | 402 | 403 | /*=************************************ 404 | * Obsolete Functions 405 | **************************************/ 406 | /* Deprecation warnings */ 407 | /* Should these warnings be a problem, 408 | it is generally possible to disable them, 409 | typically with -Wno-deprecated-declarations for gcc 410 | or _CRT_SECURE_NO_WARNINGS in Visual. 411 | Otherwise, it's also possible to define LZ4_DISABLE_DEPRECATE_WARNINGS */ 412 | #ifdef LZ4_DISABLE_DEPRECATE_WARNINGS 413 | # define LZ4_DEPRECATED(message) /* disable deprecation warnings */ 414 | #else 415 | # define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 416 | # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ 417 | # define LZ4_DEPRECATED(message) [[deprecated(message)]] 418 | # elif (LZ4_GCC_VERSION >= 405) || defined(__clang__) 419 | # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) 420 | # elif (LZ4_GCC_VERSION >= 301) 421 | # define LZ4_DEPRECATED(message) __attribute__((deprecated)) 422 | # elif defined(_MSC_VER) 423 | # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) 424 | # else 425 | # pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") 426 | # define LZ4_DEPRECATED(message) 427 | # endif 428 | #endif /* LZ4_DISABLE_DEPRECATE_WARNINGS */ 429 | 430 | /* Obsolete compression functions */ 431 | LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress (const char* source, char* dest, int sourceSize); 432 | LZ4_DEPRECATED("use LZ4_compress_default() instead") int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); 433 | LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); 434 | LZ4_DEPRECATED("use LZ4_compress_fast_extState() instead") int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); 435 | LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); 436 | LZ4_DEPRECATED("use LZ4_compress_fast_continue() instead") int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); 437 | 438 | /* Obsolete decompression functions */ 439 | /* These function names are completely deprecated and must no longer be used. 440 | They are only provided in lz4.c for compatibility with older programs. 441 | - LZ4_uncompress is the same as LZ4_decompress_fast 442 | - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe 443 | These function prototypes are now disabled; uncomment them only if you really need them. 444 | It is highly recommended to stop using these prototypes and migrate to maintained ones */ 445 | /* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ 446 | /* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ 447 | 448 | /* Obsolete streaming functions; use new streaming interface whenever possible */ 449 | LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); 450 | LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); 451 | LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); 452 | LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); 453 | 454 | /* Obsolete streaming decoding functions */ 455 | LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); 456 | LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); 457 | 458 | 459 | #if defined (__cplusplus) 460 | } 461 | #endif 462 | 463 | #endif /* LZ4_H_2983827168210 */ 464 | -------------------------------------------------------------------------------- /c_src/lz4_nif.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "erl_nif.h" 3 | #include "lz4.h" 4 | #include "lz4hc.h" 5 | 6 | static ERL_NIF_TERM nif_compress(ErlNifEnv* env, int argc, 7 | const ERL_NIF_TERM argv[]); 8 | static ERL_NIF_TERM nif_uncompress(ErlNifEnv* env, int argc, 9 | const ERL_NIF_TERM argv[]); 10 | 11 | static ErlNifFunc nif_funcs[] = 12 | { 13 | {"compress", 2, nif_compress}, 14 | {"uncompress", 2, nif_uncompress} 15 | }; 16 | 17 | static ERL_NIF_TERM atom_ok; 18 | static ERL_NIF_TERM atom_error; 19 | static ERL_NIF_TERM atom_high; 20 | static ERL_NIF_TERM atom_compress_failed; 21 | static ERL_NIF_TERM atom_uncompress_failed; 22 | 23 | static ERL_NIF_TERM 24 | nif_compress(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) 25 | { 26 | ERL_NIF_TERM opts_term, head_term, tail_term, ret_term; 27 | ErlNifBinary src_bin, res_bin; 28 | bool high = false; 29 | int real_size; 30 | size_t res_size; 31 | 32 | if (!enif_inspect_binary(env, argv[0], &src_bin) || 33 | !enif_is_list(env, argv[1])) 34 | return 0; 35 | 36 | opts_term = argv[1]; 37 | while (enif_get_list_cell(env, opts_term, &head_term, &tail_term)) { 38 | if (enif_is_identical(head_term, atom_high)) 39 | high = true; 40 | opts_term = tail_term; 41 | } 42 | 43 | res_size = LZ4_compressBound(src_bin.size); 44 | enif_alloc_binary(res_size, &res_bin); 45 | 46 | if (high) 47 | real_size = LZ4_compressHC((char *)src_bin.data, 48 | (char *)res_bin.data, src_bin.size); 49 | else 50 | real_size = LZ4_compress((char *)src_bin.data, 51 | (char *)res_bin.data, src_bin.size); 52 | 53 | if (real_size >= 0) { 54 | enif_realloc_binary(&res_bin, real_size); 55 | ret_term = enif_make_tuple2(env, atom_ok, 56 | enif_make_binary(env, &res_bin)); 57 | enif_release_binary(&res_bin); 58 | return ret_term; 59 | } else { 60 | enif_release_binary(&res_bin); 61 | return enif_make_tuple2(env, atom_error, atom_compress_failed); 62 | } 63 | } 64 | 65 | static ERL_NIF_TERM 66 | nif_uncompress(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) 67 | { 68 | ERL_NIF_TERM ret_term; 69 | ErlNifBinary src_bin, res_bin; 70 | long res_size; 71 | 72 | if (!enif_inspect_binary(env, argv[0], &src_bin) || 73 | !enif_get_long(env, argv[1], &res_size)) 74 | return 0; 75 | 76 | enif_alloc_binary((size_t)res_size, &res_bin); 77 | 78 | int ret_size = LZ4_decompress_safe((char *)src_bin.data, 79 | (char *)res_bin.data, src_bin.size, res_bin.size); 80 | if (ret_size >= 0) { 81 | if (ret_size != res_bin.size) { 82 | if (!enif_realloc_binary(&res_bin, ret_size)) 83 | goto error; 84 | } 85 | ret_term = enif_make_tuple2(env, atom_ok, 86 | enif_make_binary(env, &res_bin)); 87 | enif_release_binary(&res_bin); 88 | return ret_term; 89 | } 90 | 91 | error: 92 | enif_release_binary(&res_bin); 93 | return enif_make_tuple2(env, atom_error, atom_uncompress_failed); 94 | } 95 | 96 | static int on_load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) 97 | { 98 | atom_ok = enif_make_atom(env, "ok"); 99 | atom_error = enif_make_atom(env, "error"); 100 | atom_high = enif_make_atom(env, "high"); 101 | atom_compress_failed = enif_make_atom(env, "compress_failed"); 102 | atom_uncompress_failed = enif_make_atom(env, "uncompress_failed"); 103 | return 0; 104 | } 105 | 106 | ERL_NIF_INIT(lz4, nif_funcs, &on_load, NULL, NULL, NULL); 107 | 108 | -------------------------------------------------------------------------------- /c_src/lz4hc.c: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 HC - High Compression Mode of LZ4 3 | Copyright (C) 2011-2016, Yann Collet. 4 | 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - LZ4 source repository : https://github.com/lz4/lz4 32 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 33 | */ 34 | /* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ 35 | 36 | 37 | /* ************************************* 38 | * Tuning Parameter 39 | ***************************************/ 40 | 41 | /*! 42 | * HEAPMODE : 43 | * Select how default compression function will allocate workplace memory, 44 | * in stack (0:fastest), or in heap (1:requires malloc()). 45 | * Since workplace is rather large, heap mode is recommended. 46 | */ 47 | #ifndef LZ4HC_HEAPMODE 48 | # define LZ4HC_HEAPMODE 1 49 | #endif 50 | 51 | 52 | /* ************************************* 53 | * Dependency 54 | ***************************************/ 55 | #include "lz4hc.h" 56 | 57 | 58 | /* ************************************* 59 | * Local Compiler Options 60 | ***************************************/ 61 | #if defined(__GNUC__) 62 | # pragma GCC diagnostic ignored "-Wunused-function" 63 | #endif 64 | 65 | #if defined (__clang__) 66 | # pragma clang diagnostic ignored "-Wunused-function" 67 | #endif 68 | 69 | 70 | /* ************************************* 71 | * Common LZ4 definition 72 | ***************************************/ 73 | #define LZ4_COMMONDEFS_ONLY 74 | #include "lz4.c" 75 | 76 | 77 | /* ************************************* 78 | * Local Constants 79 | ***************************************/ 80 | #define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH) 81 | 82 | 83 | /************************************** 84 | * Local Macros 85 | **************************************/ 86 | #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) 87 | #define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */ 88 | #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ 89 | 90 | static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } 91 | 92 | 93 | 94 | /************************************** 95 | * HC Compression 96 | **************************************/ 97 | static void LZ4HC_init (LZ4HC_CCtx_internal* hc4, const BYTE* start) 98 | { 99 | MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); 100 | MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); 101 | hc4->nextToUpdate = 64 KB; 102 | hc4->base = start - 64 KB; 103 | hc4->end = start; 104 | hc4->dictBase = start - 64 KB; 105 | hc4->dictLimit = 64 KB; 106 | hc4->lowLimit = 64 KB; 107 | } 108 | 109 | 110 | /* Update chains up to ip (excluded) */ 111 | FORCE_INLINE void LZ4HC_Insert (LZ4HC_CCtx_internal* hc4, const BYTE* ip) 112 | { 113 | U16* const chainTable = hc4->chainTable; 114 | U32* const hashTable = hc4->hashTable; 115 | const BYTE* const base = hc4->base; 116 | U32 const target = (U32)(ip - base); 117 | U32 idx = hc4->nextToUpdate; 118 | 119 | while (idx < target) { 120 | U32 const h = LZ4HC_hashPtr(base+idx); 121 | size_t delta = idx - hashTable[h]; 122 | if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; 123 | DELTANEXTU16(idx) = (U16)delta; 124 | hashTable[h] = idx; 125 | idx++; 126 | } 127 | 128 | hc4->nextToUpdate = target; 129 | } 130 | 131 | 132 | FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_CCtx_internal* hc4, /* Index table will be updated */ 133 | const BYTE* ip, const BYTE* const iLimit, 134 | const BYTE** matchpos, 135 | const int maxNbAttempts) 136 | { 137 | U16* const chainTable = hc4->chainTable; 138 | U32* const HashTable = hc4->hashTable; 139 | const BYTE* const base = hc4->base; 140 | const BYTE* const dictBase = hc4->dictBase; 141 | const U32 dictLimit = hc4->dictLimit; 142 | const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); 143 | U32 matchIndex; 144 | int nbAttempts=maxNbAttempts; 145 | size_t ml=0; 146 | 147 | /* HC4 match finder */ 148 | LZ4HC_Insert(hc4, ip); 149 | matchIndex = HashTable[LZ4HC_hashPtr(ip)]; 150 | 151 | while ((matchIndex>=lowLimit) && (nbAttempts)) { 152 | nbAttempts--; 153 | if (matchIndex >= dictLimit) { 154 | const BYTE* const match = base + matchIndex; 155 | if (*(match+ml) == *(ip+ml) 156 | && (LZ4_read32(match) == LZ4_read32(ip))) 157 | { 158 | size_t const mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; 159 | if (mlt > ml) { ml = mlt; *matchpos = match; } 160 | } 161 | } else { 162 | const BYTE* const match = dictBase + matchIndex; 163 | if (LZ4_read32(match) == LZ4_read32(ip)) { 164 | size_t mlt; 165 | const BYTE* vLimit = ip + (dictLimit - matchIndex); 166 | if (vLimit > iLimit) vLimit = iLimit; 167 | mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; 168 | if ((ip+mlt == vLimit) && (vLimit < iLimit)) 169 | mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit); 170 | if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ 171 | } 172 | } 173 | matchIndex -= DELTANEXTU16(matchIndex); 174 | } 175 | 176 | return (int)ml; 177 | } 178 | 179 | 180 | FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( 181 | LZ4HC_CCtx_internal* hc4, 182 | const BYTE* const ip, 183 | const BYTE* const iLowLimit, 184 | const BYTE* const iHighLimit, 185 | int longest, 186 | const BYTE** matchpos, 187 | const BYTE** startpos, 188 | const int maxNbAttempts) 189 | { 190 | U16* const chainTable = hc4->chainTable; 191 | U32* const HashTable = hc4->hashTable; 192 | const BYTE* const base = hc4->base; 193 | const U32 dictLimit = hc4->dictLimit; 194 | const BYTE* const lowPrefixPtr = base + dictLimit; 195 | const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); 196 | const BYTE* const dictBase = hc4->dictBase; 197 | U32 matchIndex; 198 | int nbAttempts = maxNbAttempts; 199 | int delta = (int)(ip-iLowLimit); 200 | 201 | 202 | /* First Match */ 203 | LZ4HC_Insert(hc4, ip); 204 | matchIndex = HashTable[LZ4HC_hashPtr(ip)]; 205 | 206 | while ((matchIndex>=lowLimit) && (nbAttempts)) { 207 | nbAttempts--; 208 | if (matchIndex >= dictLimit) { 209 | const BYTE* matchPtr = base + matchIndex; 210 | if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) { 211 | if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { 212 | int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); 213 | int back = 0; 214 | 215 | while ((ip+back > iLowLimit) 216 | && (matchPtr+back > lowPrefixPtr) 217 | && (ip[back-1] == matchPtr[back-1])) 218 | back--; 219 | 220 | mlt -= back; 221 | 222 | if (mlt > longest) { 223 | longest = (int)mlt; 224 | *matchpos = matchPtr+back; 225 | *startpos = ip+back; 226 | } 227 | } 228 | } 229 | } else { 230 | const BYTE* const matchPtr = dictBase + matchIndex; 231 | if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { 232 | size_t mlt; 233 | int back=0; 234 | const BYTE* vLimit = ip + (dictLimit - matchIndex); 235 | if (vLimit > iHighLimit) vLimit = iHighLimit; 236 | mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; 237 | if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) 238 | mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); 239 | while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--; 240 | mlt -= back; 241 | if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } 242 | } 243 | } 244 | matchIndex -= DELTANEXTU16(matchIndex); 245 | } 246 | 247 | return longest; 248 | } 249 | 250 | 251 | typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; 252 | 253 | #define LZ4HC_DEBUG 0 254 | #if LZ4HC_DEBUG 255 | static unsigned debug = 0; 256 | #endif 257 | 258 | FORCE_INLINE int LZ4HC_encodeSequence ( 259 | const BYTE** ip, 260 | BYTE** op, 261 | const BYTE** anchor, 262 | int matchLength, 263 | const BYTE* const match, 264 | limitedOutput_directive limitedOutputBuffer, 265 | BYTE* oend) 266 | { 267 | int length; 268 | BYTE* token; 269 | 270 | #if LZ4HC_DEBUG 271 | if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); 272 | #endif 273 | 274 | /* Encode Literal length */ 275 | length = (int)(*ip - *anchor); 276 | token = (*op)++; 277 | if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ 278 | if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } 279 | else *token = (BYTE)(length<>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ 291 | if (length>=(int)ML_MASK) { 292 | *token += ML_MASK; 293 | length -= ML_MASK; 294 | for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } 295 | if (length > 254) { length-=255; *(*op)++ = 255; } 296 | *(*op)++ = (BYTE)length; 297 | } else { 298 | *token += (BYTE)(length); 299 | } 300 | 301 | /* Prepare next loop */ 302 | *ip += matchLength; 303 | *anchor = *ip; 304 | 305 | return 0; 306 | } 307 | 308 | #include "lz4opt.h" 309 | 310 | static int LZ4HC_compress_hashChain ( 311 | LZ4HC_CCtx_internal* const ctx, 312 | const char* const source, 313 | char* const dest, 314 | int const inputSize, 315 | int const maxOutputSize, 316 | unsigned maxNbAttempts, 317 | limitedOutput_directive limit 318 | ) 319 | { 320 | const BYTE* ip = (const BYTE*) source; 321 | const BYTE* anchor = ip; 322 | const BYTE* const iend = ip + inputSize; 323 | const BYTE* const mflimit = iend - MFLIMIT; 324 | const BYTE* const matchlimit = (iend - LASTLITERALS); 325 | 326 | BYTE* op = (BYTE*) dest; 327 | BYTE* const oend = op + maxOutputSize; 328 | 329 | int ml, ml2, ml3, ml0; 330 | const BYTE* ref = NULL; 331 | const BYTE* start2 = NULL; 332 | const BYTE* ref2 = NULL; 333 | const BYTE* start3 = NULL; 334 | const BYTE* ref3 = NULL; 335 | const BYTE* start0; 336 | const BYTE* ref0; 337 | 338 | /* init */ 339 | ctx->end += inputSize; 340 | 341 | ip++; 342 | 343 | /* Main Loop */ 344 | while (ip < mflimit) { 345 | ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); 346 | if (!ml) { ip++; continue; } 347 | 348 | /* saved, in case we would skip too much */ 349 | start0 = ip; 350 | ref0 = ref; 351 | ml0 = ml; 352 | 353 | _Search2: 354 | if (ip+ml < mflimit) 355 | ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 0, matchlimit, ml, &ref2, &start2, maxNbAttempts); 356 | else ml2 = ml; 357 | 358 | if (ml2 == ml) { /* No better match */ 359 | if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; 360 | continue; 361 | } 362 | 363 | if (start0 < ip) { 364 | if (start2 < ip + ml0) { /* empirical */ 365 | ip = start0; 366 | ref = ref0; 367 | ml = ml0; 368 | } 369 | } 370 | 371 | /* Here, start0==ip */ 372 | if ((start2 - ip) < 3) { /* First Match too small : removed */ 373 | ml = ml2; 374 | ip = start2; 375 | ref =ref2; 376 | goto _Search2; 377 | } 378 | 379 | _Search3: 380 | /* 381 | * Currently we have : 382 | * ml2 > ml1, and 383 | * ip1+3 <= ip2 (usually < ip1+ml1) 384 | */ 385 | if ((start2 - ip) < OPTIMAL_ML) { 386 | int correction; 387 | int new_ml = ml; 388 | if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; 389 | if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; 390 | correction = new_ml - (int)(start2 - ip); 391 | if (correction > 0) { 392 | start2 += correction; 393 | ref2 += correction; 394 | ml2 -= correction; 395 | } 396 | } 397 | /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ 398 | 399 | if (start2 + ml2 < mflimit) 400 | ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); 401 | else ml3 = ml2; 402 | 403 | if (ml3 == ml2) { /* No better match : 2 sequences to encode */ 404 | /* ip & ref are known; Now for ml */ 405 | if (start2 < ip+ml) ml = (int)(start2 - ip); 406 | /* Now, encode 2 sequences */ 407 | if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; 408 | ip = start2; 409 | if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0; 410 | continue; 411 | } 412 | 413 | if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */ 414 | if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ 415 | if (start2 < ip+ml) { 416 | int correction = (int)(ip+ml - start2); 417 | start2 += correction; 418 | ref2 += correction; 419 | ml2 -= correction; 420 | if (ml2 < MINMATCH) { 421 | start2 = start3; 422 | ref2 = ref3; 423 | ml2 = ml3; 424 | } 425 | } 426 | 427 | if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; 428 | ip = start3; 429 | ref = ref3; 430 | ml = ml3; 431 | 432 | start0 = start2; 433 | ref0 = ref2; 434 | ml0 = ml2; 435 | goto _Search2; 436 | } 437 | 438 | start2 = start3; 439 | ref2 = ref3; 440 | ml2 = ml3; 441 | goto _Search3; 442 | } 443 | 444 | /* 445 | * OK, now we have 3 ascending matches; let's write at least the first one 446 | * ip & ref are known; Now for ml 447 | */ 448 | if (start2 < ip+ml) { 449 | if ((start2 - ip) < (int)ML_MASK) { 450 | int correction; 451 | if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; 452 | if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; 453 | correction = ml - (int)(start2 - ip); 454 | if (correction > 0) { 455 | start2 += correction; 456 | ref2 += correction; 457 | ml2 -= correction; 458 | } 459 | } else { 460 | ml = (int)(start2 - ip); 461 | } 462 | } 463 | if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; 464 | 465 | ip = start2; 466 | ref = ref2; 467 | ml = ml2; 468 | 469 | start2 = start3; 470 | ref2 = ref3; 471 | ml2 = ml3; 472 | 473 | goto _Search3; 474 | } 475 | 476 | /* Encode Last Literals */ 477 | { int lastRun = (int)(iend - anchor); 478 | if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ 479 | if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } 480 | else *op++ = (BYTE)(lastRun< 9) { 510 | switch (compressionLevel) { 511 | case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); 512 | case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); 513 | default: 514 | case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); 515 | } 516 | } 517 | return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit); 518 | } 519 | 520 | 521 | int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } 522 | 523 | int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) 524 | { 525 | LZ4HC_CCtx_internal* ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; 526 | if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ 527 | LZ4HC_init (ctx, (const BYTE*)src); 528 | if (maxDstSize < LZ4_compressBound(srcSize)) 529 | return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput); 530 | else 531 | return LZ4HC_compress_generic (ctx, src, dst, srcSize, maxDstSize, compressionLevel, noLimit); 532 | } 533 | 534 | int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) 535 | { 536 | #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 537 | LZ4_streamHC_t* const statePtr = (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); 538 | #else 539 | LZ4_streamHC_t state; 540 | LZ4_streamHC_t* const statePtr = &state; 541 | #endif 542 | int const cSize = LZ4_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize, compressionLevel); 543 | #if defined(LZ4HC_HEAPMODE) && LZ4HC_HEAPMODE==1 544 | free(statePtr); 545 | #endif 546 | return cSize; 547 | } 548 | 549 | 550 | 551 | /************************************** 552 | * Streaming Functions 553 | **************************************/ 554 | /* allocation */ 555 | LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); } 556 | int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; } 557 | 558 | 559 | /* initialization */ 560 | void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel) 561 | { 562 | LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ 563 | LZ4_streamHCPtr->internal_donotuse.base = NULL; 564 | LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel; 565 | LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel); 566 | } 567 | 568 | int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) 569 | { 570 | LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; 571 | if (dictSize > 64 KB) { 572 | dictionary += dictSize - 64 KB; 573 | dictSize = 64 KB; 574 | } 575 | LZ4HC_init (ctxPtr, (const BYTE*)dictionary); 576 | ctxPtr->end = (const BYTE*)dictionary + dictSize; 577 | if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) 578 | LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); 579 | else 580 | if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); 581 | return dictSize; 582 | } 583 | 584 | 585 | /* compression */ 586 | 587 | static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) 588 | { 589 | if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) 590 | LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); 591 | else 592 | if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ 593 | 594 | /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ 595 | ctxPtr->lowLimit = ctxPtr->dictLimit; 596 | ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); 597 | ctxPtr->dictBase = ctxPtr->base; 598 | ctxPtr->base = newBlock - ctxPtr->dictLimit; 599 | ctxPtr->end = newBlock; 600 | ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ 601 | } 602 | 603 | static int LZ4_compressHC_continue_generic (LZ4_streamHC_t* LZ4_streamHCPtr, 604 | const char* source, char* dest, 605 | int inputSize, int maxOutputSize, limitedOutput_directive limit) 606 | { 607 | LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; 608 | /* auto-init if forgotten */ 609 | if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source); 610 | 611 | /* Check overflow */ 612 | if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { 613 | size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; 614 | if (dictSize > 64 KB) dictSize = 64 KB; 615 | LZ4_loadDictHC(LZ4_streamHCPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); 616 | } 617 | 618 | /* Check if blocks follow each other */ 619 | if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source); 620 | 621 | /* Check overlapping input/dictionary space */ 622 | { const BYTE* sourceEnd = (const BYTE*) source + inputSize; 623 | const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; 624 | const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; 625 | if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) { 626 | if (sourceEnd > dictEnd) sourceEnd = dictEnd; 627 | ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); 628 | if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; 629 | } 630 | } 631 | 632 | return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit); 633 | } 634 | 635 | int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) 636 | { 637 | if (maxOutputSize < LZ4_compressBound(inputSize)) 638 | return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); 639 | else 640 | return LZ4_compressHC_continue_generic (LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); 641 | } 642 | 643 | 644 | /* dictionary saving */ 645 | 646 | int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) 647 | { 648 | LZ4HC_CCtx_internal* const streamPtr = &LZ4_streamHCPtr->internal_donotuse; 649 | int const prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); 650 | if (dictSize > 64 KB) dictSize = 64 KB; 651 | if (dictSize < 4) dictSize = 0; 652 | if (dictSize > prefixSize) dictSize = prefixSize; 653 | memmove(safeBuffer, streamPtr->end - dictSize, dictSize); 654 | { U32 const endIndex = (U32)(streamPtr->end - streamPtr->base); 655 | streamPtr->end = (const BYTE*)safeBuffer + dictSize; 656 | streamPtr->base = streamPtr->end - endIndex; 657 | streamPtr->dictLimit = endIndex - dictSize; 658 | streamPtr->lowLimit = endIndex - dictSize; 659 | if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; 660 | } 661 | return dictSize; 662 | } 663 | 664 | 665 | /*********************************** 666 | * Deprecated Functions 667 | ***********************************/ 668 | /* These functions currently generate deprecation warnings */ 669 | /* Deprecated compression functions */ 670 | int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } 671 | int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } 672 | int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } 673 | int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); } 674 | int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); } 675 | int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); } 676 | int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } 677 | int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); } 678 | int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); } 679 | int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); } 680 | 681 | 682 | /* Deprecated streaming functions */ 683 | int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } 684 | 685 | int LZ4_resetStreamStateHC(void* state, char* inputBuffer) 686 | { 687 | LZ4HC_CCtx_internal *ctx = &((LZ4_streamHC_t*)state)->internal_donotuse; 688 | if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ 689 | LZ4HC_init(ctx, (const BYTE*)inputBuffer); 690 | ctx->inputBuffer = (BYTE*)inputBuffer; 691 | return 0; 692 | } 693 | 694 | void* LZ4_createHC (char* inputBuffer) 695 | { 696 | LZ4_streamHC_t* hc4 = (LZ4_streamHC_t*)ALLOCATOR(1, sizeof(LZ4_streamHC_t)); 697 | if (hc4 == NULL) return NULL; /* not enough memory */ 698 | LZ4HC_init (&hc4->internal_donotuse, (const BYTE*)inputBuffer); 699 | hc4->internal_donotuse.inputBuffer = (BYTE*)inputBuffer; 700 | return hc4; 701 | } 702 | 703 | int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return 0; } 704 | 705 | int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) 706 | { 707 | return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, 0, compressionLevel, noLimit); 708 | } 709 | 710 | int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) 711 | { 712 | return LZ4HC_compress_generic (&((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); 713 | } 714 | 715 | char* LZ4_slideInputBufferHC(void* LZ4HC_Data) 716 | { 717 | LZ4HC_CCtx_internal* const hc4 = &((LZ4_streamHC_t*)LZ4HC_Data)->internal_donotuse; 718 | int const dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); 719 | return (char*)(hc4->inputBuffer + dictSize); 720 | } 721 | -------------------------------------------------------------------------------- /c_src/lz4hc.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 HC - High Compression Mode of LZ4 3 | Header File 4 | Copyright (C) 2011-2016, Yann Collet. 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - LZ4 source repository : https://github.com/lz4/lz4 32 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 33 | */ 34 | #ifndef LZ4_HC_H_19834876238432 35 | #define LZ4_HC_H_19834876238432 36 | 37 | #if defined (__cplusplus) 38 | extern "C" { 39 | #endif 40 | 41 | /* --- Dependency --- */ 42 | /* note : lz4hc is not an independent module, it requires lz4.h/lz4.c for proper compilation */ 43 | #include "lz4.h" /* stddef, LZ4LIB_API, LZ4_DEPRECATED */ 44 | 45 | 46 | /* --- Useful constants --- */ 47 | #define LZ4HC_CLEVEL_MIN 3 48 | #define LZ4HC_CLEVEL_DEFAULT 9 49 | #define LZ4HC_CLEVEL_OPT_MIN 11 50 | #define LZ4HC_CLEVEL_MAX 12 51 | 52 | 53 | /*-************************************ 54 | * Block Compression 55 | **************************************/ 56 | /*! LZ4_compress_HC() : 57 | * Compress data from `src` into `dst`, using the more powerful but slower "HC" algorithm. 58 | * `dst` must be already allocated. 59 | * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") 60 | * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") 61 | * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work. 62 | * Values >LZ4HC_MAX_CLEVEL behave the same as LZ4HC_MAX_CLEVEL. 63 | * @return : the number of bytes written into 'dst' 64 | * or 0 if compression fails. 65 | */ 66 | LZ4LIB_API int LZ4_compress_HC (const char* src, char* dst, int srcSize, int dstCapacity, int compressionLevel); 67 | 68 | 69 | /* Note : 70 | * Decompression functions are provided within "lz4.h" (BSD license) 71 | */ 72 | 73 | 74 | /*! LZ4_compress_HC_extStateHC() : 75 | * Same as LZ4_compress_HC(), but using an externally allocated memory segment for `state`. 76 | * `state` size is provided by LZ4_sizeofStateHC(). 77 | * Memory segment must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). 78 | */ 79 | LZ4LIB_API int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); 80 | LZ4LIB_API int LZ4_sizeofStateHC(void); 81 | 82 | 83 | /*-************************************ 84 | * Streaming Compression 85 | * Bufferless synchronous API 86 | **************************************/ 87 | typedef union LZ4_streamHC_u LZ4_streamHC_t; /* incomplete type (defined later) */ 88 | 89 | /*! LZ4_createStreamHC() and LZ4_freeStreamHC() : 90 | * These functions create and release memory for LZ4 HC streaming state. 91 | * Newly created states are automatically initialized. 92 | * Existing states can be re-used several times, using LZ4_resetStreamHC(). 93 | * These methods are API and ABI stable, they can be used in combination with a DLL. 94 | */ 95 | LZ4LIB_API LZ4_streamHC_t* LZ4_createStreamHC(void); 96 | LZ4LIB_API int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); 97 | 98 | LZ4LIB_API void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); 99 | LZ4LIB_API int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); 100 | 101 | LZ4LIB_API int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); 102 | 103 | LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); 104 | 105 | /* 106 | These functions compress data in successive blocks of any size, using previous blocks as dictionary. 107 | One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. 108 | There is an exception for ring buffers, which can be smaller than 64 KB. 109 | Ring buffers scenario is automatically detected and handled by LZ4_compress_HC_continue(). 110 | 111 | Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). 112 | A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). 113 | 114 | Then, use LZ4_compress_HC_continue() to compress each successive block. 115 | Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. 116 | 'dst' buffer should be sized to handle worst case scenarios, using LZ4_compressBound(), to ensure operation success. 117 | 118 | If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block, 119 | you must save it to a safer memory space, using LZ4_saveDictHC(). 120 | Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'. 121 | */ 122 | 123 | 124 | /*-****************************************** 125 | * !!!!! STATIC LINKING ONLY !!!!! 126 | *******************************************/ 127 | 128 | /*-************************************* 129 | * PRIVATE DEFINITIONS : 130 | * Do not use these definitions. 131 | * They are exposed to allow static allocation of `LZ4_streamHC_t`. 132 | * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 133 | **************************************/ 134 | #define LZ4HC_DICTIONARY_LOGSIZE 17 135 | #define LZ4HC_MAXD (1<= 199901L) /* C99 */) 144 | #include 145 | 146 | typedef struct 147 | { 148 | uint32_t hashTable[LZ4HC_HASHTABLESIZE]; 149 | uint16_t chainTable[LZ4HC_MAXD]; 150 | const uint8_t* end; /* next block here to continue on current prefix */ 151 | const uint8_t* base; /* All index relative to this position */ 152 | const uint8_t* dictBase; /* alternate base for extDict */ 153 | uint8_t* inputBuffer; /* deprecated */ 154 | uint32_t dictLimit; /* below that point, need extDict */ 155 | uint32_t lowLimit; /* below that point, no more dict */ 156 | uint32_t nextToUpdate; /* index from which to continue dictionary update */ 157 | uint32_t searchNum; /* only for optimal parser */ 158 | uint32_t compressionLevel; 159 | } LZ4HC_CCtx_internal; 160 | 161 | #else 162 | 163 | typedef struct 164 | { 165 | unsigned int hashTable[LZ4HC_HASHTABLESIZE]; 166 | unsigned short chainTable[LZ4HC_MAXD]; 167 | const unsigned char* end; /* next block here to continue on current prefix */ 168 | const unsigned char* base; /* All index relative to this position */ 169 | const unsigned char* dictBase; /* alternate base for extDict */ 170 | unsigned char* inputBuffer; /* deprecated */ 171 | unsigned int dictLimit; /* below that point, need extDict */ 172 | unsigned int lowLimit; /* below that point, no more dict */ 173 | unsigned int nextToUpdate; /* index from which to continue dictionary update */ 174 | unsigned int searchNum; /* only for optimal parser */ 175 | unsigned int compressionLevel; 176 | } LZ4HC_CCtx_internal; 177 | 178 | #endif 179 | 180 | #define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 393268 */ 181 | #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) 182 | union LZ4_streamHC_u { 183 | size_t table[LZ4_STREAMHCSIZE_SIZET]; 184 | LZ4HC_CCtx_internal internal_donotuse; 185 | }; /* previously typedef'd to LZ4_streamHC_t */ 186 | /* 187 | LZ4_streamHC_t : 188 | This structure allows static allocation of LZ4 HC streaming state. 189 | State must be initialized using LZ4_resetStreamHC() before first use. 190 | 191 | Static allocation shall only be used in combination with static linking. 192 | When invoking LZ4 from a DLL, use create/free functions instead, which are API and ABI stable. 193 | */ 194 | 195 | 196 | /*-************************************ 197 | * Deprecated Functions 198 | **************************************/ 199 | /* see lz4.h LZ4_DISABLE_DEPRECATE_WARNINGS to turn off deprecation warnings */ 200 | 201 | /* deprecated compression functions */ 202 | /* these functions will trigger warning messages in future releases */ 203 | LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC (const char* source, char* dest, int inputSize); 204 | LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); 205 | LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); 206 | LZ4_DEPRECATED("use LZ4_compress_HC() instead") int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); 207 | LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); 208 | LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); 209 | LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); 210 | LZ4_DEPRECATED("use LZ4_compress_HC_extStateHC() instead") int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); 211 | LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); 212 | LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); 213 | 214 | /* Deprecated Streaming functions using older model; should no longer be used */ 215 | LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer); 216 | LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data); 217 | LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data); 218 | LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); 219 | LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); 220 | LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void); 221 | LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer); 222 | 223 | 224 | #if defined (__cplusplus) 225 | } 226 | #endif 227 | 228 | #endif /* LZ4_HC_H_19834876238432 */ 229 | -------------------------------------------------------------------------------- /c_src/lz4opt.h: -------------------------------------------------------------------------------- 1 | /* 2 | lz4opt.h - Optimal Mode of LZ4 3 | Copyright (C) 2015-2016, Przemyslaw Skibinski 4 | Note : this file is intended to be included within lz4hc.c 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - LZ4 source repository : https://github.com/lz4/lz4 33 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 34 | */ 35 | 36 | #define LZ4_OPT_NUM (1<<12) 37 | 38 | 39 | typedef struct 40 | { 41 | int off; 42 | int len; 43 | } LZ4HC_match_t; 44 | 45 | typedef struct 46 | { 47 | int price; 48 | int off; 49 | int mlen; 50 | int litlen; 51 | } LZ4HC_optimal_t; 52 | 53 | 54 | /* price in bits */ 55 | FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen) 56 | { 57 | size_t price = 8*litlen; 58 | if (litlen >= (size_t)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255); 59 | return price; 60 | } 61 | 62 | 63 | /* requires mlen >= MINMATCH */ 64 | FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen) 65 | { 66 | size_t price = 16 + 8; /* 16-bit offset + token */ 67 | 68 | price += LZ4HC_literalsPrice(litlen); 69 | 70 | mlen -= MINMATCH; 71 | if (mlen >= (size_t)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255); 72 | 73 | return price; 74 | } 75 | 76 | 77 | /*-************************************* 78 | * Binary Tree search 79 | ***************************************/ 80 | FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches ( 81 | LZ4HC_CCtx_internal* ctx, 82 | const BYTE* const ip, 83 | const BYTE* const iHighLimit, 84 | size_t best_mlen, 85 | LZ4HC_match_t* matches, 86 | int* matchNum) 87 | { 88 | U16* const chainTable = ctx->chainTable; 89 | U32* const HashTable = ctx->hashTable; 90 | const BYTE* const base = ctx->base; 91 | const U32 dictLimit = ctx->dictLimit; 92 | const U32 current = (U32)(ip - base); 93 | const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1); 94 | const BYTE* const dictBase = ctx->dictBase; 95 | const BYTE* match; 96 | int nbAttempts = ctx->searchNum; 97 | int mnum = 0; 98 | U16 *ptr0, *ptr1, delta0, delta1; 99 | U32 matchIndex; 100 | size_t matchLength = 0; 101 | U32* HashPos; 102 | 103 | if (ip + MINMATCH > iHighLimit) return 1; 104 | 105 | /* HC4 match finder */ 106 | HashPos = &HashTable[LZ4HC_hashPtr(ip)]; 107 | matchIndex = *HashPos; 108 | *HashPos = current; 109 | 110 | ptr0 = &DELTANEXTMAXD(current*2+1); 111 | ptr1 = &DELTANEXTMAXD(current*2); 112 | delta0 = delta1 = (U16)(current - matchIndex); 113 | 114 | while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { 115 | nbAttempts--; 116 | if (matchIndex >= dictLimit) { 117 | match = base + matchIndex; 118 | matchLength = LZ4_count(ip, match, iHighLimit); 119 | } else { 120 | const BYTE* vLimit = ip + (dictLimit - matchIndex); 121 | match = dictBase + matchIndex; 122 | if (vLimit > iHighLimit) vLimit = iHighLimit; 123 | matchLength = LZ4_count(ip, match, vLimit); 124 | if ((ip+matchLength == vLimit) && (vLimit < iHighLimit)) 125 | matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit); 126 | } 127 | 128 | if (matchLength > best_mlen) { 129 | best_mlen = matchLength; 130 | if (matches) { 131 | if (matchIndex >= dictLimit) 132 | matches[mnum].off = (int)(ip - match); 133 | else 134 | matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */ 135 | matches[mnum].len = (int)matchLength; 136 | mnum++; 137 | } 138 | if (best_mlen > LZ4_OPT_NUM) break; 139 | } 140 | 141 | if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ 142 | break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ 143 | 144 | if (*(ip+matchLength) < *(match+matchLength)) { 145 | *ptr0 = delta0; 146 | ptr0 = &DELTANEXTMAXD(matchIndex*2); 147 | if (*ptr0 == (U16)-1) break; 148 | delta0 = *ptr0; 149 | delta1 += delta0; 150 | matchIndex -= delta0; 151 | } else { 152 | *ptr1 = delta1; 153 | ptr1 = &DELTANEXTMAXD(matchIndex*2+1); 154 | if (*ptr1 == (U16)-1) break; 155 | delta1 = *ptr1; 156 | delta0 += delta1; 157 | matchIndex -= delta1; 158 | } 159 | } 160 | 161 | *ptr0 = (U16)-1; 162 | *ptr1 = (U16)-1; 163 | if (matchNum) *matchNum = mnum; 164 | /* if (best_mlen > 8) return best_mlen-8; */ 165 | if (!matchNum) return 1; 166 | return 1; 167 | } 168 | 169 | 170 | FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit) 171 | { 172 | const BYTE* const base = ctx->base; 173 | const U32 target = (U32)(ip - base); 174 | U32 idx = ctx->nextToUpdate; 175 | while(idx < target) 176 | idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); 177 | } 178 | 179 | 180 | /** Tree updater, providing best match */ 181 | FORCE_INLINE int LZ4HC_BinTree_GetAllMatches ( 182 | LZ4HC_CCtx_internal* ctx, 183 | const BYTE* const ip, const BYTE* const iHighLimit, 184 | size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate) 185 | { 186 | int mnum = 0; 187 | if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */ 188 | if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit); 189 | best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum); 190 | ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen); 191 | return mnum; 192 | } 193 | 194 | 195 | #define SET_PRICE(pos, mlen, offset, litlen, price) \ 196 | { \ 197 | while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ 198 | opt[pos].mlen = (int)mlen; \ 199 | opt[pos].off = (int)offset; \ 200 | opt[pos].litlen = (int)litlen; \ 201 | opt[pos].price = (int)price; \ 202 | } 203 | 204 | 205 | static int LZ4HC_compress_optimal ( 206 | LZ4HC_CCtx_internal* ctx, 207 | const char* const source, 208 | char* dest, 209 | int inputSize, 210 | int maxOutputSize, 211 | limitedOutput_directive limit, 212 | const size_t sufficient_len, 213 | const int fullUpdate 214 | ) 215 | { 216 | LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1]; 217 | LZ4HC_match_t matches[LZ4_OPT_NUM + 1]; 218 | const BYTE *inr = NULL; 219 | size_t res, cur, cur2; 220 | size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; 221 | 222 | const BYTE* ip = (const BYTE*) source; 223 | const BYTE* anchor = ip; 224 | const BYTE* const iend = ip + inputSize; 225 | const BYTE* const mflimit = iend - MFLIMIT; 226 | const BYTE* const matchlimit = (iend - LASTLITERALS); 227 | BYTE* op = (BYTE*) dest; 228 | BYTE* const oend = op + maxOutputSize; 229 | 230 | /* init */ 231 | ctx->end += inputSize; 232 | ip++; 233 | 234 | /* Main Loop */ 235 | while (ip < mflimit) { 236 | memset(opt, 0, sizeof(LZ4HC_optimal_t)); 237 | last_pos = 0; 238 | llen = ip - anchor; 239 | match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate); 240 | if (!match_num) { ip++; continue; } 241 | 242 | if ((size_t)matches[match_num-1].len > sufficient_len) { 243 | best_mlen = matches[match_num-1].len; 244 | best_off = matches[match_num-1].off; 245 | cur = 0; 246 | last_pos = 1; 247 | goto encode; 248 | } 249 | 250 | /* set prices using matches at position = 0 */ 251 | for (i = 0; i < match_num; i++) { 252 | mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; 253 | best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM; 254 | while (mlen <= best_mlen) { 255 | litlen = 0; 256 | price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); 257 | SET_PRICE(mlen, mlen, matches[i].off, litlen, price); 258 | mlen++; 259 | } 260 | } 261 | 262 | if (last_pos < MINMATCH) { ip++; continue; } 263 | 264 | /* check further positions */ 265 | opt[0].mlen = opt[1].mlen = 1; 266 | for (cur = 1; cur <= last_pos; cur++) { 267 | inr = ip + cur; 268 | 269 | if (opt[cur-1].mlen == 1) { 270 | litlen = opt[cur-1].litlen + 1; 271 | if (cur != litlen) { 272 | price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen); 273 | } else { 274 | price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen); 275 | } 276 | } else { 277 | litlen = 1; 278 | price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen); 279 | } 280 | 281 | mlen = 1; 282 | best_mlen = 0; 283 | if (cur > last_pos || price < (size_t)opt[cur].price) 284 | SET_PRICE(cur, mlen, best_mlen, litlen, price); 285 | 286 | if (cur == last_pos || inr >= mflimit) break; 287 | 288 | match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate); 289 | if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) { 290 | best_mlen = matches[match_num-1].len; 291 | best_off = matches[match_num-1].off; 292 | last_pos = cur + 1; 293 | goto encode; 294 | } 295 | 296 | /* set prices using matches at position = cur */ 297 | for (i = 0; i < match_num; i++) { 298 | mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH; 299 | cur2 = cur; 300 | best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2; 301 | 302 | while (mlen <= best_mlen) { 303 | if (opt[cur2].mlen == 1) { 304 | litlen = opt[cur2].litlen; 305 | 306 | if (cur2 != litlen) 307 | price = opt[cur2 - litlen].price + LZ4HC_sequencePrice(litlen, mlen); 308 | else 309 | price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen); 310 | } else { 311 | litlen = 0; 312 | price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen); 313 | } 314 | 315 | if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) { 316 | SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); 317 | } 318 | mlen++; 319 | } 320 | } 321 | } /* for (cur = 1; cur <= last_pos; cur++) */ 322 | 323 | best_mlen = opt[last_pos].mlen; 324 | best_off = opt[last_pos].off; 325 | cur = last_pos - best_mlen; 326 | 327 | encode: /* cur, last_pos, best_mlen, best_off have to be set */ 328 | opt[0].mlen = 1; 329 | while (1) { 330 | mlen = opt[cur].mlen; 331 | offset = opt[cur].off; 332 | opt[cur].mlen = (int)best_mlen; 333 | opt[cur].off = (int)best_off; 334 | best_mlen = mlen; 335 | best_off = offset; 336 | if (mlen > cur) break; 337 | cur -= mlen; 338 | } 339 | 340 | cur = 0; 341 | while (cur < last_pos) { 342 | mlen = opt[cur].mlen; 343 | if (mlen == 1) { ip++; cur++; continue; } 344 | offset = opt[cur].off; 345 | cur += mlen; 346 | 347 | res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); 348 | if (res) return 0; 349 | } 350 | } 351 | 352 | /* Encode Last Literals */ 353 | { int lastRun = (int)(iend - anchor); 354 | if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ 355 | if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } 356 | else *op++ = (BYTE)(lastRun< 3 | @copyright 2012- SUZUKI Tetsuya 4 | @version 0.2.2 5 | @reference RealTime Data Compression: LZ4 6 | @reference lz4 - Extremely Fast Compression algorithm 7 | @reference https://github.com/Cyan4973/lz4 (official mirror) 8 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Lz4.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [app: :lz4, 6 | version: "0.2.4-a", 7 | elixir: ">= 1.0.3", 8 | compilers: [:erlang, :elixir], 9 | deps: deps, 10 | description: description, 11 | package: package] 12 | end 13 | 14 | def application, do: [] 15 | 16 | def deps do 17 | [ 18 | {:ex_doc, ">= 0.0.0", only: :dev}, 19 | ] 20 | end 21 | 22 | defp description do 23 | "LZ4 bindings for Erlang" 24 | end 25 | 26 | defp package do 27 | [files: ["c_src/Makefile", "c_src/*.c", "c_src/*.h", "src", "test", "README.md", "CHANGES", "rebar.config"], 28 | maintainers: ["SUZUKI Tetsuya"], 29 | licenses: ["Apache License 2.0", "BSD 2-Clause"], 30 | links: %{"GitHub" => "https://github.com/szktty/erlang-lz4.git"}] 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{"earmark": {:hex, :earmark, "1.2.0", "bf1ce17aea43ab62f6943b97bd6e3dc032ce45d4f787504e3adf738e54b42f3a", [:mix], [], "hexpm"}, 2 | "ex_doc": {:hex, :ex_doc, "0.15.1", "d5f9d588fd802152516fccfdb96d6073753f77314fcfee892b15b6724ca0d596", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}], "hexpm"}} 3 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [{i, "src"}, 2 | debug_info, 3 | warnings_as_errors, 4 | {w, all}, 5 | warn_export_all]}. 6 | 7 | {clean_files, [".eunit", 8 | "ebin/*.beam"]}. 9 | 10 | {eunit_opts, [{report,{eunit_surefire,[{dir,"."}]}}]}. 11 | 12 | %% The extension of the artifact is modified in rebar.config.script for win32 13 | {artifacts, ["priv/lz4.so"]}. 14 | 15 | %% Compile nif using port compiler plugin 16 | {plugins, [pc]}. 17 | 18 | {provider_hooks, [ 19 | {pre, [ 20 | {compile, {pc, compile}}, 21 | {clean, {pc, clean}} 22 | ]} 23 | ]}. 24 | 25 | {port_specs, [ 26 | {"priv/lz4.so", [ 27 | "c_src/*.c" 28 | ]} 29 | ]}. 30 | -------------------------------------------------------------------------------- /rebar.config.script: -------------------------------------------------------------------------------- 1 | %% -*- erlang -*- 2 | 3 | IsWindows = case os:type() of {win32, _} -> true; {_, _} -> false end, 4 | 5 | MaybeSwitchToDll = fun(File) -> 6 | case IsWindows of 7 | false -> 8 | File; 9 | true -> 10 | case filename:extension(File) of 11 | ".so" -> filename:rootname(File, ".so") ++ ".dll"; 12 | [] -> File ++ ".exe"; 13 | _Other -> File 14 | end 15 | end 16 | end, 17 | 18 | {artifacts, Artifacts0} = lists:keyfind(artifacts, 1, CONFIG), 19 | Artifacts1 = [MaybeSwitchToDll(F) || F <- Artifacts0], 20 | lists:keyreplace(artifacts, 1, CONFIG, {artifacts, Artifacts1}). 21 | -------------------------------------------------------------------------------- /src/lz4.app.src: -------------------------------------------------------------------------------- 1 | {application, lz4, 2 | [ 3 | {description, ""}, 4 | {vsn, "0.2.2"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib 9 | ]}, 10 | {modules, [lz4]}, 11 | {env, []} 12 | ]}. 13 | -------------------------------------------------------------------------------- /src/lz4.erl: -------------------------------------------------------------------------------- 1 | -module(lz4). 2 | 3 | -export([compress/1, compress/2, uncompress/2, 4 | pack/1, pack/2, unpack/1]). 5 | 6 | -compile(no_native). 7 | -on_load(init/0). 8 | 9 | -type option() :: high. 10 | %% Compressor option. 11 | %% 12 | %%
`high'
13 | %%
Compresses with high ratio.
14 | 15 | -type pack() :: binary(). 16 | %% Binary included compressed data and original size stored 17 | %% at first 4 bytes in little endian. 18 | 19 | -define(nif_stub, nif_stub_error(?LINE)). 20 | nif_stub_error(Line) -> 21 | erlang:nif_error({nif_not_loaded,module,?MODULE,line,Line}). 22 | 23 | init() -> 24 | PrivDir = case code:priv_dir(?MODULE) of 25 | {error, bad_name} -> 26 | EbinDir = filename:dirname(code:which(?MODULE)), 27 | AppPath = filename:dirname(EbinDir), 28 | filename:join(AppPath, "priv"); 29 | Path -> 30 | Path 31 | end, 32 | erlang:load_nif(filename:join(PrivDir, lz4), 0). 33 | 34 | %% @doc Equals `compress(Binary, [])'. 35 | %% @see compress/2 36 | -spec compress(binary()) -> {ok, binary()} | {error, term()}. 37 | compress(Binary) -> 38 | compress(Binary, []). 39 | 40 | %% @doc Returns an compressed binary. Note that the compressed binary 41 | %% does not include original size to be needed at uncompressing. 42 | %% @see uncompress/2 43 | %% @see pack/2 44 | -spec compress(binary(), [option()]) -> {ok, binary()} | {error, term()}. 45 | compress(_Binary, _Options) -> 46 | ?nif_stub. 47 | 48 | %% @doc Returns an uncompressed binary. 49 | %% You need to specify original size as `OrigSize'. 50 | %% @see compress/2 51 | -spec uncompress(binary(), integer()) -> {ok, binary()} | {error, term()}. 52 | uncompress(_Binary, _OrigSize) -> 53 | ?nif_stub. 54 | 55 | %% @doc Equals `pack(Binary, [])'. 56 | %% @see pack/2 57 | -spec pack(binary()) -> {ok, pack()} | {error, term()}. 58 | pack(Binary) -> 59 | pack(Binary, []). 60 | 61 | %% @doc Returns a binary included compressed data and original size. 62 | %% Use `unpack/1' to uncompress the returned binary. 63 | %% @see compress/2 64 | %% @see unpack/1 65 | -spec pack(binary(), [option()]) -> {ok, pack()} | {error, term()}. 66 | pack(Binary, Options) -> 67 | case compress(Binary, Options) of 68 | {ok, Compressed} -> 69 | OrigSize = byte_size(Binary), 70 | {ok, <>}; 72 | Error -> 73 | Error 74 | end. 75 | 76 | %% @doc Return a uncompressed binary compressed with `pack/2'. 77 | %% @see pack/2 78 | -spec unpack(pack()) -> {ok, binary()} | {error, term()}. 79 | unpack(<>=_Binary) -> 80 | uncompress(Binary, OrigSize). 81 | 82 | -------------------------------------------------------------------------------- /test/lz4_tests.erl: -------------------------------------------------------------------------------- 1 | -module(lz4_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | test_data() -> 6 | Raw = <<"Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.">>, 7 | binary:copy(Raw, 10000). 8 | 9 | compress_test() -> 10 | Raw = test_data(), 11 | {ok, Comp} = lz4:compress(Raw), 12 | {ok, Uncomp} = lz4:uncompress(Comp, byte_size(Raw)), 13 | ?assertEqual(Raw, Uncomp). 14 | 15 | high_compress_test() -> 16 | Raw = test_data(), 17 | {ok, Comp} = lz4:compress(Raw, [high]), 18 | {ok, Uncomp} = lz4:uncompress(Comp, byte_size(Raw)), 19 | ?assertEqual(Raw, Uncomp). 20 | 21 | pack_test() -> 22 | Raw = test_data(), 23 | {ok, Pack} = lz4:pack(Raw), 24 | {ok, Unpack} = lz4:unpack(Pack), 25 | <> = Pack, 26 | ?assertEqual(byte_size(Raw), Size), 27 | ?assertEqual(Raw, Unpack). 28 | 29 | zero_compress_test() -> 30 | Raw = <<>>, 31 | {ok, Comp} = lz4:compress(Raw), 32 | {ok, Uncomp} = lz4:uncompress(Comp, byte_size(Raw)), 33 | ?assertEqual(Raw, Uncomp). 34 | 35 | zero_high_compress_test() -> 36 | Raw = <<>>, 37 | {ok, Comp} = lz4:compress(Raw, [high]), 38 | {ok, Uncomp} = lz4:uncompress(Comp, byte_size(Raw)), 39 | ?assertEqual(Raw, Uncomp). 40 | 41 | zero_pack_test() -> 42 | Raw = <<>>, 43 | {ok, Pack} = lz4:pack(Raw), 44 | {ok, Unpack} = lz4:unpack(Pack), 45 | <> = Pack, 46 | ?assertEqual(byte_size(Raw), Size), 47 | ?assertEqual(Raw, Unpack). 48 | --------------------------------------------------------------------------------