├── .gitignore ├── LICENSE ├── README.md ├── FSE ├── debug.c ├── error_public.h ├── hist.h ├── fseU16.h ├── debug.h ├── error_private.h ├── compiler.h ├── hist.c ├── entropy_common.c ├── fse_decompress.c ├── fseU16.c ├── mem.h ├── bitstream.h ├── huf.h ├── fse_compress.c └── fse.h ├── halac_decode_v.0.1.9.h ├── halac_encode_v.0.1.9.h ├── halac_encode_v.0.1.9.cpp └── halac_decode_v.0.1.9.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 HAKAN ABBAS 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HALAC High Availability Lossless Audio Compression 2 | I took a break from HALIC for GDCC 2023. Then I tried something a little different. In the past(2018-2019), I had been working on the lossless audio compression. However, I could not bring together the work I did. Now I have a little time and I think I developed a fast codec. I worked on 16 bit, 2 channel audio data (.wav). Higher bit and channel options can be added if necessary. As a result, the approach is the same. 3 | HALAC, like the HALIC, focuses on a reasonable compression ratio and high processing speed. The compression rate for audio data is usually limited. So I wanted a solution that can work faster with a few percent concessions. 4 | I used a quick estimation with ANS(FSE). I don't know if there are other codecs using ANS, but the majority uses "Rice Coding". GPU or SIMD was not used. Also now in the multihread version. 5 | I tried to find the middle way by working with different music genres. 6 | 7 | ![sibel_can](https://github.com/Hakan-Abbas/HALAC-High-Availability-Lossless-Audio-Compression-/assets/158841237/acfeeacd-7815-4a25-b1ac-e465c682ebb4) 8 | 9 | ![HALAC_0 2 4_MT_BENCHMARK1](https://github.com/Hakan-Abbas/HALAC-High-Availability-Lossless-Audio-Compression-/assets/158841237/23c74e3b-1f90-45ec-9d6e-c43593b2c527) 10 | -------------------------------------------------------------------------------- /FSE/debug.c: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | debug 3 | Part of FSE library 4 | Copyright (C) 2013-present, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 33 | ****************************************************************** */ 34 | 35 | 36 | /* 37 | * This module only hosts one global variable 38 | * which can be used to dynamically influence the verbosity of traces, 39 | * such as DEBUGLOG and RAWLOG 40 | */ 41 | 42 | #include "debug.h" 43 | 44 | int g_debuglevel = DEBUGLEVEL; 45 | -------------------------------------------------------------------------------- /FSE/error_public.h: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | Error codes list 3 | Copyright (C) 2016, Yann Collet 4 | 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 32 | - Public forum : https://groups.google.com/forum/#!forum/lz4c 33 | ****************************************************************** */ 34 | #ifndef ERROR_PUBLIC_H_MODULE 35 | #define ERROR_PUBLIC_H_MODULE 36 | 37 | #if defined (__cplusplus) 38 | extern "C" { 39 | #endif 40 | 41 | 42 | /* ***************************************** 43 | * error codes list 44 | ******************************************/ 45 | typedef enum { 46 | FSE_error_no_error, 47 | FSE_error_GENERIC, 48 | FSE_error_dstSize_tooSmall, 49 | FSE_error_srcSize_wrong, 50 | FSE_error_corruption_detected, 51 | FSE_error_tableLog_tooLarge, 52 | FSE_error_maxSymbolValue_tooLarge, 53 | FSE_error_maxSymbolValue_tooSmall, 54 | FSE_error_workSpace_tooSmall, 55 | FSE_error_maxCode 56 | } FSE_ErrorCode; 57 | 58 | /* note : compare with size_t function results using FSE_getError() */ 59 | 60 | 61 | #if defined (__cplusplus) 62 | } 63 | #endif 64 | 65 | #endif /* ERROR_PUBLIC_H_MODULE */ 66 | -------------------------------------------------------------------------------- /halac_decode_v.0.1.9.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "FSE/fse.h" 11 | #include "FSE/huf.h" 12 | ////////////////////////////////////////// 13 | using namespace std; 14 | namespace fs = std::filesystem; 15 | ////////////////////////////////////////// 16 | typedef char i8; 17 | typedef unsigned char u8; 18 | typedef short i16; 19 | typedef unsigned short u16; 20 | typedef int i32; 21 | typedef unsigned int u32; 22 | typedef long long i64; 23 | typedef unsigned long long u64; 24 | ////////////////////////////////////////// 25 | const i8 VERSION[] = "HALAC 0.1.9 - Single Thread"; 26 | string INPUT_FILENAME, OUTPUT_FILENAME; 27 | FILE *INPUT_FILE, *OUTPUT_FILE; 28 | u32 FILE_SIZE, INPUT_SIZE, DATA_SIZE, SAMPLE_RATE, METADATA_SIZE, OUTPUT_SIZE = 0, TOTAL_OUTPUT_SIZE = 0; 29 | u16 CHANNEL_COUNT, SAMPLE_BIT, SAMPLE_BYTE; 30 | const u32 BLOCK_SIZE = 1024 * 24; 31 | const u8 ORDER = 8; 32 | 33 | ////////////////////////////////////////// 34 | i8 TEMP1[14]; 35 | i8 TEMP2[6]; 36 | i8 METADATA[1024 * 16]; 37 | ////////////////////////////////////////// 38 | static inline i32 Unsigned_To_Signed(u32 num) { 39 | u32 mask = -(num & 1); 40 | return (num >> 1) ^ mask; 41 | } 42 | 43 | void READ_HALAC_HEADER() { 44 | INPUT_FILE = fopen(INPUT_FILENAME.c_str(), "rb"); 45 | 46 | char version[11]; 47 | fread(version, 1, 11, INPUT_FILE); 48 | 49 | if (version[0] != 'H' || version[1] != 'A' || version[2] != 'L' || version[3] != 'A' || version[4] != 'C') { 50 | printf("Only HALAC file type is supported for input!\n"); 51 | exit(0); 52 | } 53 | 54 | fread(&FILE_SIZE, 1, 4, INPUT_FILE); 55 | fread(&TEMP1, 1, 14, INPUT_FILE); 56 | fread(&CHANNEL_COUNT, 1, 2, INPUT_FILE); 57 | fread(&SAMPLE_RATE, 1, 4, INPUT_FILE); 58 | fread(&TEMP2, 1, 6, INPUT_FILE); 59 | fread(&SAMPLE_BIT, 1, 2, INPUT_FILE); 60 | SAMPLE_BYTE = SAMPLE_BIT / 8; 61 | 62 | fread(&METADATA_SIZE, 1, 4, INPUT_FILE); 63 | if (METADATA_SIZE > 0) fread(&METADATA, 1, METADATA_SIZE, INPUT_FILE); 64 | 65 | fread(&INPUT_SIZE, 1, 4, INPUT_FILE); 66 | } 67 | 68 | void WRITE_WAV_HEADER() { 69 | OUTPUT_FILE = fopen(OUTPUT_FILENAME.c_str(), "wb"); 70 | 71 | i8 RIFF[4] = { 'R','I','F','F' }; 72 | i8 data[4] = { 'd', 'a', 't', 'a' }; 73 | i8 LIST[4] = { 'L', 'I', 'S', 'T' }; 74 | 75 | fwrite(RIFF, 1, 4, OUTPUT_FILE); 76 | fwrite(&FILE_SIZE, 1, 4, OUTPUT_FILE); 77 | fwrite(TEMP1, 1, 14, OUTPUT_FILE); 78 | fwrite(&CHANNEL_COUNT, 1, 2, OUTPUT_FILE); 79 | fwrite(&SAMPLE_RATE, 1, 4, OUTPUT_FILE); 80 | fwrite(TEMP2, 1, 6, OUTPUT_FILE); 81 | fwrite(&SAMPLE_BIT, 1, 2, OUTPUT_FILE); 82 | 83 | if (METADATA_SIZE > 0) { 84 | fwrite(LIST, 1, 4, OUTPUT_FILE); 85 | fwrite(&METADATA_SIZE, 1, 4, OUTPUT_FILE); 86 | fwrite(METADATA, 1, METADATA_SIZE, OUTPUT_FILE); 87 | } 88 | 89 | fwrite(data, 1, 4, OUTPUT_FILE); 90 | fwrite(&INPUT_SIZE, 1, 4, OUTPUT_FILE); 91 | } -------------------------------------------------------------------------------- /FSE/hist.h: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * hist : Histogram functions 3 | * part of Finite State Entropy project 4 | * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. 5 | * 6 | * You can contact the author at : 7 | * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy 8 | * - Public forum : https://groups.google.com/forum/#!forum/lz4c 9 | * 10 | * This source code is licensed under both the BSD-style license (found in the 11 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 12 | * in the COPYING file in the root directory of this source tree). 13 | * You may select, at your option, one of the above-listed licenses. 14 | ****************************************************************** */ 15 | 16 | /* --- dependencies --- */ 17 | #include /* size_t */ 18 | 19 | 20 | /* --- simple histogram functions --- */ 21 | 22 | /*! HIST_count(): 23 | * Provides the precise count of each byte within a table 'count'. 24 | * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). 25 | * Updates *maxSymbolValuePtr with actual largest symbol value detected. 26 | * @return : count of the most frequent symbol (which isn't identified). 27 | * or an error code, which can be tested using HIST_isError(). 28 | * note : if return == srcSize, there is only one symbol. 29 | */ 30 | size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, 31 | const void* src, size_t srcSize); 32 | 33 | unsigned HIST_isError(size_t code); /**< tells if a return value is an error code */ 34 | 35 | 36 | /* --- advanced histogram functions --- */ 37 | 38 | #define HIST_WKSP_SIZE_U32 1024 39 | #define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned)) 40 | /** HIST_count_wksp() : 41 | * Same as HIST_count(), but using an externally provided scratch buffer. 42 | * Benefit is this function will use very little stack space. 43 | * `workSpace` is a writable buffer which must be 4-bytes aligned, 44 | * `workSpaceSize` must be >= HIST_WKSP_SIZE 45 | */ 46 | size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, 47 | const void* src, size_t srcSize, 48 | void* workSpace, size_t workSpaceSize); 49 | 50 | /** HIST_countFast() : 51 | * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr. 52 | * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` 53 | */ 54 | size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, 55 | const void* src, size_t srcSize); 56 | 57 | /** HIST_countFast_wksp() : 58 | * Same as HIST_countFast(), but using an externally provided scratch buffer. 59 | * `workSpace` is a writable buffer which must be 4-bytes aligned, 60 | * `workSpaceSize` must be >= HIST_WKSP_SIZE 61 | */ 62 | size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, 63 | const void* src, size_t srcSize, 64 | void* workSpace, size_t workSpaceSize); 65 | 66 | /*! HIST_count_simple() : 67 | * Same as HIST_countFast(), this function is unsafe, 68 | * and will segfault if any value within `src` is `> *maxSymbolValuePtr`. 69 | * It is also a bit slower for large inputs. 70 | * However, it does not need any additional memory (not even on stack). 71 | * @return : count of the most frequent symbol. 72 | * Note this function doesn't produce any error (i.e. it must succeed). 73 | */ 74 | unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, 75 | const void* src, size_t srcSize); 76 | -------------------------------------------------------------------------------- /FSE/fseU16.h: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | FSEU16 : Finite State Entropy coder for 16-bits input 3 | header file 4 | Copyright (C) 2013-2016, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy 33 | - Public forum : https://groups.google.com/forum/#!forum/lz4c 34 | ****************************************************************** */ 35 | #ifndef FSE_U16_H_30982039483 36 | #define FSE_U16_H_30982039483 37 | 38 | #if defined (__cplusplus) 39 | extern "C" { 40 | #endif 41 | 42 | 43 | /*-***************************************** 44 | * Tuning parameters 45 | *******************************************/ 46 | /* FSE_MAX_SYMBOL_VALUE : 47 | * Maximum nb of symbol values authorized. 48 | * Required for allocation purposes */ 49 | #ifndef FSEU16_MAX_SYMBOL_VALUE 50 | # define FSEU16_MAX_SYMBOL_VALUE 1024 /* This is just an example, typical value for zlib */ 51 | #endif 52 | #ifdef FSE_MAX_SYMBOL_VALUE 53 | # undef FSE_MAX_SYMBOL_VALUE 54 | #endif 55 | #define FSE_MAX_SYMBOL_VALUE FSEU16_MAX_SYMBOL_VALUE 56 | 57 | /*-***************************************** 58 | * Includes 59 | *******************************************/ 60 | #include /* size_t, ptrdiff_t */ 61 | 62 | 63 | /* ***************************************** 64 | * FSE U16 functions 65 | *******************************************/ 66 | 67 | /*!FSE_compressU16() : 68 | data is presented or regenerated as a table of unsigned short (2 bytes per symbol), 69 | which is useful for alphabet size > 256. 70 | Important ! All symbol values within input table must be <= 'maxSymbolValue'. 71 | Maximum allowed 'maxSymbolValue' is controlled by constant FSE_MAX_SYMBOL_VALUE 72 | Special values : if result == 0, data is not compressible => Nothing is stored within cSrc !! 73 | if result == 1, data is one constant element x srcSize times. Use RLE compression. 74 | if FSE_isError(result), it's an error code.*/ 75 | size_t FSE_compressU16(void* dst, size_t dstCapacity, 76 | const unsigned short* src, size_t srcSize, 77 | unsigned maxSymbolValue, unsigned tableLog); 78 | 79 | size_t FSE_decompressU16(unsigned short* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize); 80 | 81 | 82 | 83 | #if defined (__cplusplus) 84 | } 85 | #endif 86 | 87 | #endif /* FSE_U16_H_30982039483 */ 88 | -------------------------------------------------------------------------------- /FSE/debug.h: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * debug 3 | * Part of FSE library 4 | * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. 5 | * 6 | * You can contact the author at : 7 | * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 8 | * 9 | * This source code is licensed under both the BSD-style license (found in the 10 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 11 | * in the COPYING file in the root directory of this source tree). 12 | * You may select, at your option, one of the above-listed licenses. 13 | ****************************************************************** */ 14 | 15 | 16 | /* 17 | * The purpose of this header is to enable debug functions. 18 | * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time, 19 | * and DEBUG_STATIC_ASSERT() for compile-time. 20 | * 21 | * By default, DEBUGLEVEL==0, which means run-time debug is disabled. 22 | * 23 | * Level 1 enables assert() only. 24 | * Starting level 2, traces can be generated and pushed to stderr. 25 | * The higher the level, the more verbose the traces. 26 | * 27 | * It's possible to dynamically adjust level using variable g_debug_level, 28 | * which is only declared if DEBUGLEVEL>=2, 29 | * and is a global variable, not multi-thread protected (use with care) 30 | */ 31 | 32 | #ifndef DEBUG_H_12987983217 33 | #define DEBUG_H_12987983217 34 | 35 | #if defined (__cplusplus) 36 | extern "C" { 37 | #endif 38 | 39 | 40 | /* static assert is triggered at compile time, leaving no runtime artefact. 41 | * static assert only works with compile-time constants. 42 | * Also, this variant can only be used inside a function. */ 43 | #define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) 44 | 45 | 46 | /* DEBUGLEVEL is expected to be defined externally, 47 | * typically through compiler command line. 48 | * Value must be a number. */ 49 | #ifndef DEBUGLEVEL 50 | # define DEBUGLEVEL 0 51 | #endif 52 | 53 | 54 | /* DEBUGFILE can be defined externally, 55 | * typically through compiler command line. 56 | * note : currently useless. 57 | * Value must be stderr or stdout */ 58 | #ifndef DEBUGFILE 59 | # define DEBUGFILE stderr 60 | #endif 61 | 62 | 63 | /* recommended values for DEBUGLEVEL : 64 | * 0 : release mode, no debug, all run-time checks disabled 65 | * 1 : enables assert() only, no display 66 | * 2 : reserved, for currently active debug path 67 | * 3 : events once per object lifetime (CCtx, CDict, etc.) 68 | * 4 : events once per frame 69 | * 5 : events once per block 70 | * 6 : events once per sequence (verbose) 71 | * 7+: events at every position (*very* verbose) 72 | * 73 | * It's generally inconvenient to output traces > 5. 74 | * In which case, it's possible to selectively trigger high verbosity levels 75 | * by modifying g_debug_level. 76 | */ 77 | 78 | #if (DEBUGLEVEL>=1) 79 | # include 80 | #else 81 | # ifndef assert /* assert may be already defined, due to prior #include */ 82 | # define assert(condition) ((void)0) /* disable assert (default) */ 83 | # endif 84 | #endif 85 | 86 | #if (DEBUGLEVEL>=2) 87 | # include 88 | extern int g_debuglevel; /* the variable is only declared, 89 | it actually lives in debug.c, 90 | and is shared by the whole process. 91 | It's not thread-safe. 92 | It's useful when enabling very verbose levels 93 | on selective conditions (such as position in src) */ 94 | 95 | # define RAWLOG(l, ...) { \ 96 | if (l<=g_debuglevel) { \ 97 | fprintf(stderr, __VA_ARGS__); \ 98 | } } 99 | # define DEBUGLOG(l, ...) { \ 100 | if (l<=g_debuglevel) { \ 101 | fprintf(stderr, __FILE__ ": " __VA_ARGS__); \ 102 | fprintf(stderr, " \n"); \ 103 | } } 104 | #else 105 | # define RAWLOG(l, ...) {} /* disabled */ 106 | # define DEBUGLOG(l, ...) {} /* disabled */ 107 | #endif 108 | 109 | 110 | #if defined (__cplusplus) 111 | } 112 | #endif 113 | 114 | #endif /* DEBUG_H_12987983217 */ 115 | -------------------------------------------------------------------------------- /FSE/error_private.h: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | Error codes and messages 3 | Copyright (C) 2013-2016, Yann Collet 4 | 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - Homepage : http://www.zstd.net 32 | ****************************************************************** */ 33 | /* Note : this module is expected to remain private, do not expose it */ 34 | 35 | #ifndef ERROR_H_MODULE 36 | #define ERROR_H_MODULE 37 | 38 | #if defined (__cplusplus) 39 | extern "C" { 40 | #endif 41 | 42 | 43 | /* **************************************** 44 | * Dependencies 45 | ******************************************/ 46 | #include /* size_t */ 47 | #include "error_public.h" /* enum list */ 48 | 49 | 50 | /* **************************************** 51 | * Compiler-specific 52 | ******************************************/ 53 | #if defined(__GNUC__) 54 | # define ERR_STATIC static __attribute__((unused)) 55 | #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 56 | # define ERR_STATIC static inline 57 | #elif defined(_MSC_VER) 58 | # define ERR_STATIC static __inline 59 | #else 60 | # define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ 61 | #endif 62 | 63 | 64 | /*-**************************************** 65 | * Customization (error_public.h) 66 | ******************************************/ 67 | typedef FSE_ErrorCode ERR_enum; 68 | #define PREFIX(name) FSE_error_##name 69 | 70 | 71 | /*-**************************************** 72 | * Error codes handling 73 | ******************************************/ 74 | #ifdef ERROR 75 | # undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ 76 | #endif 77 | #define ERROR(name) ((size_t)-PREFIX(name)) 78 | 79 | ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } 80 | 81 | ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } 82 | 83 | /* check and forward error code */ 84 | #define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e 85 | #define CHECK_F(f) { CHECK_V_F(_var_err__, f); } 86 | 87 | 88 | /*-**************************************** 89 | * Error Strings 90 | ******************************************/ 91 | 92 | ERR_STATIC const char* ERR_getErrorString(ERR_enum code) 93 | { 94 | static const char* notErrorCode = "Unspecified error code"; 95 | switch( code ) 96 | { 97 | case PREFIX(no_error): return "No error detected"; 98 | case PREFIX(GENERIC): return "Error (generic)"; 99 | case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; 100 | case PREFIX(srcSize_wrong): return "Src size is incorrect"; 101 | case PREFIX(corruption_detected): return "Corrupted block detected"; 102 | case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; 103 | case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; 104 | case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; 105 | case PREFIX(workSpace_tooSmall): return "workspace buffer is too small"; 106 | case PREFIX(maxCode): 107 | default: return notErrorCode; 108 | } 109 | } 110 | 111 | ERR_STATIC const char* ERR_getErrorName(size_t code) 112 | { 113 | return ERR_getErrorString(ERR_getErrorCode(code)); 114 | } 115 | 116 | #if defined (__cplusplus) 117 | } 118 | #endif 119 | 120 | #endif /* ERROR_H_MODULE */ 121 | -------------------------------------------------------------------------------- /halac_encode_v.0.1.9.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "FSE/fse.h" 11 | #include "FSE/huf.h" 12 | ////////////////////////////////////////// 13 | using namespace std; 14 | namespace fs = std::filesystem; 15 | ////////////////////////////////////////// 16 | typedef char i8; 17 | typedef unsigned char u8; 18 | typedef short i16; 19 | typedef unsigned short u16; 20 | typedef int i32; 21 | typedef unsigned int u32; 22 | typedef long long i64; 23 | typedef unsigned long long u64; 24 | ////////////////////////////////////////// 25 | const i8 VERSION[] = "HALAC 0.1.9 - Single Thread"; 26 | string INPUT_FILENAME, OUTPUT_FILENAME; 27 | FILE* INPUT_FILE, * OUTPUT_FILE; 28 | u32 FILE_SIZE, INPUT_SIZE, DATA_SIZE, SAMPLE_RATE, METADATA_SIZE, OUTPUT_SIZE = 0, TOTAL_OUTPUT_SIZE = 0; 29 | u16 CHANNEL_COUNT, SAMPLE_BIT, SAMPLE_BYTE; 30 | const u32 BLOCK_SIZE = 1024 * 24; 31 | const u8 ORDER = 8; 32 | ////////////////////////////////////////// 33 | i8 FILE_TYPE[4]; 34 | i8 TEMP1[14]; 35 | i8 TEMP2[6]; 36 | i8 METADATA[1024 * 16]; 37 | ////////////////////////////////////////// 38 | static inline u32 Signed_To_Unsigned(i32 num) { 39 | u32 mask = num >> 31; 40 | return (num << 1) ^ mask; 41 | } 42 | 43 | u32 HUF_ENCODE(u8* data, u32 data_size, u8* OUT, u32 &OUT_COUNTER) { 44 | const u32 max_compressed_size = HUF_compressBound(data_size); 45 | u8 compressed[max_compressed_size]; 46 | u32 compressed_size = HUF_compress(compressed, max_compressed_size, data, data_size); 47 | 48 | memcpy(OUT + OUT_COUNTER, &compressed_size, 4); OUT_COUNTER += 4; 49 | 50 | if (compressed_size == 0) { 51 | memcpy(OUT + OUT_COUNTER, data, data_size); OUT_COUNTER += data_size; 52 | compressed_size = data_size; 53 | } 54 | else { 55 | memcpy(OUT + OUT_COUNTER, compressed, compressed_size); OUT_COUNTER += compressed_size; 56 | } 57 | 58 | OUTPUT_SIZE += compressed_size; 59 | return compressed_size; 60 | } 61 | 62 | u32 FSE_ENCODE(u8* data, u32 data_size, u8* OUT, u32 &OUT_COUNTER) { 63 | const u32 max_compressed_size = FSE_compressBound(data_size); 64 | u8 compressed[max_compressed_size]; 65 | u32 compressed_size = FSE_compress(compressed, max_compressed_size, data, data_size); 66 | 67 | memcpy(OUT + OUT_COUNTER, &compressed_size, 4); OUT_COUNTER += 4; 68 | 69 | if (compressed_size > 1) { 70 | memcpy(OUT + OUT_COUNTER, compressed, compressed_size); OUT_COUNTER += compressed_size; 71 | } 72 | else if (compressed_size == 1) { 73 | memcpy(OUT + OUT_COUNTER, &data[0], 1); OUT_COUNTER += 1; 74 | compressed_size++; 75 | } 76 | else { 77 | memcpy(OUT + OUT_COUNTER, data, data_size); OUT_COUNTER += data_size; 78 | compressed_size = data_size; 79 | } 80 | 81 | OUTPUT_SIZE += compressed_size; 82 | return compressed_size; 83 | } 84 | 85 | void READ_WAV_HEADER() { 86 | INPUT_FILE = fopen(INPUT_FILENAME.c_str(), "rb"); 87 | 88 | char RIFF[4]; 89 | fread(RIFF, 1, 4, INPUT_FILE); 90 | 91 | if (RIFF[0] != 'R' || RIFF[1] != 'I' || RIFF[2] != 'F' || RIFF[3] != 'F') { 92 | printf("Only WAV file type is supported for input!\n"); 93 | exit(0); 94 | } 95 | 96 | fread(&FILE_SIZE, 1, 4, INPUT_FILE); 97 | if (FILE_SIZE < 1024) { 98 | printf("Minimum file size must be largen than 1kb!\n"); 99 | return; 100 | } 101 | 102 | fread(&TEMP1, 1, 14, INPUT_FILE); 103 | fread(&CHANNEL_COUNT, 1, 2, INPUT_FILE); 104 | fread(&SAMPLE_RATE, 1, 4, INPUT_FILE); 105 | fread(&TEMP2, 1, 6, INPUT_FILE); 106 | fread(&SAMPLE_BIT, 1, 2, INPUT_FILE); 107 | SAMPLE_BYTE = SAMPLE_BIT / 8; 108 | METADATA_SIZE = 0; 109 | 110 | ////////////////// 111 | fread(FILE_TYPE, 1, 4, INPUT_FILE); 112 | 113 | if (FILE_TYPE[0] == 'd' && FILE_TYPE[1] == 'a' && FILE_TYPE[2] == 't' && FILE_TYPE[3] == 'a') { 114 | fread(&INPUT_SIZE, 1, 4, INPUT_FILE); 115 | } 116 | else { 117 | fread(&METADATA_SIZE, 1, 4, INPUT_FILE); 118 | fread(METADATA, 1, METADATA_SIZE, INPUT_FILE); 119 | 120 | fseek(INPUT_FILE, 4, SEEK_CUR); 121 | fread(&INPUT_SIZE, 1, 4, INPUT_FILE); 122 | } 123 | } 124 | 125 | void WRITE_HALAC_HEADER() { 126 | OUTPUT_FILE = fopen(OUTPUT_FILENAME.c_str(), "wb"); 127 | 128 | fwrite(VERSION, 1, 11, OUTPUT_FILE); 129 | fwrite(&FILE_SIZE, 1, 4, OUTPUT_FILE); 130 | fwrite(&TEMP1, 1, 14, OUTPUT_FILE); 131 | fwrite(&CHANNEL_COUNT, 1, 2, OUTPUT_FILE); 132 | fwrite(&SAMPLE_RATE, 1, 4, OUTPUT_FILE); 133 | fwrite(&TEMP2, 1, 6, OUTPUT_FILE); 134 | fwrite(&SAMPLE_BIT, 1, 2, OUTPUT_FILE); 135 | 136 | fwrite(&METADATA_SIZE, 1, 4, OUTPUT_FILE); 137 | if (METADATA_SIZE > 0) { 138 | fwrite(METADATA, 1, METADATA_SIZE, OUTPUT_FILE); 139 | } 140 | 141 | fwrite(&INPUT_SIZE, 1, 4, OUTPUT_FILE); 142 | } -------------------------------------------------------------------------------- /FSE/compiler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_COMPILER_H 12 | #define ZSTD_COMPILER_H 13 | 14 | /*-******************************************************* 15 | * Compiler specifics 16 | *********************************************************/ 17 | /* force inlining */ 18 | 19 | #if !defined(ZSTD_NO_INLINE) 20 | #if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ 21 | # define INLINE_KEYWORD inline 22 | #else 23 | # define INLINE_KEYWORD 24 | #endif 25 | 26 | #if defined(__GNUC__) || defined(__ICCARM__) 27 | # define FORCE_INLINE_ATTR __attribute__((always_inline)) 28 | #elif defined(_MSC_VER) 29 | # define FORCE_INLINE_ATTR __forceinline 30 | #else 31 | # define FORCE_INLINE_ATTR 32 | #endif 33 | 34 | #else 35 | 36 | #define INLINE_KEYWORD 37 | #define FORCE_INLINE_ATTR 38 | 39 | #endif 40 | 41 | /** 42 | On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). 43 | This explictly marks such functions as __cdecl so that the code will still compile 44 | if a CC other than __cdecl has been made the default. 45 | */ 46 | #if defined(_MSC_VER) 47 | # define WIN_CDECL __cdecl 48 | #else 49 | # define WIN_CDECL 50 | #endif 51 | 52 | /** 53 | * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant 54 | * parameters. They must be inlined for the compiler to eliminate the constant 55 | * branches. 56 | */ 57 | #define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR 58 | /** 59 | * HINT_INLINE is used to help the compiler generate better code. It is *not* 60 | * used for "templates", so it can be tweaked based on the compilers 61 | * performance. 62 | * 63 | * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the 64 | * always_inline attribute. 65 | * 66 | * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline 67 | * attribute. 68 | */ 69 | #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 70 | # define HINT_INLINE static INLINE_KEYWORD 71 | #else 72 | # define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR 73 | #endif 74 | 75 | /* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ 76 | #if defined(__GNUC__) 77 | # define UNUSED_ATTR __attribute__((unused)) 78 | #else 79 | # define UNUSED_ATTR 80 | #endif 81 | 82 | /* force no inlining */ 83 | #ifdef _MSC_VER 84 | # define FORCE_NOINLINE static __declspec(noinline) 85 | #else 86 | # if defined(__GNUC__) || defined(__ICCARM__) 87 | # define FORCE_NOINLINE static __attribute__((__noinline__)) 88 | # else 89 | # define FORCE_NOINLINE static 90 | # endif 91 | #endif 92 | 93 | /* target attribute */ 94 | #ifndef __has_attribute 95 | #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ 96 | #endif 97 | #if defined(__GNUC__) || defined(__ICCARM__) 98 | # define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) 99 | #else 100 | # define TARGET_ATTRIBUTE(target) 101 | #endif 102 | 103 | /* Enable runtime BMI2 dispatch based on the CPU. 104 | * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. 105 | */ 106 | #ifndef DYNAMIC_BMI2 107 | #if ((defined(__clang__) && __has_attribute(__target__)) \ 108 | || (defined(__GNUC__) \ 109 | && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ 110 | && (defined(__x86_64__) || defined(_M_X86)) \ 111 | && !defined(__BMI2__) 112 | # define DYNAMIC_BMI2 1 113 | #else 114 | # define DYNAMIC_BMI2 0 115 | #endif 116 | #endif 117 | 118 | /* prefetch 119 | * can be disabled, by declaring NO_PREFETCH build macro */ 120 | #if defined(NO_PREFETCH) 121 | # define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ 122 | # define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ 123 | #else 124 | # if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ 125 | # include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ 126 | # define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) 127 | # define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) 128 | # elif defined(__aarch64__) 129 | # define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) 130 | # define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) 131 | # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) 132 | # define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) 133 | # define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) 134 | # else 135 | # define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ 136 | # define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ 137 | # endif 138 | #endif /* NO_PREFETCH */ 139 | 140 | #define CACHELINE_SIZE 64 141 | 142 | #define PREFETCH_AREA(p, s) { \ 143 | const char* const _ptr = (const char*)(p); \ 144 | size_t const _size = (size_t)(s); \ 145 | size_t _pos; \ 146 | for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ 147 | PREFETCH_L2(_ptr + _pos); \ 148 | } \ 149 | } 150 | 151 | /* vectorization 152 | * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */ 153 | #if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) 154 | # if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) 155 | # define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) 156 | # else 157 | # define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")") 158 | # endif 159 | #else 160 | # define DONT_VECTORIZE 161 | #endif 162 | 163 | /* Tell the compiler that a branch is likely or unlikely. 164 | * Only use these macros if it causes the compiler to generate better code. 165 | * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc 166 | * and clang, please do. 167 | */ 168 | #if defined(__GNUC__) 169 | #define LIKELY(x) (__builtin_expect((x), 1)) 170 | #define UNLIKELY(x) (__builtin_expect((x), 0)) 171 | #else 172 | #define LIKELY(x) (x) 173 | #define UNLIKELY(x) (x) 174 | #endif 175 | 176 | /* disable warnings */ 177 | #ifdef _MSC_VER /* Visual Studio */ 178 | # include /* For Visual 2005 */ 179 | # pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ 180 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 181 | # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ 182 | # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ 183 | # pragma warning(disable : 4324) /* disable: C4324: padded structure */ 184 | #endif 185 | 186 | #endif /* ZSTD_COMPILER_H */ 187 | -------------------------------------------------------------------------------- /FSE/hist.c: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * hist : Histogram functions 3 | * part of Finite State Entropy project 4 | * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. 5 | * 6 | * You can contact the author at : 7 | * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy 8 | * - Public forum : https://groups.google.com/forum/#!forum/lz4c 9 | * 10 | * This source code is licensed under both the BSD-style license (found in the 11 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 12 | * in the COPYING file in the root directory of this source tree). 13 | * You may select, at your option, one of the above-listed licenses. 14 | ****************************************************************** */ 15 | 16 | /* --- dependencies --- */ 17 | #include "mem.h" /* U32, BYTE, etc. */ 18 | #include "debug.h" /* assert, DEBUGLOG */ 19 | #include "error_private.h" /* ERROR */ 20 | #include "hist.h" 21 | 22 | 23 | /* --- Error management --- */ 24 | unsigned HIST_isError(size_t code) { return ERR_isError(code); } 25 | 26 | /*-************************************************************** 27 | * Histogram functions 28 | ****************************************************************/ 29 | unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, 30 | const void* src, size_t srcSize) 31 | { 32 | const BYTE* ip = (const BYTE*)src; 33 | const BYTE* const end = ip + srcSize; 34 | unsigned maxSymbolValue = *maxSymbolValuePtr; 35 | unsigned largestCount=0; 36 | 37 | memset(count, 0, (maxSymbolValue+1) * sizeof(*count)); 38 | if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } 39 | 40 | while (ip largestCount) largestCount = count[s]; 51 | } 52 | 53 | return largestCount; 54 | } 55 | 56 | typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; 57 | 58 | /* HIST_count_parallel_wksp() : 59 | * store histogram into 4 intermediate tables, recombined at the end. 60 | * this design makes better use of OoO cpus, 61 | * and is noticeably faster when some values are heavily repeated. 62 | * But it needs some additional workspace for intermediate tables. 63 | * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32. 64 | * @return : largest histogram frequency, 65 | * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */ 66 | static size_t HIST_count_parallel_wksp( 67 | unsigned* count, unsigned* maxSymbolValuePtr, 68 | const void* source, size_t sourceSize, 69 | HIST_checkInput_e check, 70 | U32* const workSpace) 71 | { 72 | const BYTE* ip = (const BYTE*)source; 73 | const BYTE* const iend = ip+sourceSize; 74 | size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count); 75 | unsigned max=0; 76 | U32* const Counting1 = workSpace; 77 | U32* const Counting2 = Counting1 + 256; 78 | U32* const Counting3 = Counting2 + 256; 79 | U32* const Counting4 = Counting3 + 256; 80 | 81 | /* safety checks */ 82 | assert(*maxSymbolValuePtr <= 255); 83 | if (!sourceSize) { 84 | memset(count, 0, countSize); 85 | *maxSymbolValuePtr = 0; 86 | return 0; 87 | } 88 | memset(workSpace, 0, 4*256*sizeof(unsigned)); 89 | 90 | /* by stripes of 16 bytes */ 91 | { U32 cached = MEM_read32(ip); ip += 4; 92 | while (ip < iend-15) { 93 | U32 c = cached; cached = MEM_read32(ip); ip += 4; 94 | Counting1[(BYTE) c ]++; 95 | Counting2[(BYTE)(c>>8) ]++; 96 | Counting3[(BYTE)(c>>16)]++; 97 | Counting4[ c>>24 ]++; 98 | c = cached; cached = MEM_read32(ip); ip += 4; 99 | Counting1[(BYTE) c ]++; 100 | Counting2[(BYTE)(c>>8) ]++; 101 | Counting3[(BYTE)(c>>16)]++; 102 | Counting4[ c>>24 ]++; 103 | c = cached; cached = MEM_read32(ip); ip += 4; 104 | Counting1[(BYTE) c ]++; 105 | Counting2[(BYTE)(c>>8) ]++; 106 | Counting3[(BYTE)(c>>16)]++; 107 | Counting4[ c>>24 ]++; 108 | c = cached; cached = MEM_read32(ip); ip += 4; 109 | Counting1[(BYTE) c ]++; 110 | Counting2[(BYTE)(c>>8) ]++; 111 | Counting3[(BYTE)(c>>16)]++; 112 | Counting4[ c>>24 ]++; 113 | } 114 | ip-=4; 115 | } 116 | 117 | /* finish last symbols */ 118 | while (ip max) max = Counting1[s]; 124 | } } 125 | 126 | { unsigned maxSymbolValue = 255; 127 | while (!Counting1[maxSymbolValue]) maxSymbolValue--; 128 | if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall); 129 | *maxSymbolValuePtr = maxSymbolValue; 130 | memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */ 131 | } 132 | return (size_t)max; 133 | } 134 | 135 | 136 | /* HIST_countFast_wksp() : 137 | * Same as HIST_countFast(), but using an externally provided scratch buffer. 138 | * `workSpace` is a writable buffer which must be 4-bytes aligned, 139 | * `workSpaceSize` must be >= HIST_WKSP_SIZE 140 | */ 141 | size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, 142 | const void* source, size_t sourceSize, 143 | void* workSpace, size_t workSpaceSize) 144 | { 145 | if (sourceSize < 1500) /* heuristic threshold */ 146 | return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize); 147 | if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ 148 | if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); 149 | return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); 150 | } 151 | 152 | /* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ 153 | size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, 154 | const void* source, size_t sourceSize) 155 | { 156 | unsigned tmpCounters[HIST_WKSP_SIZE_U32]; 157 | return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); 158 | } 159 | 160 | /* HIST_count_wksp() : 161 | * Same as HIST_count(), but using an externally provided scratch buffer. 162 | * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ 163 | size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, 164 | const void* source, size_t sourceSize, 165 | void* workSpace, size_t workSpaceSize) 166 | { 167 | if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ 168 | if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); 169 | if (*maxSymbolValuePtr < 255) 170 | return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace); 171 | *maxSymbolValuePtr = 255; 172 | return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); 173 | } 174 | 175 | size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, 176 | const void* src, size_t srcSize) 177 | { 178 | unsigned tmpCounters[HIST_WKSP_SIZE_U32]; 179 | return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters)); 180 | } 181 | -------------------------------------------------------------------------------- /halac_encode_v.0.1.9.cpp: -------------------------------------------------------------------------------- 1 | #include "halac_encode_v.0.1.9.h" 2 | 3 | void Forward_Linear_Prediction(double* coeffs, const i32* data, u32 size) { 4 | int numSamples = size - 1; 5 | double autocorr[ORDER + 1]; 6 | 7 | for (int lag = 0; lag <= ORDER; lag++) { 8 | autocorr[lag] = 0.0; 9 | int samples = numSamples - lag; 10 | do { autocorr[lag] += (double)data[samples] * data[samples + lag]; } while (samples--); 11 | } 12 | 13 | double lpCoeffs[ORDER + 1]; 14 | lpCoeffs[0] = 1.0; 15 | for (int i = 1; i <= ORDER; i++) lpCoeffs[i] = 0.0; 16 | 17 | double predError = autocorr[0] != 0.0 ? autocorr[0] : 1.0; 18 | 19 | for (int order = 0; order < ORDER; order++) { 20 | double reflection = -lpCoeffs[0] * autocorr[order + 1]; 21 | for (int j = 1; j <= order; j++) reflection -= lpCoeffs[j] * autocorr[order + 1 - j]; 22 | reflection /= predError; 23 | 24 | double tempCoeffs[ORDER + 1]; 25 | for (int n = 0; n <= order + 1; n++) tempCoeffs[n] = lpCoeffs[n] + reflection * lpCoeffs[order + 1 - n]; 26 | for (int n = 0; n <= order + 1; n++) lpCoeffs[n] = tempCoeffs[n]; 27 | 28 | predError -= predError * reflection * reflection; 29 | } 30 | 31 | for (int i = 0; i < ORDER; i++) coeffs[i] = lpCoeffs[i + 1]; 32 | } 33 | 34 | void COMPRESS(i32* data, u32 sample_count, u8* OUT, u32 &OUT_COUNTER) { 35 | 36 | //////////////////////////// 37 | double coeffs[ORDER]; 38 | Forward_Linear_Prediction(coeffs, data, sample_count); 39 | 40 | float coeffs_float[ORDER]; 41 | for (int i = 0; i < ORDER; i++) coeffs[i] = coeffs_float[i] = coeffs[i]; 42 | 43 | double predicted; 44 | i32 data_error[sample_count]; 45 | for (int i = 0; i < ORDER; i++) data_error[i] = data[i]; 46 | 47 | for (int k = ORDER; k < sample_count; k++) { 48 | predicted = 0.0; 49 | for (int j = 0; j < ORDER; j++) { 50 | predicted -= coeffs[j] * data[k - 1 - j]; 51 | } 52 | 53 | data_error[k] = data[k] - (int)predicted; 54 | } 55 | 56 | memcpy(OUT + OUT_COUNTER, coeffs_float, ORDER * 4); 57 | OUT_COUNTER += ORDER * 4; 58 | //////////////////////////// 59 | 60 | u8 data_first[sample_count]; 61 | u8 data_last[sample_count]; 62 | u32 over_data[1024 * 16]; 63 | u16 over_counter = 0; 64 | 65 | for (int i = 0; i < sample_count; i++) { 66 | u32 value = Signed_To_Unsigned(data_error[i]); 67 | 68 | if (value < 65535) { 69 | data_first[i] = value >> 8; 70 | data_last[i] = value; 71 | } 72 | else { 73 | over_data[over_counter] = value - 65535; 74 | over_counter++; 75 | 76 | data_first[i] = 255; 77 | data_last[i] = 255; 78 | } 79 | 80 | } 81 | memcpy(OUT + OUT_COUNTER, &over_counter, 2); 82 | OUT_COUNTER += 2; 83 | 84 | if (over_counter > 0) { 85 | memcpy(OUT + OUT_COUNTER, &over_data, over_counter * 4); 86 | OUT_COUNTER += over_counter * 4; 87 | } 88 | 89 | FSE_ENCODE(data_first, sample_count, OUT, OUT_COUNTER); 90 | HUF_ENCODE(data_last, sample_count, OUT, OUT_COUNTER); 91 | } 92 | 93 | void TRANSFORM(i16* data, u32 data_size, u8* OUT, u32 &OUT_COUNTER) { 94 | //////////////////////////// 95 | if (OUT_COUNTER > 1024 * 512) { 96 | fwrite(OUT, 1, OUT_COUNTER, OUTPUT_FILE); 97 | OUT_COUNTER = 0; 98 | } 99 | 100 | u32 sample_count = data_size / CHANNEL_COUNT; 101 | i32 data_left[sample_count]; 102 | i32 data_right[sample_count]; 103 | for (int i = 0; i < sample_count; i++) { 104 | data_left[i] = data[2 * i]; 105 | data_right[i] = data[2 * i + 1]; 106 | } 107 | //////////////////////////// 108 | int L, R, X, Y; 109 | for (int i = 0; i < sample_count; i++) { 110 | L = data_left[i]; 111 | R = data_right[i]; 112 | X = L - R; 113 | Y = R + (X / 2); 114 | data_left[i] = X; 115 | data_right[i] = Y; 116 | } 117 | //////////////////////////// 118 | i32 data_left_delta[sample_count]; 119 | i32 data_right_delta[sample_count]; 120 | data_left_delta[0] = data_left[0]; 121 | data_right_delta[0] = data_right[0]; 122 | 123 | for (int i = 1; i < sample_count; i++) { 124 | data_left_delta[i] = data_left[i] - data_left[i - 1]; 125 | data_right_delta[i] = data_right[i] - data_right[i - 1]; 126 | } 127 | //////////////////////////// 128 | 129 | COMPRESS(data_left_delta, sample_count, OUT, OUT_COUNTER); 130 | COMPRESS(data_right_delta, sample_count, OUT, OUT_COUNTER); 131 | } 132 | 133 | void ENCODING() { 134 | READ_WAV_HEADER(); 135 | WRITE_HALAC_HEADER(); 136 | ////////////// 137 | u8 OUT[1024 * 1024]; 138 | u32 OUT_COUNTER = 0; 139 | 140 | u32 data_mod = INPUT_SIZE % BLOCK_SIZE; 141 | u32 block_count = INPUT_SIZE / BLOCK_SIZE - 1; 142 | u32 last_block_size; 143 | 144 | if (block_count <= 0) last_block_size = INPUT_SIZE; 145 | else last_block_size = BLOCK_SIZE + data_mod; 146 | 147 | i16 data[BLOCK_SIZE]; 148 | for (int i = 0; i < block_count; i++) { 149 | fread(data, 2, BLOCK_SIZE / SAMPLE_BYTE, INPUT_FILE); 150 | TRANSFORM(data, BLOCK_SIZE / SAMPLE_BYTE, OUT, OUT_COUNTER); 151 | } 152 | 153 | fread(data, 2, last_block_size / SAMPLE_BYTE, INPUT_FILE); 154 | TRANSFORM(data, last_block_size / SAMPLE_BYTE, OUT, OUT_COUNTER); 155 | 156 | if (OUT_COUNTER > 0) { 157 | fwrite(OUT, 1, OUT_COUNTER, OUTPUT_FILE); 158 | } 159 | 160 | ////////////// 161 | fclose(INPUT_FILE); 162 | fclose(OUTPUT_FILE); 163 | } 164 | 165 | int main(int argc, char** argv) 166 | { 167 | /* 168 | ///////////// Reading Console Arguments ///////////// 169 | if (argc != 3) { 170 | printf(VERSION); 171 | printf(" Encoder\n" 172 | "Hakan Abbas (abbas.hakan@gmail.com)\n" 173 | "Only 16 bit, 2 channels .wav types are supported for input!\n" 174 | "Please use this format: \"encoder.exe input_file output_file\"\n"); 175 | return 0; 176 | } 177 | INPUT_FILENAME = argv[1]; 178 | OUTPUT_FILENAME = argv[2]; 179 | ENCODING(); 180 | ///////////////////////////////////////////////////// 181 | */ 182 | 183 | ////////////////////////////////////////// 184 | clock_t startTime = clock(); 185 | double time; 186 | 187 | string path; 188 | //path = "../WAV/ISKENDER_PAYDAS"; 189 | //path = "../WAV/test"; 190 | //path = "../WAV/TEST_AUDIO_SQUEEZE_CHART"; 191 | //path = "../WAV/HANDE_YENER"; 192 | //path = "../WAV/SIBEL_CAN"; 193 | //path = "../WAV/TEST_AUDIOS"; 194 | //path = "../WAV/YILDIZ_TILBE"; 195 | //path = "../WAV/single"; 196 | path = "D:/TEST_AUDIOS/YABANCI/Busta_Rhymes"; 197 | ////////////////////////////////////////// 198 | 199 | vector files; 200 | for (const auto& entry : fs::directory_iterator(path)) { 201 | if (!entry.is_directory() && (entry.path().extension() == ".wav")) 202 | files.emplace_back(entry.path().string()); 203 | } 204 | sort(files.begin(), files.end()); 205 | ////////////////////////////////////////// 206 | 207 | TOTAL_OUTPUT_SIZE = 0; 208 | for (u32 i = 0; i < files.size(); i++) { 209 | INPUT_FILENAME = files.at(i); 210 | OUTPUT_FILENAME = INPUT_FILENAME.substr(0, INPUT_FILENAME.rfind('.')) + ".halac"; 211 | cout << OUTPUT_FILENAME << endl; 212 | 213 | OUTPUT_SIZE = 0; 214 | ENCODING(); 215 | TOTAL_OUTPUT_SIZE += OUTPUT_SIZE; 216 | } 217 | cout << TOTAL_OUTPUT_SIZE << endl; 218 | 219 | ////////////////////////////////////////// 220 | time = static_cast(clock() - (double)startTime) / CLOCKS_PER_SEC; 221 | printf("-----------\n"); 222 | printf("Total Time: %2.3f seconds\n", time); 223 | ////////////////////////////////////////// 224 | 225 | return 0; 226 | } 227 | -------------------------------------------------------------------------------- /FSE/entropy_common.c: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * Common functions of New Generation Entropy library 3 | * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. 4 | * 5 | * You can contact the author at : 6 | * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy 7 | * - Public forum : https://groups.google.com/forum/#!forum/lz4c 8 | * 9 | * This source code is licensed under both the BSD-style license (found in the 10 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 11 | * in the COPYING file in the root directory of this source tree). 12 | * You may select, at your option, one of the above-listed licenses. 13 | ****************************************************************** */ 14 | 15 | /* ************************************* 16 | * Dependencies 17 | ***************************************/ 18 | #include "mem.h" 19 | #include "error_private.h" /* ERR_*, ERROR */ 20 | #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ 21 | #include "fse.h" 22 | #define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ 23 | #include "huf.h" 24 | 25 | 26 | /*=== Version ===*/ 27 | unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } 28 | 29 | 30 | /*=== Error Management ===*/ 31 | unsigned FSE_isError(size_t code) { return ERR_isError(code); } 32 | const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } 33 | 34 | unsigned HUF_isError(size_t code) { return ERR_isError(code); } 35 | const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } 36 | 37 | 38 | /*-************************************************************** 39 | * FSE NCount encoding-decoding 40 | ****************************************************************/ 41 | size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, 42 | const void* headerBuffer, size_t hbSize) 43 | { 44 | const BYTE* const istart = (const BYTE*) headerBuffer; 45 | const BYTE* const iend = istart + hbSize; 46 | const BYTE* ip = istart; 47 | int nbBits; 48 | int remaining; 49 | int threshold; 50 | U32 bitStream; 51 | int bitCount; 52 | unsigned charnum = 0; 53 | int previous0 = 0; 54 | 55 | if (hbSize < 4) { 56 | /* This function only works when hbSize >= 4 */ 57 | char buffer[4] = {0}; 58 | memcpy(buffer, headerBuffer, hbSize); 59 | { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, 60 | buffer, sizeof(buffer)); 61 | if (FSE_isError(countSize)) return countSize; 62 | if (countSize > hbSize) return ERROR(corruption_detected); 63 | return countSize; 64 | } } 65 | assert(hbSize >= 4); 66 | 67 | /* init */ 68 | memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ 69 | bitStream = MEM_readLE32(ip); 70 | nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ 71 | if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); 72 | bitStream >>= 4; 73 | bitCount = 4; 74 | *tableLogPtr = nbBits; 75 | remaining = (1<1) & (charnum<=*maxSVPtr)) { 80 | if (previous0) { 81 | unsigned n0 = charnum; 82 | while ((bitStream & 0xFFFF) == 0xFFFF) { 83 | n0 += 24; 84 | if (ip < iend-5) { 85 | ip += 2; 86 | bitStream = MEM_readLE32(ip) >> bitCount; 87 | } else { 88 | bitStream >>= 16; 89 | bitCount += 16; 90 | } } 91 | while ((bitStream & 3) == 3) { 92 | n0 += 3; 93 | bitStream >>= 2; 94 | bitCount += 2; 95 | } 96 | n0 += bitStream & 3; 97 | bitCount += 2; 98 | if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); 99 | while (charnum < n0) normalizedCounter[charnum++] = 0; 100 | if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { 101 | assert((bitCount >> 3) <= 3); /* For first condition to work */ 102 | ip += bitCount>>3; 103 | bitCount &= 7; 104 | bitStream = MEM_readLE32(ip) >> bitCount; 105 | } else { 106 | bitStream >>= 2; 107 | } } 108 | { int const max = (2*threshold-1) - remaining; 109 | int count; 110 | 111 | if ((bitStream & (threshold-1)) < (U32)max) { 112 | count = bitStream & (threshold-1); 113 | bitCount += nbBits-1; 114 | } else { 115 | count = bitStream & (2*threshold-1); 116 | if (count >= threshold) count -= max; 117 | bitCount += nbBits; 118 | } 119 | 120 | count--; /* extra accuracy */ 121 | remaining -= count < 0 ? -count : count; /* -1 means +1 */ 122 | normalizedCounter[charnum++] = (short)count; 123 | previous0 = !count; 124 | while (remaining < threshold) { 125 | nbBits--; 126 | threshold >>= 1; 127 | } 128 | 129 | if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { 130 | ip += bitCount>>3; 131 | bitCount &= 7; 132 | } else { 133 | bitCount -= (int)(8 * (iend - 4 - ip)); 134 | ip = iend - 4; 135 | } 136 | bitStream = MEM_readLE32(ip) >> (bitCount & 31); 137 | } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ 138 | if (remaining != 1) return ERROR(corruption_detected); 139 | if (bitCount > 32) return ERROR(corruption_detected); 140 | *maxSVPtr = charnum-1; 141 | 142 | ip += (bitCount+7)>>3; 143 | return ip-istart; 144 | } 145 | 146 | 147 | /*! HUF_readStats() : 148 | Read compact Huffman tree, saved by HUF_writeCTable(). 149 | `huffWeight` is destination buffer. 150 | `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. 151 | @return : size read from `src` , or an error Code . 152 | Note : Needed by HUF_readCTable() and HUF_readDTableX?() . 153 | */ 154 | size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, 155 | U32* nbSymbolsPtr, U32* tableLogPtr, 156 | const void* src, size_t srcSize) 157 | { 158 | U32 weightTotal; 159 | const BYTE* ip = (const BYTE*) src; 160 | size_t iSize; 161 | size_t oSize; 162 | 163 | if (!srcSize) return ERROR(srcSize_wrong); 164 | iSize = ip[0]; 165 | /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ 166 | 167 | if (iSize >= 128) { /* special header */ 168 | oSize = iSize - 127; 169 | iSize = ((oSize+1)/2); 170 | if (iSize+1 > srcSize) return ERROR(srcSize_wrong); 171 | if (oSize >= hwSize) return ERROR(corruption_detected); 172 | ip += 1; 173 | { U32 n; 174 | for (n=0; n> 4; 176 | huffWeight[n+1] = ip[n/2] & 15; 177 | } } } 178 | else { /* header compressed with FSE (normal case) */ 179 | FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */ 180 | if (iSize+1 > srcSize) return ERROR(srcSize_wrong); 181 | oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */ 182 | if (FSE_isError(oSize)) return oSize; 183 | } 184 | 185 | /* collect weight stats */ 186 | memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); 187 | weightTotal = 0; 188 | { U32 n; for (n=0; n= HUF_TABLELOG_MAX) return ERROR(corruption_detected); 190 | rankStats[huffWeight[n]]++; 191 | weightTotal += (1 << huffWeight[n]) >> 1; 192 | } } 193 | if (weightTotal == 0) return ERROR(corruption_detected); 194 | 195 | /* get last non-null symbol weight (implied, total must be 2^n) */ 196 | { U32 const tableLog = BIT_highbit32(weightTotal) + 1; 197 | if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); 198 | *tableLogPtr = tableLog; 199 | /* determine last weight */ 200 | { U32 const total = 1 << tableLog; 201 | U32 const rest = total - weightTotal; 202 | U32 const verif = 1 << BIT_highbit32(rest); 203 | U32 const lastWeight = BIT_highbit32(rest) + 1; 204 | if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ 205 | huffWeight[oSize] = (BYTE)lastWeight; 206 | rankStats[lastWeight]++; 207 | } } 208 | 209 | /* check tree construction validity */ 210 | if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ 211 | 212 | /* results */ 213 | *nbSymbolsPtr = (U32)(oSize+1); 214 | return iSize+1; 215 | } 216 | -------------------------------------------------------------------------------- /halac_decode_v.0.1.9.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "halac_decode_v.0.1.9.h" 3 | 4 | void TRANSFORM(i32* data_L, i32* data_R, u32 sample_count, u8* OUT, u32& OUT_COUNTER) { 5 | 6 | i32 data_delta_L[sample_count]; 7 | i32 data_delta_R[sample_count]; 8 | 9 | data_delta_L[0] = data_L[0]; 10 | data_delta_R[0] = data_R[0]; 11 | 12 | i32 total = data_delta_L[0]; 13 | for (u32 i = 1; i < sample_count; i++) { 14 | total += data_L[i]; 15 | data_delta_L[i] = total; 16 | } 17 | 18 | total = data_delta_R[0]; 19 | for (u32 i = 1; i < sample_count; i++) { 20 | total += data_R[i]; 21 | data_delta_R[i] = total; 22 | } 23 | /////////////////////////// 24 | 25 | i32 data_transform_L[sample_count]; 26 | i32 data_transform_R[sample_count]; 27 | int X, Y; 28 | 29 | for (u32 i = 0; i < sample_count; i++) { 30 | X = data_delta_L[i]; 31 | Y = data_delta_R[i]; 32 | 33 | data_transform_R[i] = Y - (X / 2); 34 | data_transform_L[i] = X + data_transform_R[i]; 35 | } 36 | /////////////////////////// 37 | 38 | i16 data[sample_count * CHANNEL_COUNT]; 39 | for (u32 i = 0; i < sample_count; i++) { 40 | data[2 * i] = data_transform_L[i]; 41 | data[2 * i + 1] = data_transform_R[i]; 42 | } 43 | 44 | memcpy(OUT + OUT_COUNTER, data, sample_count * CHANNEL_COUNT * 2); 45 | OUT_COUNTER += sample_count * CHANNEL_COUNT * 2; 46 | } 47 | 48 | void UNCOMPRESS(u32 block_size, u8* OUT, u32& OUT_COUNTER) { 49 | //////////////////////////// 50 | if (OUT_COUNTER > 1024 * 512) { 51 | fwrite(OUT, 1, OUT_COUNTER, OUTPUT_FILE); 52 | OUT_COUNTER = 0; 53 | } 54 | //////////////////////////// 55 | 56 | float coeffs_L[ORDER] = { 0 }; 57 | fread(coeffs_L, 4, ORDER, INPUT_FILE); 58 | //////////////////////////////////// 59 | 60 | u16 over_counter_L; 61 | fread(&over_counter_L, 1, 2, INPUT_FILE); 62 | u32 over_data_L[over_counter_L]; 63 | if (over_counter_L > 0) fread(over_data_L, 4, over_counter_L, INPUT_FILE); 64 | ////////////////////////////////// 65 | 66 | u32 fse_size_L_first; 67 | fread(&fse_size_L_first, 1, 4, INPUT_FILE); 68 | u8 fse_data_L_first[fse_size_L_first]; 69 | u8 uncompressed_fse_data_L_first[block_size]; 70 | 71 | if (fse_size_L_first > 1) { 72 | fread(fse_data_L_first, 1, fse_size_L_first, INPUT_FILE); 73 | FSE_decompress(uncompressed_fse_data_L_first, block_size, fse_data_L_first, fse_size_L_first); 74 | } 75 | else if (fse_size_L_first == 0) { 76 | fread(uncompressed_fse_data_L_first, 1, block_size, INPUT_FILE); 77 | } 78 | else if (fse_size_L_first == 1) { 79 | fread(fse_data_L_first, 1, 1, INPUT_FILE); 80 | fill(uncompressed_fse_data_L_first, uncompressed_fse_data_L_first + block_size, fse_data_L_first[0]); 81 | } 82 | ////////////////////////////////// 83 | u32 huf_size_L_last; 84 | fread(&huf_size_L_last, 1, 4, INPUT_FILE); 85 | 86 | u8 huf_data_L_last[huf_size_L_last]; 87 | u8 uncompressed_huf_data_L_last[block_size]; 88 | 89 | if (huf_size_L_last > 1) { 90 | fread(huf_data_L_last, 1, huf_size_L_last, INPUT_FILE); 91 | HUF_decompress(uncompressed_huf_data_L_last, block_size, huf_data_L_last, huf_size_L_last); 92 | } 93 | else if (huf_size_L_last == 0) { 94 | fread(uncompressed_huf_data_L_last, 1, block_size, INPUT_FILE); 95 | } 96 | else if (huf_size_L_last == 1) { 97 | fread(huf_data_L_last, 1, 1, INPUT_FILE); 98 | fill(uncompressed_huf_data_L_last, uncompressed_huf_data_L_last + block_size, huf_data_L_last[0]); 99 | } 100 | 101 | ////////////////////////////////// 102 | float coeffs_R[ORDER] = { 0 }; 103 | fread(coeffs_R, 4, ORDER, INPUT_FILE); 104 | //////////////////////////////////// 105 | 106 | u16 over_counter_R; 107 | fread(&over_counter_R, 1, 2, INPUT_FILE); 108 | u32 over_data_R[over_counter_R]; 109 | if (over_counter_R > 0) fread(over_data_R, 4, over_counter_R, INPUT_FILE); 110 | ////////////////////////////////// 111 | 112 | u32 fse_size_R_first; 113 | fread(&fse_size_R_first, 1, 4, INPUT_FILE); 114 | 115 | u8 fse_data_R_first[fse_size_R_first]; 116 | u8 uncompressed_fse_data_R_first[block_size]; 117 | 118 | if (fse_size_R_first > 1) { 119 | fread(fse_data_R_first, 1, fse_size_R_first, INPUT_FILE); 120 | FSE_decompress(uncompressed_fse_data_R_first, block_size, fse_data_R_first, fse_size_R_first); 121 | } 122 | else if (fse_size_R_first == 0) { 123 | fread(uncompressed_fse_data_R_first, 1, block_size, INPUT_FILE); 124 | } 125 | else if (fse_size_R_first == 1) { 126 | fread(fse_data_R_first, 1, 1, INPUT_FILE); 127 | fill(uncompressed_fse_data_R_first, uncompressed_fse_data_R_first + block_size, fse_data_R_first[0]); 128 | } 129 | ////////////////////////////////// 130 | u32 huf_size_R_last; 131 | fread(&huf_size_R_last, 1, 4, INPUT_FILE); 132 | 133 | u8 huf_data_R_last[huf_size_R_last]; 134 | u8 uncompressed_huf_data_R_last[block_size]; 135 | 136 | if (huf_size_R_last > 1) { 137 | fread(huf_data_R_last, 1, huf_size_R_last, INPUT_FILE); 138 | HUF_decompress(uncompressed_huf_data_R_last, block_size, huf_data_R_last, huf_size_R_last); 139 | } 140 | else if (huf_size_R_last == 0) { 141 | fread(uncompressed_huf_data_R_last, 1, block_size, INPUT_FILE); 142 | } 143 | else if (huf_size_R_last == 1) { 144 | fread(huf_data_R_last, 1, 1, INPUT_FILE); 145 | fill(uncompressed_huf_data_R_last, uncompressed_huf_data_R_last + block_size, huf_data_R_last[0]); 146 | } 147 | ////////////////////////////////// 148 | 149 | u32 data_L; 150 | u32 data_R; 151 | i32 data_L_signed[block_size]; 152 | i32 data_R_signed[block_size]; 153 | u8 counter = 0; 154 | 155 | for (int i = 0; i < block_size; i++) { 156 | data_L = (uncompressed_fse_data_L_first[i] << 8) + uncompressed_huf_data_L_last[i]; 157 | 158 | if (data_L == 65535) { 159 | data_L += over_data_L[counter]; 160 | counter++; 161 | } 162 | 163 | data_L_signed[i] = Unsigned_To_Signed(data_L); 164 | } 165 | 166 | counter = 0; 167 | for (int i = 0; i < block_size; i++) { 168 | data_R = (uncompressed_fse_data_R_first[i] << 8) + uncompressed_huf_data_R_last[i]; 169 | 170 | if (data_R == 65535) { 171 | data_R += over_data_R[counter]; 172 | counter++; 173 | } 174 | 175 | data_R_signed[i] = Unsigned_To_Signed(data_R); 176 | } 177 | /////////////////// 178 | 179 | double coeffs_double_L[ORDER]; 180 | double coeffs_double_R[ORDER]; 181 | 182 | for (int i = 0; i < ORDER; i++) { 183 | coeffs_double_L[i] = coeffs_L[i]; 184 | coeffs_double_R[i] = coeffs_R[i]; 185 | } 186 | 187 | double predicted; 188 | for (int k = ORDER; k < block_size; k++) { 189 | predicted = 0.0; 190 | for (int j = 0; j < ORDER; j++) { 191 | predicted -= coeffs_double_L[j] * data_L_signed[k - 1 - j]; 192 | } 193 | 194 | data_L_signed[k] = data_L_signed[k] + (int)predicted; 195 | } 196 | 197 | for (int k = ORDER; k < block_size; k++) { 198 | predicted = 0.0; 199 | for (int j = 0; j < ORDER; j++) { 200 | predicted -= coeffs_double_R[j] * data_R_signed[k - 1 - j]; 201 | } 202 | 203 | data_R_signed[k] = data_R_signed[k] + (int)predicted; 204 | } 205 | 206 | TRANSFORM(data_L_signed, data_R_signed, block_size, OUT, OUT_COUNTER); 207 | } 208 | 209 | void DECODING() { 210 | READ_HALAC_HEADER(); 211 | WRITE_WAV_HEADER(); 212 | ////////////// 213 | u8 OUT[1024 * 1024]; 214 | u32 OUT_COUNTER = 0; 215 | 216 | u32 data_mod = INPUT_SIZE % BLOCK_SIZE; 217 | u32 block_count = INPUT_SIZE / BLOCK_SIZE - 1; 218 | for (int i = 0; i < block_count; i++) UNCOMPRESS(BLOCK_SIZE / (CHANNEL_COUNT * SAMPLE_BYTE), OUT, OUT_COUNTER); 219 | 220 | u32 last_block_size; 221 | if (block_count <= 0) last_block_size = INPUT_SIZE; 222 | else last_block_size = BLOCK_SIZE + data_mod; 223 | UNCOMPRESS(last_block_size / (CHANNEL_COUNT * SAMPLE_BYTE), OUT, OUT_COUNTER); 224 | 225 | if (OUT_COUNTER > 0) { 226 | fwrite(OUT, 1, OUT_COUNTER, OUTPUT_FILE); 227 | } 228 | } 229 | 230 | int main(int argc, char** argv) 231 | { 232 | /* 233 | ///////////// Reading Console Arguments ///////////// 234 | if (argc != 3) { 235 | printf(VERSION); 236 | printf(" Decoder\n" 237 | "Hakan Abbas (abbas.hakan@gmail.com)\n" 238 | "Only .halac types are supported for input!\n" 239 | "Please use this format: \"decoder.exe input_file output_file\"\n"); 240 | return 0; 241 | } 242 | INPUT_FILENAME = argv[1]; 243 | OUTPUT_FILENAME = argv[2]; 244 | DECODING(); 245 | ///////////////////////////////////////////////////// 246 | */ 247 | 248 | ////////////////////////////////////////// 249 | clock_t startTime = clock(); 250 | double time; 251 | 252 | string path; 253 | //path = "../WAV/ISKENDER_PAYDAS"; 254 | //path = "../WAV/test"; 255 | //path = "../WAV/TEST_AUDIO_SQUEEZE_CHART"; 256 | //path = "../WAV/HANDE_YENER"; 257 | //path = "../WAV/SIBEL_CAN"; 258 | //path = "../WAV/TEST_AUDIOS"; 259 | //path = "../WAV/YILDIZ_TILBE"; 260 | //path = "../WAV/single"; 261 | path = "D:/TEST_AUDIOS/YABANCI/Busta_Rhymes"; 262 | ////////////////////////////////////////// 263 | 264 | vector files; 265 | for (const auto& entry : fs::directory_iterator(path)) { 266 | if (!entry.is_directory() && (entry.path().extension() == ".halac")) 267 | files.emplace_back(entry.path().string()); 268 | } 269 | sort(files.begin(), files.end()); 270 | ////////////////////////////////////////// 271 | 272 | TOTAL_OUTPUT_SIZE = 0; 273 | for (u32 i = 0; i < files.size(); i++) { 274 | INPUT_FILENAME = files.at(i); 275 | OUTPUT_FILENAME = INPUT_FILENAME + ".org"; 276 | cout << OUTPUT_FILENAME << endl; 277 | 278 | OUTPUT_SIZE = 0; 279 | DECODING(); 280 | } 281 | 282 | ////////////////////////////////////////// 283 | time = static_cast(clock() - (double)startTime) / CLOCKS_PER_SEC; 284 | printf("-----------\n"); 285 | printf("Total Time: %2.3f seconds\n", time); 286 | ////////////////////////////////////////// 287 | 288 | return 0; 289 | } 290 | 291 | -------------------------------------------------------------------------------- /FSE/fse_decompress.c: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * FSE : Finite State Entropy decoder 3 | * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. 4 | * 5 | * You can contact the author at : 6 | * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy 7 | * - Public forum : https://groups.google.com/forum/#!forum/lz4c 8 | * 9 | * This source code is licensed under both the BSD-style license (found in the 10 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 11 | * in the COPYING file in the root directory of this source tree). 12 | * You may select, at your option, one of the above-listed licenses. 13 | ****************************************************************** */ 14 | 15 | 16 | /* ************************************************************** 17 | * Includes 18 | ****************************************************************/ 19 | #include /* malloc, free, qsort */ 20 | #include /* memcpy, memset */ 21 | #include "debug.h" /* assert */ 22 | #include "bitstream.h" 23 | #include "compiler.h" 24 | #define FSE_STATIC_LINKING_ONLY 25 | #include "fse.h" 26 | #include "error_private.h" 27 | 28 | 29 | /* ************************************************************** 30 | * Error Management 31 | ****************************************************************/ 32 | #define FSE_isError ERR_isError 33 | #define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ 34 | 35 | 36 | /* ************************************************************** 37 | * Templates 38 | ****************************************************************/ 39 | /* 40 | designed to be included 41 | for type-specific functions (template emulation in C) 42 | Objective is to write these functions only once, for improved maintenance 43 | */ 44 | 45 | /* safety checks */ 46 | #ifndef FSE_FUNCTION_EXTENSION 47 | # error "FSE_FUNCTION_EXTENSION must be defined" 48 | #endif 49 | #ifndef FSE_FUNCTION_TYPE 50 | # error "FSE_FUNCTION_TYPE must be defined" 51 | #endif 52 | 53 | /* Function names */ 54 | #define FSE_CAT(X,Y) X##Y 55 | #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) 56 | #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) 57 | 58 | 59 | /* Function templates */ 60 | FSE_DTable* FSE_createDTable (unsigned tableLog) 61 | { 62 | if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; 63 | return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); 64 | } 65 | 66 | void FSE_freeDTable (FSE_DTable* dt) 67 | { 68 | free(dt); 69 | } 70 | 71 | size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) 72 | { 73 | void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ 74 | FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); 75 | U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; 76 | 77 | U32 const maxSV1 = maxSymbolValue + 1; 78 | U32 const tableSize = 1 << tableLog; 79 | U32 highThreshold = tableSize-1; 80 | 81 | /* Sanity Checks */ 82 | if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); 83 | if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); 84 | 85 | /* Init, lay down lowprob symbols */ 86 | { FSE_DTableHeader DTableH; 87 | DTableH.tableLog = (U16)tableLog; 88 | DTableH.fastMode = 1; 89 | { S16 const largeLimit= (S16)(1 << (tableLog-1)); 90 | U32 s; 91 | for (s=0; s= largeLimit) DTableH.fastMode=0; 97 | symbolNext[s] = normalizedCounter[s]; 98 | } } } 99 | memcpy(dt, &DTableH, sizeof(DTableH)); 100 | } 101 | 102 | /* Spread symbols */ 103 | { U32 const tableMask = tableSize-1; 104 | U32 const step = FSE_TABLESTEP(tableSize); 105 | U32 s, position = 0; 106 | for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ 112 | } } 113 | if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ 114 | } 115 | 116 | /* Build Decoding table */ 117 | { U32 u; 118 | for (u=0; utableLog = 0; 142 | DTableH->fastMode = 0; 143 | 144 | cell->newState = 0; 145 | cell->symbol = symbolValue; 146 | cell->nbBits = 0; 147 | 148 | return 0; 149 | } 150 | 151 | 152 | size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) 153 | { 154 | void* ptr = dt; 155 | FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; 156 | void* dPtr = dt + 1; 157 | FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; 158 | const unsigned tableSize = 1 << nbBits; 159 | const unsigned tableMask = tableSize - 1; 160 | const unsigned maxSV1 = tableMask+1; 161 | unsigned s; 162 | 163 | /* Sanity checks */ 164 | if (nbBits < 1) return ERROR(GENERIC); /* min size */ 165 | 166 | /* Build Decoding Table */ 167 | DTableH->tableLog = (U16)nbBits; 168 | DTableH->fastMode = 1; 169 | for (s=0; s sizeof(bitD.bitContainer)*8) /* This test must be static */ 205 | BIT_reloadDStream(&bitD); 206 | 207 | op[1] = FSE_GETSYMBOL(&state2); 208 | 209 | if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ 210 | { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } 211 | 212 | op[2] = FSE_GETSYMBOL(&state1); 213 | 214 | if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ 215 | BIT_reloadDStream(&bitD); 216 | 217 | op[3] = FSE_GETSYMBOL(&state2); 218 | } 219 | 220 | /* tail */ 221 | /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ 222 | while (1) { 223 | if (op>(omax-2)) return ERROR(dstSize_tooSmall); 224 | *op++ = FSE_GETSYMBOL(&state1); 225 | if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { 226 | *op++ = FSE_GETSYMBOL(&state2); 227 | break; 228 | } 229 | 230 | if (op>(omax-2)) return ERROR(dstSize_tooSmall); 231 | *op++ = FSE_GETSYMBOL(&state2); 232 | if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { 233 | *op++ = FSE_GETSYMBOL(&state1); 234 | break; 235 | } } 236 | 237 | return op-ostart; 238 | } 239 | 240 | 241 | size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, 242 | const void* cSrc, size_t cSrcSize, 243 | const FSE_DTable* dt) 244 | { 245 | const void* ptr = dt; 246 | const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; 247 | const U32 fastMode = DTableH->fastMode; 248 | 249 | /* select fast mode (static) */ 250 | if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); 251 | return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); 252 | } 253 | 254 | 255 | size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog) 256 | { 257 | const BYTE* const istart = (const BYTE*)cSrc; 258 | const BYTE* ip = istart; 259 | short counting[FSE_MAX_SYMBOL_VALUE+1]; 260 | unsigned tableLog; 261 | unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; 262 | 263 | /* normal FSE decoding mode */ 264 | size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); 265 | if (FSE_isError(NCountLength)) return NCountLength; 266 | if (tableLog > maxLog) return ERROR(tableLog_tooLarge); 267 | assert(NCountLength <= cSrcSize); 268 | ip += NCountLength; 269 | cSrcSize -= NCountLength; 270 | 271 | CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) ); 272 | 273 | return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ 274 | } 275 | 276 | 277 | typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; 278 | 279 | size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) 280 | { 281 | DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ 282 | return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG); 283 | } 284 | 285 | 286 | 287 | #endif /* FSE_COMMONDEFS_ONLY */ 288 | -------------------------------------------------------------------------------- /FSE/fseU16.c: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | FSEU16 : Finite State Entropy coder for 16-bits input 3 | Copyright (C) 2013-2016, Yann Collet. 4 | 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 32 | - Public forum : https://groups.google.com/forum/#!forum/lz4c 33 | ****************************************************************** */ 34 | 35 | /* ************************************************************* 36 | * Tuning parameters 37 | *****************************************************************/ 38 | /* MEMORY_USAGE : 39 | * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) 40 | * Increasing memory usage improves compression ratio 41 | * Reduced memory usage can improve speed, due to cache effect 42 | * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ 43 | #ifndef FSEU16_MAX_MEMORY_USAGE 44 | # define FSEU16_MAX_MEMORY_USAGE 15 45 | #endif 46 | #ifndef FSEU16_DEFAULT_MEMORY_USAGE 47 | # define FSEU16_DEFAULT_MEMORY_USAGE 14 48 | #endif 49 | 50 | /* ************************************************************** 51 | * Includes 52 | *****************************************************************/ 53 | #include "fseU16.h" 54 | #define FSEU16_SYMBOLVALUE_ABSOLUTEMAX 4095 55 | #if (FSEU16_MAX_SYMBOL_VALUE > FSEU16_SYMBOLVALUE_ABSOLUTEMAX) 56 | # error "FSEU16_MAX_SYMBOL_VALUE is too large !" 57 | #endif 58 | 59 | /* ************************************************************** 60 | * Compiler specifics 61 | *****************************************************************/ 62 | #ifdef _MSC_VER /* Visual Studio */ 63 | # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ 64 | #endif 65 | 66 | #if defined(__GNUC__) 67 | # pragma GCC diagnostic ignored "-Wunused-function" 68 | #endif 69 | 70 | #if defined (__clang__) 71 | # pragma clang diagnostic ignored "-Wunused-function" 72 | #endif 73 | 74 | 75 | /* ************************************************************** 76 | * Local type 77 | ****************************************************************/ 78 | typedef struct { 79 | unsigned short newState; 80 | unsigned nbBits : 4; 81 | unsigned symbol : 12; 82 | } FSE_decode_tU16; /* Note : the size of this struct must be 4 */ 83 | 84 | 85 | /* ******************************************************************* 86 | * Include type-specific functions from fse.c (C template emulation) 87 | *********************************************************************/ 88 | #define FSE_COMMONDEFS_ONLY 89 | 90 | #ifdef FSE_MAX_MEMORY_USAGE 91 | # undef FSE_MAX_MEMORY_USAGE 92 | #endif 93 | #ifdef FSE_DEFAULT_MEMORY_USAGE 94 | # undef FSE_DEFAULT_MEMORY_USAGE 95 | #endif 96 | #define FSE_MAX_MEMORY_USAGE FSEU16_MAX_MEMORY_USAGE 97 | #define FSE_DEFAULT_MEMORY_USAGE FSEU16_DEFAULT_MEMORY_USAGE 98 | 99 | #define FSE_FUNCTION_TYPE U16 100 | #define FSE_FUNCTION_EXTENSION U16 101 | 102 | #define FSE_count_generic FSE_count_genericU16 103 | #define FSE_buildCTable FSE_buildCTableU16 104 | #define FSE_buildCTable_wksp FSE_buildCTable_wksp_U16 105 | 106 | #define FSE_DECODE_TYPE FSE_decode_tU16 107 | #define FSE_createDTable FSE_createDTableU16 108 | #define FSE_freeDTable FSE_freeDTableU16 109 | #define FSE_buildDTable FSE_buildDTableU16 110 | 111 | #include "fse_compress.c" /* FSE_countU16, FSE_buildCTableU16 */ 112 | #include "fse_decompress.c" /* FSE_buildDTableU16 */ 113 | 114 | 115 | /*! FSE_countU16() : 116 | This function counts U16 values stored in `src`, 117 | and push the histogram into `count`. 118 | @return : count of most common element 119 | *maxSymbolValuePtr : will be updated with value of highest symbol. 120 | */ 121 | size_t FSE_countU16(unsigned* count, unsigned* maxSymbolValuePtr, 122 | const U16* src, size_t srcSize) 123 | { 124 | const U16* ip16 = (const U16*)src; 125 | const U16* const end = src + srcSize; 126 | unsigned maxSymbolValue = *maxSymbolValuePtr; 127 | 128 | memset(count, 0, (maxSymbolValue+1)*sizeof(*count)); 129 | if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } 130 | 131 | while (ip16 maxSymbolValue) 133 | return ERROR(maxSymbolValue_tooSmall); 134 | count[*ip16++]++; 135 | } 136 | 137 | while (!count[maxSymbolValue]) maxSymbolValue--; 138 | *maxSymbolValuePtr = maxSymbolValue; 139 | 140 | { U32 s, max=0; 141 | for (s=0; s<=maxSymbolValue; s++) 142 | if (count[s] > max) max = count[s]; 143 | return (size_t)max; 144 | } 145 | } 146 | 147 | /* ******************************************************* 148 | * U16 Compression functions 149 | *********************************************************/ 150 | size_t FSE_compressU16_usingCTable (void* dst, size_t maxDstSize, 151 | const U16* src, size_t srcSize, 152 | const FSE_CTable* ct) 153 | { 154 | const U16* const istart = src; 155 | const U16* const iend = istart + srcSize; 156 | const U16* ip; 157 | 158 | BYTE* op = (BYTE*) dst; 159 | BIT_CStream_t bitC; 160 | FSE_CState_t CState; 161 | 162 | 163 | /* init */ 164 | BIT_initCStream(&bitC, op, maxDstSize); 165 | FSE_initCState(&CState, ct); 166 | 167 | ip=iend; 168 | 169 | /* join to even */ 170 | if (srcSize & 1) { 171 | FSE_encodeSymbol(&bitC, &CState, *--ip); 172 | BIT_flushBits(&bitC); 173 | } 174 | 175 | /* join to mod 4 */ 176 | if (srcSize & 2) { 177 | FSE_encodeSymbol(&bitC, &CState, *--ip); 178 | FSE_encodeSymbol(&bitC, &CState, *--ip); 179 | BIT_flushBits(&bitC); 180 | } 181 | 182 | /* 2 or 4 encoding per loop */ 183 | while (ip>istart) { 184 | FSE_encodeSymbol(&bitC, &CState, *--ip); 185 | 186 | if (sizeof(size_t)*8 < FSE_MAX_TABLELOG*2+7 ) /* This test must be static */ 187 | BIT_flushBits(&bitC); 188 | 189 | FSE_encodeSymbol(&bitC, &CState, *--ip); 190 | 191 | if (sizeof(size_t)*8 > FSE_MAX_TABLELOG*4+7 ) { /* This test must be static */ 192 | FSE_encodeSymbol(&bitC, &CState, *--ip); 193 | FSE_encodeSymbol(&bitC, &CState, *--ip); 194 | } 195 | BIT_flushBits(&bitC); 196 | } 197 | 198 | FSE_flushCState(&bitC, &CState); 199 | return BIT_closeCStream(&bitC); 200 | } 201 | 202 | 203 | size_t FSE_compressU16(void* dst, size_t maxDstSize, 204 | const unsigned short* src, size_t srcSize, 205 | unsigned maxSymbolValue, unsigned tableLog) 206 | { 207 | const U16* const istart = src; 208 | const U16* ip = istart; 209 | 210 | BYTE* const ostart = (BYTE*) dst; 211 | BYTE* const omax = ostart + maxDstSize; 212 | BYTE* op = ostart; 213 | 214 | U32 counting[FSE_MAX_SYMBOL_VALUE+1] = {0}; 215 | S16 norm[FSE_MAX_SYMBOL_VALUE+1]; 216 | 217 | /* checks */ 218 | if (srcSize <= 1) return srcSize; 219 | if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE; 220 | if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; 221 | if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); 222 | if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); 223 | 224 | /* Scan for stats */ 225 | { size_t const maxCount = FSE_countU16 (counting, &maxSymbolValue, ip, srcSize); 226 | if (FSE_isError(maxCount)) return maxCount; 227 | if (maxCount == srcSize) return 1; /* src contains one constant element x srcSize times. Use RLE compression. */ 228 | } 229 | /* Normalize */ 230 | tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); 231 | { size_t const errorCode = FSE_normalizeCount (norm, tableLog, counting, srcSize, maxSymbolValue); 232 | if (FSE_isError(errorCode)) return errorCode; 233 | } 234 | /* Write table description header */ 235 | { size_t const NSize = FSE_writeNCount (op, omax-op, norm, maxSymbolValue, tableLog); 236 | if (FSE_isError(NSize)) return NSize; 237 | op += NSize; 238 | } 239 | /* Compress */ 240 | { FSE_CTable CTable[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; 241 | size_t const errorCode = FSE_buildCTableU16 (CTable, norm, maxSymbolValue, tableLog); 242 | if (FSE_isError(errorCode)) return errorCode; 243 | op += FSE_compressU16_usingCTable (op, omax - op, ip, srcSize, CTable); 244 | } 245 | 246 | /* check compressibility */ 247 | if ( (size_t)(op-ostart) >= (size_t)(srcSize-1)*(sizeof(U16)) ) 248 | return 0; /* no compression */ 249 | 250 | return op-ostart; 251 | } 252 | 253 | 254 | /* ******************************************************* 255 | * U16 Decompression functions 256 | *********************************************************/ 257 | 258 | U16 FSE_decodeSymbolU16(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) 259 | { 260 | const FSE_decode_tU16 DInfo = ((const FSE_decode_tU16*)(DStatePtr->table))[DStatePtr->state]; 261 | U16 symbol; 262 | size_t lowBits; 263 | const U32 nbBits = DInfo.nbBits; 264 | 265 | symbol = (U16)(DInfo.symbol); 266 | lowBits = BIT_readBits(bitD, nbBits); 267 | DStatePtr->state = DInfo.newState + lowBits; 268 | 269 | return symbol; 270 | } 271 | 272 | 273 | size_t FSE_decompressU16_usingDTable (U16* dst, size_t maxDstSize, 274 | const void* cSrc, size_t cSrcSize, 275 | const FSE_DTable* dt) 276 | { 277 | U16* const ostart = dst; 278 | U16* op = ostart; 279 | U16* const oend = ostart + maxDstSize; 280 | BIT_DStream_t bitD; 281 | FSE_DState_t state; 282 | 283 | /* Init */ 284 | memset(&bitD, 0, sizeof(bitD)); 285 | BIT_initDStream(&bitD, cSrc, cSrcSize); 286 | FSE_initDState(&state, &bitD, dt); 287 | 288 | while((BIT_reloadDStream(&bitD) < BIT_DStream_completed) && (op /* size_t, ptrdiff_t */ 22 | #include /* memcpy */ 23 | 24 | 25 | /*-**************************************** 26 | * Compiler specifics 27 | ******************************************/ 28 | #if defined(_MSC_VER) /* Visual Studio */ 29 | # include /* _byteswap_ulong */ 30 | # include /* _byteswap_* */ 31 | #endif 32 | #if defined(__GNUC__) 33 | # define MEM_STATIC static __inline __attribute__((unused)) 34 | #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 35 | # define MEM_STATIC static inline 36 | #elif defined(_MSC_VER) 37 | # define MEM_STATIC static __inline 38 | #else 39 | # define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ 40 | #endif 41 | 42 | #ifndef __has_builtin 43 | # define __has_builtin(x) 0 /* compat. with non-clang compilers */ 44 | #endif 45 | 46 | /* code only tested on 32 and 64 bits systems */ 47 | #define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; } 48 | MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } 49 | 50 | /* detects whether we are being compiled under msan */ 51 | #if defined (__has_feature) 52 | # if __has_feature(memory_sanitizer) 53 | # define MEMORY_SANITIZER 1 54 | # endif 55 | #endif 56 | 57 | #if defined (MEMORY_SANITIZER) 58 | /* Not all platforms that support msan provide sanitizers/msan_interface.h. 59 | * We therefore declare the functions we need ourselves, rather than trying to 60 | * include the header file... */ 61 | 62 | #include /* intptr_t */ 63 | 64 | /* Make memory region fully initialized (without changing its contents). */ 65 | void __msan_unpoison(const volatile void *a, size_t size); 66 | 67 | /* Make memory region fully uninitialized (without changing its contents). 68 | This is a legacy interface that does not update origin information. Use 69 | __msan_allocated_memory() instead. */ 70 | void __msan_poison(const volatile void *a, size_t size); 71 | 72 | /* Returns the offset of the first (at least partially) poisoned byte in the 73 | memory range, or -1 if the whole range is good. */ 74 | intptr_t __msan_test_shadow(const volatile void *x, size_t size); 75 | #endif 76 | 77 | /* detects whether we are being compiled under asan */ 78 | #if defined (__has_feature) 79 | # if __has_feature(address_sanitizer) 80 | # define ADDRESS_SANITIZER 1 81 | # endif 82 | #elif defined(__SANITIZE_ADDRESS__) 83 | # define ADDRESS_SANITIZER 1 84 | #endif 85 | 86 | #if defined (ADDRESS_SANITIZER) 87 | /* Not all platforms that support asan provide sanitizers/asan_interface.h. 88 | * We therefore declare the functions we need ourselves, rather than trying to 89 | * include the header file... */ 90 | 91 | /** 92 | * Marks a memory region ([addr, addr+size)) as unaddressable. 93 | * 94 | * This memory must be previously allocated by your program. Instrumented 95 | * code is forbidden from accessing addresses in this region until it is 96 | * unpoisoned. This function is not guaranteed to poison the entire region - 97 | * it could poison only a subregion of [addr, addr+size) due to ASan 98 | * alignment restrictions. 99 | * 100 | * \note This function is not thread-safe because no two threads can poison or 101 | * unpoison memory in the same memory region simultaneously. 102 | * 103 | * \param addr Start of memory region. 104 | * \param size Size of memory region. */ 105 | void __asan_poison_memory_region(void const volatile *addr, size_t size); 106 | 107 | /** 108 | * Marks a memory region ([addr, addr+size)) as addressable. 109 | * 110 | * This memory must be previously allocated by your program. Accessing 111 | * addresses in this region is allowed until this region is poisoned again. 112 | * This function could unpoison a super-region of [addr, addr+size) due 113 | * to ASan alignment restrictions. 114 | * 115 | * \note This function is not thread-safe because no two threads can 116 | * poison or unpoison memory in the same memory region simultaneously. 117 | * 118 | * \param addr Start of memory region. 119 | * \param size Size of memory region. */ 120 | void __asan_unpoison_memory_region(void const volatile *addr, size_t size); 121 | #endif 122 | 123 | 124 | /*-************************************************************** 125 | * Basic Types 126 | *****************************************************************/ 127 | #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) 128 | # include 129 | typedef uint8_t BYTE; 130 | typedef uint16_t U16; 131 | typedef int16_t S16; 132 | typedef uint32_t U32; 133 | typedef int32_t S32; 134 | typedef uint64_t U64; 135 | typedef int64_t S64; 136 | #else 137 | # include 138 | #if CHAR_BIT != 8 139 | # error "this implementation requires char to be exactly 8-bit type" 140 | #endif 141 | typedef unsigned char BYTE; 142 | #if USHRT_MAX != 65535 143 | # error "this implementation requires short to be exactly 16-bit type" 144 | #endif 145 | typedef unsigned short U16; 146 | typedef signed short S16; 147 | #if UINT_MAX != 4294967295 148 | # error "this implementation requires int to be exactly 32-bit type" 149 | #endif 150 | typedef unsigned int U32; 151 | typedef signed int S32; 152 | /* note : there are no limits defined for long long type in C90. 153 | * limits exist in C99, however, in such case, is preferred */ 154 | typedef unsigned long long U64; 155 | typedef signed long long S64; 156 | #endif 157 | 158 | 159 | /*-************************************************************** 160 | * Memory I/O 161 | *****************************************************************/ 162 | /* MEM_FORCE_MEMORY_ACCESS : 163 | * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. 164 | * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. 165 | * The below switch allow to select different access method for improved performance. 166 | * Method 0 (default) : use `memcpy()`. Safe and portable. 167 | * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). 168 | * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. 169 | * Method 2 : direct access. This method is portable but violate C standard. 170 | * It can generate buggy code on targets depending on alignment. 171 | * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) 172 | * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. 173 | * Prefer these methods in priority order (0 > 1 > 2) 174 | */ 175 | #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ 176 | # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) 177 | # define MEM_FORCE_MEMORY_ACCESS 2 178 | # elif defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) 179 | # define MEM_FORCE_MEMORY_ACCESS 1 180 | # endif 181 | #endif 182 | 183 | MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } 184 | MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } 185 | 186 | MEM_STATIC unsigned MEM_isLittleEndian(void) 187 | { 188 | const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ 189 | return one.c[0]; 190 | } 191 | 192 | #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) 193 | 194 | /* violates C standard, by lying on structure alignment. 195 | Only use if no other choice to achieve best performance on target platform */ 196 | MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } 197 | MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } 198 | MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } 199 | MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } 200 | 201 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } 202 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } 203 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } 204 | 205 | #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) 206 | 207 | /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ 208 | /* currently only defined for gcc and icc */ 209 | #if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) 210 | __pragma( pack(push, 1) ) 211 | typedef struct { U16 v; } unalign16; 212 | typedef struct { U32 v; } unalign32; 213 | typedef struct { U64 v; } unalign64; 214 | typedef struct { size_t v; } unalignArch; 215 | __pragma( pack(pop) ) 216 | #else 217 | typedef struct { U16 v; } __attribute__((packed)) unalign16; 218 | typedef struct { U32 v; } __attribute__((packed)) unalign32; 219 | typedef struct { U64 v; } __attribute__((packed)) unalign64; 220 | typedef struct { size_t v; } __attribute__((packed)) unalignArch; 221 | #endif 222 | 223 | MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; } 224 | MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; } 225 | MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; } 226 | MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; } 227 | 228 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; } 229 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; } 230 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; } 231 | 232 | #else 233 | 234 | /* default method, safe and standard. 235 | can sometimes prove slower */ 236 | 237 | MEM_STATIC U16 MEM_read16(const void* memPtr) 238 | { 239 | U16 val; memcpy(&val, memPtr, sizeof(val)); return val; 240 | } 241 | 242 | MEM_STATIC U32 MEM_read32(const void* memPtr) 243 | { 244 | U32 val; memcpy(&val, memPtr, sizeof(val)); return val; 245 | } 246 | 247 | MEM_STATIC U64 MEM_read64(const void* memPtr) 248 | { 249 | U64 val; memcpy(&val, memPtr, sizeof(val)); return val; 250 | } 251 | 252 | MEM_STATIC size_t MEM_readST(const void* memPtr) 253 | { 254 | size_t val; memcpy(&val, memPtr, sizeof(val)); return val; 255 | } 256 | 257 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) 258 | { 259 | memcpy(memPtr, &value, sizeof(value)); 260 | } 261 | 262 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) 263 | { 264 | memcpy(memPtr, &value, sizeof(value)); 265 | } 266 | 267 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) 268 | { 269 | memcpy(memPtr, &value, sizeof(value)); 270 | } 271 | 272 | #endif /* MEM_FORCE_MEMORY_ACCESS */ 273 | 274 | MEM_STATIC U32 MEM_swap32(U32 in) 275 | { 276 | #if defined(_MSC_VER) /* Visual Studio */ 277 | return _byteswap_ulong(in); 278 | #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ 279 | || (defined(__clang__) && __has_builtin(__builtin_bswap32)) 280 | return __builtin_bswap32(in); 281 | #else 282 | return ((in << 24) & 0xff000000 ) | 283 | ((in << 8) & 0x00ff0000 ) | 284 | ((in >> 8) & 0x0000ff00 ) | 285 | ((in >> 24) & 0x000000ff ); 286 | #endif 287 | } 288 | 289 | MEM_STATIC U64 MEM_swap64(U64 in) 290 | { 291 | #if defined(_MSC_VER) /* Visual Studio */ 292 | return _byteswap_uint64(in); 293 | #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ 294 | || (defined(__clang__) && __has_builtin(__builtin_bswap64)) 295 | return __builtin_bswap64(in); 296 | #else 297 | return ((in << 56) & 0xff00000000000000ULL) | 298 | ((in << 40) & 0x00ff000000000000ULL) | 299 | ((in << 24) & 0x0000ff0000000000ULL) | 300 | ((in << 8) & 0x000000ff00000000ULL) | 301 | ((in >> 8) & 0x00000000ff000000ULL) | 302 | ((in >> 24) & 0x0000000000ff0000ULL) | 303 | ((in >> 40) & 0x000000000000ff00ULL) | 304 | ((in >> 56) & 0x00000000000000ffULL); 305 | #endif 306 | } 307 | 308 | MEM_STATIC size_t MEM_swapST(size_t in) 309 | { 310 | if (MEM_32bits()) 311 | return (size_t)MEM_swap32((U32)in); 312 | else 313 | return (size_t)MEM_swap64((U64)in); 314 | } 315 | 316 | /*=== Little endian r/w ===*/ 317 | 318 | MEM_STATIC U16 MEM_readLE16(const void* memPtr) 319 | { 320 | if (MEM_isLittleEndian()) 321 | return MEM_read16(memPtr); 322 | else { 323 | const BYTE* p = (const BYTE*)memPtr; 324 | return (U16)(p[0] + (p[1]<<8)); 325 | } 326 | } 327 | 328 | MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) 329 | { 330 | if (MEM_isLittleEndian()) { 331 | MEM_write16(memPtr, val); 332 | } else { 333 | BYTE* p = (BYTE*)memPtr; 334 | p[0] = (BYTE)val; 335 | p[1] = (BYTE)(val>>8); 336 | } 337 | } 338 | 339 | MEM_STATIC U32 MEM_readLE24(const void* memPtr) 340 | { 341 | return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); 342 | } 343 | 344 | MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) 345 | { 346 | MEM_writeLE16(memPtr, (U16)val); 347 | ((BYTE*)memPtr)[2] = (BYTE)(val>>16); 348 | } 349 | 350 | MEM_STATIC U32 MEM_readLE32(const void* memPtr) 351 | { 352 | if (MEM_isLittleEndian()) 353 | return MEM_read32(memPtr); 354 | else 355 | return MEM_swap32(MEM_read32(memPtr)); 356 | } 357 | 358 | MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) 359 | { 360 | if (MEM_isLittleEndian()) 361 | MEM_write32(memPtr, val32); 362 | else 363 | MEM_write32(memPtr, MEM_swap32(val32)); 364 | } 365 | 366 | MEM_STATIC U64 MEM_readLE64(const void* memPtr) 367 | { 368 | if (MEM_isLittleEndian()) 369 | return MEM_read64(memPtr); 370 | else 371 | return MEM_swap64(MEM_read64(memPtr)); 372 | } 373 | 374 | MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) 375 | { 376 | if (MEM_isLittleEndian()) 377 | MEM_write64(memPtr, val64); 378 | else 379 | MEM_write64(memPtr, MEM_swap64(val64)); 380 | } 381 | 382 | MEM_STATIC size_t MEM_readLEST(const void* memPtr) 383 | { 384 | if (MEM_32bits()) 385 | return (size_t)MEM_readLE32(memPtr); 386 | else 387 | return (size_t)MEM_readLE64(memPtr); 388 | } 389 | 390 | MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) 391 | { 392 | if (MEM_32bits()) 393 | MEM_writeLE32(memPtr, (U32)val); 394 | else 395 | MEM_writeLE64(memPtr, (U64)val); 396 | } 397 | 398 | /*=== Big endian r/w ===*/ 399 | 400 | MEM_STATIC U32 MEM_readBE32(const void* memPtr) 401 | { 402 | if (MEM_isLittleEndian()) 403 | return MEM_swap32(MEM_read32(memPtr)); 404 | else 405 | return MEM_read32(memPtr); 406 | } 407 | 408 | MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) 409 | { 410 | if (MEM_isLittleEndian()) 411 | MEM_write32(memPtr, MEM_swap32(val32)); 412 | else 413 | MEM_write32(memPtr, val32); 414 | } 415 | 416 | MEM_STATIC U64 MEM_readBE64(const void* memPtr) 417 | { 418 | if (MEM_isLittleEndian()) 419 | return MEM_swap64(MEM_read64(memPtr)); 420 | else 421 | return MEM_read64(memPtr); 422 | } 423 | 424 | MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) 425 | { 426 | if (MEM_isLittleEndian()) 427 | MEM_write64(memPtr, MEM_swap64(val64)); 428 | else 429 | MEM_write64(memPtr, val64); 430 | } 431 | 432 | MEM_STATIC size_t MEM_readBEST(const void* memPtr) 433 | { 434 | if (MEM_32bits()) 435 | return (size_t)MEM_readBE32(memPtr); 436 | else 437 | return (size_t)MEM_readBE64(memPtr); 438 | } 439 | 440 | MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) 441 | { 442 | if (MEM_32bits()) 443 | MEM_writeBE32(memPtr, (U32)val); 444 | else 445 | MEM_writeBE64(memPtr, (U64)val); 446 | } 447 | 448 | 449 | #if defined (__cplusplus) 450 | } 451 | #endif 452 | 453 | #endif /* MEM_H_MODULE */ 454 | -------------------------------------------------------------------------------- /FSE/bitstream.h: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * bitstream 3 | * Part of FSE library 4 | * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. 5 | * 6 | * You can contact the author at : 7 | * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 8 | * 9 | * This source code is licensed under both the BSD-style license (found in the 10 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 11 | * in the COPYING file in the root directory of this source tree). 12 | * You may select, at your option, one of the above-listed licenses. 13 | ****************************************************************** */ 14 | #ifndef BITSTREAM_H_MODULE 15 | #define BITSTREAM_H_MODULE 16 | 17 | #if defined (__cplusplus) 18 | extern "C" { 19 | #endif 20 | 21 | /* 22 | * This API consists of small unitary functions, which must be inlined for best performance. 23 | * Since link-time-optimization is not available for all compilers, 24 | * these functions are defined into a .h to be included. 25 | */ 26 | 27 | /*-**************************************** 28 | * Dependencies 29 | ******************************************/ 30 | #include "mem.h" /* unaligned access routines */ 31 | #include "compiler.h" /* UNLIKELY() */ 32 | #include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */ 33 | #include "error_private.h" /* error codes and messages */ 34 | 35 | 36 | /*========================================= 37 | * Target specific 38 | =========================================*/ 39 | #if defined(__BMI__) && defined(__GNUC__) 40 | # include /* support for bextr (experimental) */ 41 | #elif defined(__ICCARM__) 42 | # include 43 | #endif 44 | 45 | #define STREAM_ACCUMULATOR_MIN_32 25 46 | #define STREAM_ACCUMULATOR_MIN_64 57 47 | #define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) 48 | 49 | 50 | /*-****************************************** 51 | * bitStream encoding API (write forward) 52 | ********************************************/ 53 | /* bitStream can mix input from multiple sources. 54 | * A critical property of these streams is that they encode and decode in **reverse** direction. 55 | * So the first bit sequence you add will be the last to be read, like a LIFO stack. 56 | */ 57 | typedef struct { 58 | size_t bitContainer; 59 | unsigned bitPos; 60 | char* startPtr; 61 | char* ptr; 62 | char* endPtr; 63 | } BIT_CStream_t; 64 | 65 | MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); 66 | MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); 67 | MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); 68 | MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); 69 | 70 | /* Start with initCStream, providing the size of buffer to write into. 71 | * bitStream will never write outside of this buffer. 72 | * `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. 73 | * 74 | * bits are first added to a local register. 75 | * Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. 76 | * Writing data into memory is an explicit operation, performed by the flushBits function. 77 | * Hence keep track how many bits are potentially stored into local register to avoid register overflow. 78 | * After a flushBits, a maximum of 7 bits might still be stored into local register. 79 | * 80 | * Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. 81 | * 82 | * Last operation is to close the bitStream. 83 | * The function returns the final size of CStream in bytes. 84 | * If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) 85 | */ 86 | 87 | 88 | /*-******************************************** 89 | * bitStream decoding API (read backward) 90 | **********************************************/ 91 | typedef struct { 92 | size_t bitContainer; 93 | unsigned bitsConsumed; 94 | const char* ptr; 95 | const char* start; 96 | const char* limitPtr; 97 | } BIT_DStream_t; 98 | 99 | typedef enum { BIT_DStream_unfinished = 0, 100 | BIT_DStream_endOfBuffer = 1, 101 | BIT_DStream_completed = 2, 102 | BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ 103 | /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ 104 | 105 | MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); 106 | MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); 107 | MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); 108 | MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); 109 | 110 | 111 | /* Start by invoking BIT_initDStream(). 112 | * A chunk of the bitStream is then stored into a local register. 113 | * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). 114 | * You can then retrieve bitFields stored into the local register, **in reverse order**. 115 | * Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. 116 | * A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. 117 | * Otherwise, it can be less than that, so proceed accordingly. 118 | * Checking if DStream has reached its end can be performed with BIT_endOfDStream(). 119 | */ 120 | 121 | 122 | /*-**************************************** 123 | * unsafe API 124 | ******************************************/ 125 | MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); 126 | /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ 127 | 128 | MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); 129 | /* unsafe version; does not check buffer overflow */ 130 | 131 | MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); 132 | /* faster, but works only if nbBits >= 1 */ 133 | 134 | 135 | 136 | /*-************************************************************** 137 | * Internal functions 138 | ****************************************************************/ 139 | MEM_STATIC unsigned BIT_highbit32 (U32 val) 140 | { 141 | assert(val != 0); 142 | { 143 | # if defined(_MSC_VER) /* Visual */ 144 | unsigned long r=0; 145 | return _BitScanReverse ( &r, val ) ? (unsigned)r : 0; 146 | # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ 147 | return __builtin_clz (val) ^ 31; 148 | # elif defined(__ICCARM__) /* IAR Intrinsic */ 149 | return 31 - __CLZ(val); 150 | # else /* Software version */ 151 | static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 152 | 11, 14, 16, 18, 22, 25, 3, 30, 153 | 8, 12, 20, 28, 15, 17, 24, 7, 154 | 19, 27, 23, 6, 26, 5, 4, 31 }; 155 | U32 v = val; 156 | v |= v >> 1; 157 | v |= v >> 2; 158 | v |= v >> 4; 159 | v |= v >> 8; 160 | v |= v >> 16; 161 | return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; 162 | # endif 163 | } 164 | } 165 | 166 | /*===== Local Constants =====*/ 167 | static const unsigned BIT_mask[] = { 168 | 0, 1, 3, 7, 0xF, 0x1F, 169 | 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 170 | 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 171 | 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 172 | 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF, 173 | 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */ 174 | #define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0])) 175 | 176 | /*-************************************************************** 177 | * bitStream encoding 178 | ****************************************************************/ 179 | /*! BIT_initCStream() : 180 | * `dstCapacity` must be > sizeof(size_t) 181 | * @return : 0 if success, 182 | * otherwise an error code (can be tested using ERR_isError()) */ 183 | MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, 184 | void* startPtr, size_t dstCapacity) 185 | { 186 | bitC->bitContainer = 0; 187 | bitC->bitPos = 0; 188 | bitC->startPtr = (char*)startPtr; 189 | bitC->ptr = bitC->startPtr; 190 | bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); 191 | if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall); 192 | return 0; 193 | } 194 | 195 | /*! BIT_addBits() : 196 | * can add up to 31 bits into `bitC`. 197 | * Note : does not check for register overflow ! */ 198 | MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, 199 | size_t value, unsigned nbBits) 200 | { 201 | MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32); 202 | assert(nbBits < BIT_MASK_SIZE); 203 | assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); 204 | bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; 205 | bitC->bitPos += nbBits; 206 | } 207 | 208 | /*! BIT_addBitsFast() : 209 | * works only if `value` is _clean_, 210 | * meaning all high bits above nbBits are 0 */ 211 | MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, 212 | size_t value, unsigned nbBits) 213 | { 214 | assert((value>>nbBits) == 0); 215 | assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); 216 | bitC->bitContainer |= value << bitC->bitPos; 217 | bitC->bitPos += nbBits; 218 | } 219 | 220 | /*! BIT_flushBitsFast() : 221 | * assumption : bitContainer has not overflowed 222 | * unsafe version; does not check buffer overflow */ 223 | MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) 224 | { 225 | size_t const nbBytes = bitC->bitPos >> 3; 226 | assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); 227 | assert(bitC->ptr <= bitC->endPtr); 228 | MEM_writeLEST(bitC->ptr, bitC->bitContainer); 229 | bitC->ptr += nbBytes; 230 | bitC->bitPos &= 7; 231 | bitC->bitContainer >>= nbBytes*8; 232 | } 233 | 234 | /*! BIT_flushBits() : 235 | * assumption : bitContainer has not overflowed 236 | * safe version; check for buffer overflow, and prevents it. 237 | * note : does not signal buffer overflow. 238 | * overflow will be revealed later on using BIT_closeCStream() */ 239 | MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) 240 | { 241 | size_t const nbBytes = bitC->bitPos >> 3; 242 | assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); 243 | assert(bitC->ptr <= bitC->endPtr); 244 | MEM_writeLEST(bitC->ptr, bitC->bitContainer); 245 | bitC->ptr += nbBytes; 246 | if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; 247 | bitC->bitPos &= 7; 248 | bitC->bitContainer >>= nbBytes*8; 249 | } 250 | 251 | /*! BIT_closeCStream() : 252 | * @return : size of CStream, in bytes, 253 | * or 0 if it could not fit into dstBuffer */ 254 | MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) 255 | { 256 | BIT_addBitsFast(bitC, 1, 1); /* endMark */ 257 | BIT_flushBits(bitC); 258 | if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ 259 | return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); 260 | } 261 | 262 | 263 | /*-******************************************************** 264 | * bitStream decoding 265 | **********************************************************/ 266 | /*! BIT_initDStream() : 267 | * Initialize a BIT_DStream_t. 268 | * `bitD` : a pointer to an already allocated BIT_DStream_t structure. 269 | * `srcSize` must be the *exact* size of the bitStream, in bytes. 270 | * @return : size of stream (== srcSize), or an errorCode if a problem is detected 271 | */ 272 | MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) 273 | { 274 | if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } 275 | 276 | bitD->start = (const char*)srcBuffer; 277 | bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); 278 | 279 | if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ 280 | bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); 281 | bitD->bitContainer = MEM_readLEST(bitD->ptr); 282 | { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; 283 | bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ 284 | if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } 285 | } else { 286 | bitD->ptr = bitD->start; 287 | bitD->bitContainer = *(const BYTE*)(bitD->start); 288 | switch(srcSize) 289 | { 290 | case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); 291 | /* fall-through */ 292 | 293 | case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); 294 | /* fall-through */ 295 | 296 | case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); 297 | /* fall-through */ 298 | 299 | case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; 300 | /* fall-through */ 301 | 302 | case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; 303 | /* fall-through */ 304 | 305 | case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; 306 | /* fall-through */ 307 | 308 | default: break; 309 | } 310 | { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; 311 | bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; 312 | if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ 313 | } 314 | bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; 315 | } 316 | 317 | return srcSize; 318 | } 319 | 320 | MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) 321 | { 322 | return bitContainer >> start; 323 | } 324 | 325 | MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) 326 | { 327 | U32 const regMask = sizeof(bitContainer)*8 - 1; 328 | /* if start > regMask, bitstream is corrupted, and result is undefined */ 329 | assert(nbBits < BIT_MASK_SIZE); 330 | return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; 331 | } 332 | 333 | MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) 334 | { 335 | assert(nbBits < BIT_MASK_SIZE); 336 | return bitContainer & BIT_mask[nbBits]; 337 | } 338 | 339 | /*! BIT_lookBits() : 340 | * Provides next n bits from local register. 341 | * local register is not modified. 342 | * On 32-bits, maxNbBits==24. 343 | * On 64-bits, maxNbBits==56. 344 | * @return : value extracted */ 345 | MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) 346 | { 347 | /* arbitrate between double-shift and shift+mask */ 348 | #if 1 349 | /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8, 350 | * bitstream is likely corrupted, and result is undefined */ 351 | return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); 352 | #else 353 | /* this code path is slower on my os-x laptop */ 354 | U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; 355 | return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask); 356 | #endif 357 | } 358 | 359 | /*! BIT_lookBitsFast() : 360 | * unsafe version; only works if nbBits >= 1 */ 361 | MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) 362 | { 363 | U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; 364 | assert(nbBits >= 1); 365 | return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); 366 | } 367 | 368 | MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) 369 | { 370 | bitD->bitsConsumed += nbBits; 371 | } 372 | 373 | /*! BIT_readBits() : 374 | * Read (consume) next n bits from local register and update. 375 | * Pay attention to not read more than nbBits contained into local register. 376 | * @return : extracted value. */ 377 | MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) 378 | { 379 | size_t const value = BIT_lookBits(bitD, nbBits); 380 | BIT_skipBits(bitD, nbBits); 381 | return value; 382 | } 383 | 384 | /*! BIT_readBitsFast() : 385 | * unsafe version; only works only if nbBits >= 1 */ 386 | MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) 387 | { 388 | size_t const value = BIT_lookBitsFast(bitD, nbBits); 389 | assert(nbBits >= 1); 390 | BIT_skipBits(bitD, nbBits); 391 | return value; 392 | } 393 | 394 | /*! BIT_reloadDStreamFast() : 395 | * Similar to BIT_reloadDStream(), but with two differences: 396 | * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! 397 | * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this 398 | * point you must use BIT_reloadDStream() to reload. 399 | */ 400 | MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) 401 | { 402 | if (UNLIKELY(bitD->ptr < bitD->limitPtr)) 403 | return BIT_DStream_overflow; 404 | assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); 405 | bitD->ptr -= bitD->bitsConsumed >> 3; 406 | bitD->bitsConsumed &= 7; 407 | bitD->bitContainer = MEM_readLEST(bitD->ptr); 408 | return BIT_DStream_unfinished; 409 | } 410 | 411 | /*! BIT_reloadDStream() : 412 | * Refill `bitD` from buffer previously set in BIT_initDStream() . 413 | * This function is safe, it guarantees it will not read beyond src buffer. 414 | * @return : status of `BIT_DStream_t` internal register. 415 | * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ 416 | MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) 417 | { 418 | if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ 419 | return BIT_DStream_overflow; 420 | 421 | if (bitD->ptr >= bitD->limitPtr) { 422 | return BIT_reloadDStreamFast(bitD); 423 | } 424 | if (bitD->ptr == bitD->start) { 425 | if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; 426 | return BIT_DStream_completed; 427 | } 428 | /* start < ptr < limitPtr */ 429 | { U32 nbBytes = bitD->bitsConsumed >> 3; 430 | BIT_DStream_status result = BIT_DStream_unfinished; 431 | if (bitD->ptr - nbBytes < bitD->start) { 432 | nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ 433 | result = BIT_DStream_endOfBuffer; 434 | } 435 | bitD->ptr -= nbBytes; 436 | bitD->bitsConsumed -= nbBytes*8; 437 | bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ 438 | return result; 439 | } 440 | } 441 | 442 | /*! BIT_endOfDStream() : 443 | * @return : 1 if DStream has _exactly_ reached its end (all bits consumed). 444 | */ 445 | MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) 446 | { 447 | return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); 448 | } 449 | 450 | #if defined (__cplusplus) 451 | } 452 | #endif 453 | 454 | #endif /* BITSTREAM_H_MODULE */ 455 | -------------------------------------------------------------------------------- /FSE/huf.h: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * huff0 huffman codec, 3 | * part of Finite State Entropy library 4 | * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. 5 | * 6 | * You can contact the author at : 7 | * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 8 | * 9 | * This source code is licensed under both the BSD-style license (found in the 10 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 11 | * in the COPYING file in the root directory of this source tree). 12 | * You may select, at your option, one of the above-listed licenses. 13 | ****************************************************************** */ 14 | 15 | #if defined (__cplusplus) 16 | extern "C" { 17 | #endif 18 | 19 | #ifndef HUF_H_298734234 20 | #define HUF_H_298734234 21 | 22 | /* *** Dependencies *** */ 23 | #include /* size_t */ 24 | 25 | 26 | /* *** library symbols visibility *** */ 27 | /* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual, 28 | * HUF symbols remain "private" (internal symbols for library only). 29 | * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */ 30 | #if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) 31 | # define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) 32 | #elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ 33 | # define HUF_PUBLIC_API __declspec(dllexport) 34 | #elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) 35 | # define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */ 36 | #else 37 | # define HUF_PUBLIC_API 38 | #endif 39 | 40 | 41 | /* ========================== */ 42 | /* *** simple functions *** */ 43 | /* ========================== */ 44 | 45 | /** HUF_compress() : 46 | * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. 47 | * 'dst' buffer must be already allocated. 48 | * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). 49 | * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. 50 | * @return : size of compressed data (<= `dstCapacity`). 51 | * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! 52 | * if HUF_isError(return), compression failed (more details using HUF_getErrorName()) 53 | */ 54 | HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, 55 | const void* src, size_t srcSize); 56 | 57 | /** HUF_decompress() : 58 | * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', 59 | * into already allocated buffer 'dst', of minimum size 'dstSize'. 60 | * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. 61 | * Note : in contrast with FSE, HUF_decompress can regenerate 62 | * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, 63 | * because it knows size to regenerate (originalSize). 64 | * @return : size of regenerated data (== originalSize), 65 | * or an error code, which can be tested using HUF_isError() 66 | */ 67 | HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, 68 | const void* cSrc, size_t cSrcSize); 69 | 70 | 71 | /* *** Tool functions *** */ 72 | #define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ 73 | HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ 74 | 75 | /* Error Management */ 76 | HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ 77 | HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ 78 | 79 | 80 | /* *** Advanced function *** */ 81 | 82 | /** HUF_compress2() : 83 | * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`. 84 | * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX . 85 | * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ 86 | HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, 87 | const void* src, size_t srcSize, 88 | unsigned maxSymbolValue, unsigned tableLog); 89 | 90 | /** HUF_compress4X_wksp() : 91 | * Same as HUF_compress2(), but uses externally allocated `workSpace`. 92 | * `workspace` must have minimum alignment of 4, and be at least as large as HUF_WORKSPACE_SIZE */ 93 | #define HUF_WORKSPACE_SIZE ((6 << 10) + 256) 94 | #define HUF_WORKSPACE_SIZE_U32 (HUF_WORKSPACE_SIZE / sizeof(U32)) 95 | HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, 96 | const void* src, size_t srcSize, 97 | unsigned maxSymbolValue, unsigned tableLog, 98 | void* workSpace, size_t wkspSize); 99 | 100 | #endif /* HUF_H_298734234 */ 101 | 102 | /* ****************************************************************** 103 | * WARNING !! 104 | * The following section contains advanced and experimental definitions 105 | * which shall never be used in the context of a dynamic library, 106 | * because they are not guaranteed to remain stable in the future. 107 | * Only consider them in association with static linking. 108 | * *****************************************************************/ 109 | #if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY) 110 | #define HUF_H_HUF_STATIC_LINKING_ONLY 111 | 112 | /* *** Dependencies *** */ 113 | #include "mem.h" /* U32 */ 114 | 115 | 116 | /* *** Constants *** */ 117 | #define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ 118 | #define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ 119 | #define HUF_SYMBOLVALUE_MAX 255 120 | 121 | #define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ 122 | #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) 123 | # error "HUF_TABLELOG_MAX is too large !" 124 | #endif 125 | 126 | 127 | /* **************************************** 128 | * Static allocation 129 | ******************************************/ 130 | /* HUF buffer bounds */ 131 | #define HUF_CTABLEBOUND 129 132 | #define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ 133 | #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ 134 | 135 | /* static allocation of HUF's Compression Table */ 136 | #define HUF_CTABLE_SIZE_U32(maxSymbolValue) ((maxSymbolValue)+1) /* Use tables of U32, for proper alignment */ 137 | #define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_U32(maxSymbolValue) * sizeof(U32)) 138 | #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ 139 | U32 name##hb[HUF_CTABLE_SIZE_U32(maxSymbolValue)]; \ 140 | void* name##hv = &(name##hb); \ 141 | HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ 142 | 143 | /* static allocation of HUF's DTable */ 144 | typedef U32 HUF_DTable; 145 | #define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) 146 | #define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \ 147 | HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } 148 | #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ 149 | HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } 150 | 151 | 152 | /* **************************************** 153 | * Advanced decompression functions 154 | ******************************************/ 155 | size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ 156 | #ifndef HUF_FORCE_DECOMPRESS_X1 157 | size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ 158 | #endif 159 | 160 | size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ 161 | size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ 162 | size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ 163 | size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ 164 | size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ 165 | #ifndef HUF_FORCE_DECOMPRESS_X1 166 | size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ 167 | size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ 168 | #endif 169 | 170 | 171 | /* **************************************** 172 | * HUF detailed API 173 | * ****************************************/ 174 | 175 | /*! HUF_compress() does the following: 176 | * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") 177 | * 2. (optional) refine tableLog using HUF_optimalTableLog() 178 | * 3. build Huffman table from count using HUF_buildCTable() 179 | * 4. save Huffman table to memory buffer using HUF_writeCTable() 180 | * 5. encode the data stream using HUF_compress4X_usingCTable() 181 | * 182 | * The following API allows targeting specific sub-functions for advanced tasks. 183 | * For example, it's possible to compress several blocks using the same 'CTable', 184 | * or to save and regenerate 'CTable' using external methods. 185 | */ 186 | unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); 187 | typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ 188 | size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ 189 | size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); 190 | size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); 191 | size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); 192 | int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); 193 | 194 | typedef enum { 195 | HUF_repeat_none, /**< Cannot use the previous table */ 196 | HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ 197 | HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ 198 | } HUF_repeat; 199 | /** HUF_compress4X_repeat() : 200 | * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. 201 | * If it uses hufTable it does not modify hufTable or repeat. 202 | * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. 203 | * If preferRepeat then the old table will always be used if valid. */ 204 | size_t HUF_compress4X_repeat(void* dst, size_t dstSize, 205 | const void* src, size_t srcSize, 206 | unsigned maxSymbolValue, unsigned tableLog, 207 | void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ 208 | HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); 209 | 210 | /** HUF_buildCTable_wksp() : 211 | * Same as HUF_buildCTable(), but using externally allocated scratch buffer. 212 | * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. 213 | */ 214 | #define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1) 215 | #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) 216 | size_t HUF_buildCTable_wksp (HUF_CElt* tree, 217 | const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, 218 | void* workSpace, size_t wkspSize); 219 | 220 | /*! HUF_readStats() : 221 | * Read compact Huffman tree, saved by HUF_writeCTable(). 222 | * `huffWeight` is destination buffer. 223 | * @return : size read from `src` , or an error Code . 224 | * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ 225 | size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, 226 | U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, 227 | const void* src, size_t srcSize); 228 | 229 | /** HUF_readCTable() : 230 | * Loading a CTable saved with HUF_writeCTable() */ 231 | size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); 232 | 233 | /** HUF_getNbBits() : 234 | * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX 235 | * Note 1 : is not inlined, as HUF_CElt definition is private 236 | * Note 2 : const void* used, so that it can provide a statically allocated table as argument (which uses type U32) */ 237 | U32 HUF_getNbBits(const void* symbolTable, U32 symbolValue); 238 | 239 | /* 240 | * HUF_decompress() does the following: 241 | * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics 242 | * 2. build Huffman table from save, using HUF_readDTableX?() 243 | * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable() 244 | */ 245 | 246 | /** HUF_selectDecoder() : 247 | * Tells which decoder is likely to decode faster, 248 | * based on a set of pre-computed metrics. 249 | * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . 250 | * Assumption : 0 < dstSize <= 128 KB */ 251 | U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); 252 | 253 | /** 254 | * The minimum workspace size for the `workSpace` used in 255 | * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp(). 256 | * 257 | * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when 258 | * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15. 259 | * Buffer overflow errors may potentially occur if code modifications result in 260 | * a required workspace size greater than that specified in the following 261 | * macro. 262 | */ 263 | #define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10) 264 | #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) 265 | 266 | #ifndef HUF_FORCE_DECOMPRESS_X2 267 | size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize); 268 | size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); 269 | #endif 270 | #ifndef HUF_FORCE_DECOMPRESS_X1 271 | size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); 272 | size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); 273 | #endif 274 | 275 | size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); 276 | #ifndef HUF_FORCE_DECOMPRESS_X2 277 | size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); 278 | #endif 279 | #ifndef HUF_FORCE_DECOMPRESS_X1 280 | size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); 281 | #endif 282 | 283 | 284 | /* ====================== */ 285 | /* single stream variants */ 286 | /* ====================== */ 287 | 288 | size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); 289 | size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */ 290 | size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); 291 | /** HUF_compress1X_repeat() : 292 | * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. 293 | * If it uses hufTable it does not modify hufTable or repeat. 294 | * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. 295 | * If preferRepeat then the old table will always be used if valid. */ 296 | size_t HUF_compress1X_repeat(void* dst, size_t dstSize, 297 | const void* src, size_t srcSize, 298 | unsigned maxSymbolValue, unsigned tableLog, 299 | void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ 300 | HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); 301 | 302 | size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ 303 | #ifndef HUF_FORCE_DECOMPRESS_X1 304 | size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ 305 | #endif 306 | 307 | size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); 308 | size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); 309 | #ifndef HUF_FORCE_DECOMPRESS_X2 310 | size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ 311 | size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ 312 | #endif 313 | #ifndef HUF_FORCE_DECOMPRESS_X1 314 | size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ 315 | size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ 316 | #endif 317 | 318 | size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ 319 | #ifndef HUF_FORCE_DECOMPRESS_X2 320 | size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); 321 | #endif 322 | #ifndef HUF_FORCE_DECOMPRESS_X1 323 | size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); 324 | #endif 325 | 326 | /* BMI2 variants. 327 | * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. 328 | */ 329 | size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); 330 | #ifndef HUF_FORCE_DECOMPRESS_X2 331 | size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); 332 | #endif 333 | size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); 334 | size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); 335 | 336 | #endif /* HUF_STATIC_LINKING_ONLY */ 337 | 338 | #if defined (__cplusplus) 339 | } 340 | #endif 341 | -------------------------------------------------------------------------------- /FSE/fse_compress.c: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * FSE : Finite State Entropy encoder 3 | * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. 4 | * 5 | * You can contact the author at : 6 | * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy 7 | * - Public forum : https://groups.google.com/forum/#!forum/lz4c 8 | * 9 | * This source code is licensed under both the BSD-style license (found in the 10 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 11 | * in the COPYING file in the root directory of this source tree). 12 | * You may select, at your option, one of the above-listed licenses. 13 | ****************************************************************** */ 14 | 15 | /* ************************************************************** 16 | * Includes 17 | ****************************************************************/ 18 | #include /* malloc, free, qsort */ 19 | #include /* memcpy, memset */ 20 | #include "compiler.h" 21 | #include "mem.h" /* U32, U16, etc. */ 22 | #include "debug.h" /* assert, DEBUGLOG */ 23 | #include "hist.h" /* HIST_count_wksp */ 24 | #include "bitstream.h" 25 | #define FSE_STATIC_LINKING_ONLY 26 | #include "fse.h" 27 | #include "error_private.h" 28 | 29 | 30 | /* ************************************************************** 31 | * Error Management 32 | ****************************************************************/ 33 | #define FSE_isError ERR_isError 34 | 35 | 36 | /* ************************************************************** 37 | * Templates 38 | ****************************************************************/ 39 | /* 40 | designed to be included 41 | for type-specific functions (template emulation in C) 42 | Objective is to write these functions only once, for improved maintenance 43 | */ 44 | 45 | /* safety checks */ 46 | #ifndef FSE_FUNCTION_EXTENSION 47 | # error "FSE_FUNCTION_EXTENSION must be defined" 48 | #endif 49 | #ifndef FSE_FUNCTION_TYPE 50 | # error "FSE_FUNCTION_TYPE must be defined" 51 | #endif 52 | 53 | /* Function names */ 54 | #define FSE_CAT(X,Y) X##Y 55 | #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) 56 | #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) 57 | 58 | 59 | /* Function templates */ 60 | 61 | /* FSE_buildCTable_wksp() : 62 | * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). 63 | * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; 75 | FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); 76 | U32 const step = FSE_TABLESTEP(tableSize); 77 | U32 cumul[FSE_MAX_SYMBOL_VALUE+2]; 78 | 79 | FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace; 80 | U32 highThreshold = tableSize-1; 81 | 82 | /* CTable header */ 83 | if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge); 84 | tableU16[-2] = (U16) tableLog; 85 | tableU16[-1] = (U16) maxSymbolValue; 86 | assert(tableLog < 16); /* required for threshold strategy to work */ 87 | 88 | /* For explanations on how to distribute symbol values over the table : 89 | * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ 90 | 91 | #ifdef __clang_analyzer__ 92 | memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ 93 | #endif 94 | 95 | /* symbol start positions */ 96 | { U32 u; 97 | cumul[0] = 0; 98 | for (u=1; u <= maxSymbolValue+1; u++) { 99 | if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ 100 | cumul[u] = cumul[u-1] + 1; 101 | tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); 102 | } else { 103 | cumul[u] = cumul[u-1] + normalizedCounter[u-1]; 104 | } } 105 | cumul[maxSymbolValue+1] = tableSize+1; 106 | } 107 | 108 | /* Spread symbols */ 109 | { U32 position = 0; 110 | U32 symbol; 111 | for (symbol=0; symbol<=maxSymbolValue; symbol++) { 112 | int nbOccurrences; 113 | int const freq = normalizedCounter[symbol]; 114 | for (nbOccurrences=0; nbOccurrences highThreshold) 118 | position = (position + step) & tableMask; /* Low proba area */ 119 | } } 120 | 121 | assert(position==0); /* Must have initialized all positions */ 122 | } 123 | 124 | /* Build table */ 125 | { U32 u; for (u=0; u> 3) + 3; 189 | return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ 190 | } 191 | 192 | static size_t 193 | FSE_writeNCount_generic (void* header, size_t headerBufferSize, 194 | const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, 195 | unsigned writeIsSafe) 196 | { 197 | BYTE* const ostart = (BYTE*) header; 198 | BYTE* out = ostart; 199 | BYTE* const oend = ostart + headerBufferSize; 200 | int nbBits; 201 | const int tableSize = 1 << tableLog; 202 | int remaining; 203 | int threshold; 204 | U32 bitStream = 0; 205 | int bitCount = 0; 206 | unsigned symbol = 0; 207 | unsigned const alphabetSize = maxSymbolValue + 1; 208 | int previousIs0 = 0; 209 | 210 | /* Table Size */ 211 | bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; 212 | bitCount += 4; 213 | 214 | /* Init */ 215 | remaining = tableSize+1; /* +1 for extra accuracy */ 216 | threshold = tableSize; 217 | nbBits = tableLog+1; 218 | 219 | while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ 220 | if (previousIs0) { 221 | unsigned start = symbol; 222 | while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; 223 | if (symbol == alphabetSize) break; /* incorrect distribution */ 224 | while (symbol >= start+24) { 225 | start+=24; 226 | bitStream += 0xFFFFU << bitCount; 227 | if ((!writeIsSafe) && (out > oend-2)) 228 | return ERROR(dstSize_tooSmall); /* Buffer overflow */ 229 | out[0] = (BYTE) bitStream; 230 | out[1] = (BYTE)(bitStream>>8); 231 | out+=2; 232 | bitStream>>=16; 233 | } 234 | while (symbol >= start+3) { 235 | start+=3; 236 | bitStream += 3 << bitCount; 237 | bitCount += 2; 238 | } 239 | bitStream += (symbol-start) << bitCount; 240 | bitCount += 2; 241 | if (bitCount>16) { 242 | if ((!writeIsSafe) && (out > oend - 2)) 243 | return ERROR(dstSize_tooSmall); /* Buffer overflow */ 244 | out[0] = (BYTE)bitStream; 245 | out[1] = (BYTE)(bitStream>>8); 246 | out += 2; 247 | bitStream >>= 16; 248 | bitCount -= 16; 249 | } } 250 | { int count = normalizedCounter[symbol++]; 251 | int const max = (2*threshold-1) - remaining; 252 | remaining -= count < 0 ? -count : count; 253 | count++; /* +1 for extra accuracy */ 254 | if (count>=threshold) 255 | count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ 256 | bitStream += count << bitCount; 257 | bitCount += nbBits; 258 | bitCount -= (count>=1; } 262 | } 263 | if (bitCount>16) { 264 | if ((!writeIsSafe) && (out > oend - 2)) 265 | return ERROR(dstSize_tooSmall); /* Buffer overflow */ 266 | out[0] = (BYTE)bitStream; 267 | out[1] = (BYTE)(bitStream>>8); 268 | out += 2; 269 | bitStream >>= 16; 270 | bitCount -= 16; 271 | } } 272 | 273 | if (remaining != 1) 274 | return ERROR(GENERIC); /* incorrect normalized distribution */ 275 | assert(symbol <= alphabetSize); 276 | 277 | /* flush remaining bitStream */ 278 | if ((!writeIsSafe) && (out > oend - 2)) 279 | return ERROR(dstSize_tooSmall); /* Buffer overflow */ 280 | out[0] = (BYTE)bitStream; 281 | out[1] = (BYTE)(bitStream>>8); 282 | out+= (bitCount+7) /8; 283 | 284 | return (out-ostart); 285 | } 286 | 287 | 288 | size_t FSE_writeNCount (void* buffer, size_t bufferSize, 289 | const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) 290 | { 291 | if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ 292 | if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ 293 | 294 | if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) 295 | return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); 296 | 297 | return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); 298 | } 299 | 300 | 301 | /*-************************************************************** 302 | * FSE Compression Code 303 | ****************************************************************/ 304 | 305 | FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) 306 | { 307 | size_t size; 308 | if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; 309 | size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); 310 | return (FSE_CTable*)malloc(size); 311 | } 312 | 313 | void FSE_freeCTable (FSE_CTable* ct) { free(ct); } 314 | 315 | /* provides the minimum logSize to safely represent a distribution */ 316 | static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) 317 | { 318 | U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1; 319 | U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; 320 | U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; 321 | assert(srcSize > 1); /* Not supported, RLE should be used instead */ 322 | return minBits; 323 | } 324 | 325 | unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) 326 | { 327 | U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; 328 | U32 tableLog = maxTableLog; 329 | U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); 330 | assert(srcSize > 1); /* Not supported, RLE should be used instead */ 331 | if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; 332 | if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ 333 | if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ 334 | if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; 335 | if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; 336 | return tableLog; 337 | } 338 | 339 | unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) 340 | { 341 | return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); 342 | } 343 | 344 | 345 | /* Secondary normalization method. 346 | To be used when primary method fails. */ 347 | 348 | static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) 349 | { 350 | short const NOT_YET_ASSIGNED = -2; 351 | U32 s; 352 | U32 distributed = 0; 353 | U32 ToDistribute; 354 | 355 | /* Init */ 356 | U32 const lowThreshold = (U32)(total >> tableLog); 357 | U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); 358 | 359 | for (s=0; s<=maxSymbolValue; s++) { 360 | if (count[s] == 0) { 361 | norm[s]=0; 362 | continue; 363 | } 364 | if (count[s] <= lowThreshold) { 365 | norm[s] = -1; 366 | distributed++; 367 | total -= count[s]; 368 | continue; 369 | } 370 | if (count[s] <= lowOne) { 371 | norm[s] = 1; 372 | distributed++; 373 | total -= count[s]; 374 | continue; 375 | } 376 | 377 | norm[s]=NOT_YET_ASSIGNED; 378 | } 379 | ToDistribute = (1 << tableLog) - distributed; 380 | 381 | if (ToDistribute == 0) 382 | return 0; 383 | 384 | if ((total / ToDistribute) > lowOne) { 385 | /* risk of rounding to zero */ 386 | lowOne = (U32)((total * 3) / (ToDistribute * 2)); 387 | for (s=0; s<=maxSymbolValue; s++) { 388 | if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { 389 | norm[s] = 1; 390 | distributed++; 391 | total -= count[s]; 392 | continue; 393 | } } 394 | ToDistribute = (1 << tableLog) - distributed; 395 | } 396 | 397 | if (distributed == maxSymbolValue+1) { 398 | /* all values are pretty poor; 399 | probably incompressible data (should have already been detected); 400 | find max, then give all remaining points to max */ 401 | U32 maxV = 0, maxC = 0; 402 | for (s=0; s<=maxSymbolValue; s++) 403 | if (count[s] > maxC) { maxV=s; maxC=count[s]; } 404 | norm[maxV] += (short)ToDistribute; 405 | return 0; 406 | } 407 | 408 | if (total == 0) { 409 | /* all of the symbols were low enough for the lowOne or lowThreshold */ 410 | for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1)) 411 | if (norm[s] > 0) { ToDistribute--; norm[s]++; } 412 | return 0; 413 | } 414 | 415 | { U64 const vStepLog = 62 - tableLog; 416 | U64 const mid = (1ULL << (vStepLog-1)) - 1; 417 | U64 const rStep = ((((U64)1<> vStepLog); 423 | U32 const sEnd = (U32)(end >> vStepLog); 424 | U32 const weight = sEnd - sStart; 425 | if (weight < 1) 426 | return ERROR(GENERIC); 427 | norm[s] = (short)weight; 428 | tmpTotal = end; 429 | } } } 430 | 431 | return 0; 432 | } 433 | 434 | 435 | size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, 436 | const unsigned* count, size_t total, 437 | unsigned maxSymbolValue) 438 | { 439 | /* Sanity checks */ 440 | if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; 441 | if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ 442 | if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ 443 | if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ 444 | 445 | { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; 446 | U64 const scale = 62 - tableLog; 447 | U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ 448 | U64 const vStep = 1ULL<<(scale-20); 449 | int stillToDistribute = 1<> tableLog); 454 | 455 | for (s=0; s<=maxSymbolValue; s++) { 456 | if (count[s] == total) return 0; /* rle special case */ 457 | if (count[s] == 0) { normalizedCounter[s]=0; continue; } 458 | if (count[s] <= lowThreshold) { 459 | normalizedCounter[s] = -1; 460 | stillToDistribute--; 461 | } else { 462 | short proba = (short)((count[s]*step) >> scale); 463 | if (proba<8) { 464 | U64 restToBeat = vStep * rtbTable[proba]; 465 | proba += (count[s]*step) - ((U64)proba< restToBeat; 466 | } 467 | if (proba > largestP) { largestP=proba; largest=s; } 468 | normalizedCounter[s] = proba; 469 | stillToDistribute -= proba; 470 | } } 471 | if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { 472 | /* corner case, need another normalization method */ 473 | size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); 474 | if (FSE_isError(errorCode)) return errorCode; 475 | } 476 | else normalizedCounter[largest] += (short)stillToDistribute; 477 | } 478 | 479 | #if 0 480 | { /* Print Table (debug) */ 481 | U32 s; 482 | U32 nTotal = 0; 483 | for (s=0; s<=maxSymbolValue; s++) 484 | RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]); 485 | for (s=0; s<=maxSymbolValue; s++) 486 | nTotal += abs(normalizedCounter[s]); 487 | if (nTotal != (1U<>1); /* assumption : tableLog >= 1 */ 506 | FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); 507 | unsigned s; 508 | 509 | /* Sanity checks */ 510 | if (nbBits < 1) return ERROR(GENERIC); /* min size */ 511 | 512 | /* header */ 513 | tableU16[-2] = (U16) nbBits; 514 | tableU16[-1] = (U16) maxSymbolValue; 515 | 516 | /* Build table */ 517 | for (s=0; s FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ 585 | FSE_encodeSymbol(&bitC, &CState2, *--ip); 586 | FSE_encodeSymbol(&bitC, &CState1, *--ip); 587 | FSE_FLUSHBITS(&bitC); 588 | } 589 | 590 | /* 2 or 4 encoding per loop */ 591 | while ( ip>istart ) { 592 | 593 | FSE_encodeSymbol(&bitC, &CState2, *--ip); 594 | 595 | if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ 596 | FSE_FLUSHBITS(&bitC); 597 | 598 | FSE_encodeSymbol(&bitC, &CState1, *--ip); 599 | 600 | if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ 601 | FSE_encodeSymbol(&bitC, &CState2, *--ip); 602 | FSE_encodeSymbol(&bitC, &CState1, *--ip); 603 | } 604 | 605 | FSE_FLUSHBITS(&bitC); 606 | } 607 | 608 | FSE_flushCState(&bitC, &CState2); 609 | FSE_flushCState(&bitC, &CState1); 610 | return BIT_closeCStream(&bitC); 611 | } 612 | 613 | size_t FSE_compress_usingCTable (void* dst, size_t dstSize, 614 | const void* src, size_t srcSize, 615 | const FSE_CTable* ct) 616 | { 617 | unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); 618 | 619 | if (fast) 620 | return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); 621 | else 622 | return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); 623 | } 624 | 625 | 626 | size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } 627 | 628 | /* FSE_compress_wksp() : 629 | * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). 630 | * `wkspSize` size must be `(1< not compressible */ 655 | if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ 656 | } 657 | 658 | tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); 659 | CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) ); 660 | 661 | /* Write table description header */ 662 | { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); 663 | op += nc_err; 664 | } 665 | 666 | /* Compress */ 667 | CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) ); 668 | { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); 669 | if (cSize == 0) return 0; /* not enough space for compressed data */ 670 | op += cSize; 671 | } 672 | 673 | /* check compressibility */ 674 | if ( (size_t)(op-ostart) >= srcSize-1 ) return 0; 675 | 676 | return op-ostart; 677 | } 678 | 679 | typedef struct { 680 | FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; 681 | union { 682 | U32 hist_wksp[HIST_WKSP_SIZE_U32]; 683 | BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; 684 | } workspace; 685 | } fseWkspMax_t; 686 | 687 | size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) 688 | { 689 | fseWkspMax_t scratchBuffer; 690 | DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ 691 | if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); 692 | return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer)); 693 | } 694 | 695 | size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize) 696 | { 697 | return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); 698 | } 699 | 700 | 701 | #endif /* FSE_COMMONDEFS_ONLY */ 702 | -------------------------------------------------------------------------------- /FSE/fse.h: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * FSE : Finite State Entropy codec 3 | * Public Prototypes declaration 4 | * Copyright (c) 2013-2020, Yann Collet, Facebook, Inc. 5 | * 6 | * You can contact the author at : 7 | * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 8 | * 9 | * This source code is licensed under both the BSD-style license (found in the 10 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 11 | * in the COPYING file in the root directory of this source tree). 12 | * You may select, at your option, one of the above-listed licenses. 13 | ****************************************************************** */ 14 | 15 | #if defined (__cplusplus) 16 | extern "C" { 17 | #endif 18 | 19 | #ifndef FSE_H 20 | #define FSE_H 21 | 22 | 23 | /*-***************************************** 24 | * Dependencies 25 | ******************************************/ 26 | #include /* size_t, ptrdiff_t */ 27 | 28 | 29 | /*-***************************************** 30 | * FSE_PUBLIC_API : control library symbols visibility 31 | ******************************************/ 32 | #if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) 33 | # define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) 34 | #elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ 35 | # define FSE_PUBLIC_API __declspec(dllexport) 36 | #elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) 37 | # define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ 38 | #else 39 | # define FSE_PUBLIC_API 40 | #endif 41 | 42 | /*------ Version ------*/ 43 | #define FSE_VERSION_MAJOR 0 44 | #define FSE_VERSION_MINOR 9 45 | #define FSE_VERSION_RELEASE 0 46 | 47 | #define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE 48 | #define FSE_QUOTE(str) #str 49 | #define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) 50 | #define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) 51 | 52 | #define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) 53 | FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ 54 | 55 | 56 | /*-**************************************** 57 | * FSE simple functions 58 | ******************************************/ 59 | /*! FSE_compress() : 60 | Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. 61 | 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). 62 | @return : size of compressed data (<= dstCapacity). 63 | Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! 64 | if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. 65 | if FSE_isError(return), compression failed (more details using FSE_getErrorName()) 66 | */ 67 | FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, 68 | const void* src, size_t srcSize); 69 | 70 | /*! FSE_decompress(): 71 | Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', 72 | into already allocated destination buffer 'dst', of size 'dstCapacity'. 73 | @return : size of regenerated data (<= maxDstSize), 74 | or an error code, which can be tested using FSE_isError() . 75 | 76 | ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! 77 | Why ? : making this distinction requires a header. 78 | Header management is intentionally delegated to the user layer, which can better manage special cases. 79 | */ 80 | FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, 81 | const void* cSrc, size_t cSrcSize); 82 | 83 | 84 | /*-***************************************** 85 | * Tool functions 86 | ******************************************/ 87 | FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ 88 | 89 | /* Error Management */ 90 | FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ 91 | FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ 92 | 93 | 94 | /*-***************************************** 95 | * FSE advanced functions 96 | ******************************************/ 97 | /*! FSE_compress2() : 98 | Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' 99 | Both parameters can be defined as '0' to mean : use default value 100 | @return : size of compressed data 101 | Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! 102 | if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. 103 | if FSE_isError(return), it's an error code. 104 | */ 105 | FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); 106 | 107 | 108 | /*-***************************************** 109 | * FSE detailed API 110 | ******************************************/ 111 | /*! 112 | FSE_compress() does the following: 113 | 1. count symbol occurrence from source[] into table count[] (see hist.h) 114 | 2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) 115 | 3. save normalized counters to memory buffer using writeNCount() 116 | 4. build encoding table 'CTable' from normalized counters 117 | 5. encode the data stream using encoding table 'CTable' 118 | 119 | FSE_decompress() does the following: 120 | 1. read normalized counters with readNCount() 121 | 2. build decoding table 'DTable' from normalized counters 122 | 3. decode the data stream using decoding table 'DTable' 123 | 124 | The following API allows targeting specific sub-functions for advanced tasks. 125 | For example, it's possible to compress several blocks using the same 'CTable', 126 | or to save and provide normalized distribution using external method. 127 | */ 128 | 129 | /* *** COMPRESSION *** */ 130 | 131 | /*! FSE_optimalTableLog(): 132 | dynamically downsize 'tableLog' when conditions are met. 133 | It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. 134 | @return : recommended tableLog (necessarily <= 'maxTableLog') */ 135 | FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); 136 | 137 | /*! FSE_normalizeCount(): 138 | normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) 139 | 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). 140 | @return : tableLog, 141 | or an errorCode, which can be tested using FSE_isError() */ 142 | FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, 143 | const unsigned* count, size_t srcSize, unsigned maxSymbolValue); 144 | 145 | /*! FSE_NCountWriteBound(): 146 | Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. 147 | Typically useful for allocation purpose. */ 148 | FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); 149 | 150 | /*! FSE_writeNCount(): 151 | Compactly save 'normalizedCounter' into 'buffer'. 152 | @return : size of the compressed table, 153 | or an errorCode, which can be tested using FSE_isError(). */ 154 | FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, 155 | const short* normalizedCounter, 156 | unsigned maxSymbolValue, unsigned tableLog); 157 | 158 | /*! Constructor and Destructor of FSE_CTable. 159 | Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ 160 | typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ 161 | FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog); 162 | FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); 163 | 164 | /*! FSE_buildCTable(): 165 | Builds `ct`, which must be already allocated, using FSE_createCTable(). 166 | @return : 0, or an errorCode, which can be tested using FSE_isError() */ 167 | FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); 168 | 169 | /*! FSE_compress_usingCTable(): 170 | Compress `src` using `ct` into `dst` which must be already allocated. 171 | @return : size of compressed data (<= `dstCapacity`), 172 | or 0 if compressed data could not fit into `dst`, 173 | or an errorCode, which can be tested using FSE_isError() */ 174 | FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); 175 | 176 | /*! 177 | Tutorial : 178 | ---------- 179 | The first step is to count all symbols. FSE_count() does this job very fast. 180 | Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. 181 | 'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] 182 | maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) 183 | FSE_count() will return the number of occurrence of the most frequent symbol. 184 | This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. 185 | If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). 186 | 187 | The next step is to normalize the frequencies. 188 | FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. 189 | It also guarantees a minimum of 1 to any Symbol with frequency >= 1. 190 | You can use 'tableLog'==0 to mean "use default tableLog value". 191 | If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), 192 | which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). 193 | 194 | The result of FSE_normalizeCount() will be saved into a table, 195 | called 'normalizedCounter', which is a table of signed short. 196 | 'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. 197 | The return value is tableLog if everything proceeded as expected. 198 | It is 0 if there is a single symbol within distribution. 199 | If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). 200 | 201 | 'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). 202 | 'buffer' must be already allocated. 203 | For guaranteed success, buffer size must be at least FSE_headerBound(). 204 | The result of the function is the number of bytes written into 'buffer'. 205 | If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). 206 | 207 | 'normalizedCounter' can then be used to create the compression table 'CTable'. 208 | The space required by 'CTable' must be already allocated, using FSE_createCTable(). 209 | You can then use FSE_buildCTable() to fill 'CTable'. 210 | If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). 211 | 212 | 'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). 213 | Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' 214 | The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. 215 | If it returns '0', compressed data could not fit into 'dst'. 216 | If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). 217 | */ 218 | 219 | 220 | /* *** DECOMPRESSION *** */ 221 | 222 | /*! FSE_readNCount(): 223 | Read compactly saved 'normalizedCounter' from 'rBuffer'. 224 | @return : size read from 'rBuffer', 225 | or an errorCode, which can be tested using FSE_isError(). 226 | maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ 227 | FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, 228 | unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, 229 | const void* rBuffer, size_t rBuffSize); 230 | 231 | /*! Constructor and Destructor of FSE_DTable. 232 | Note that its size depends on 'tableLog' */ 233 | typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ 234 | FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); 235 | FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); 236 | 237 | /*! FSE_buildDTable(): 238 | Builds 'dt', which must be already allocated, using FSE_createDTable(). 239 | return : 0, or an errorCode, which can be tested using FSE_isError() */ 240 | FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); 241 | 242 | /*! FSE_decompress_usingDTable(): 243 | Decompress compressed source `cSrc` of size `cSrcSize` using `dt` 244 | into `dst` which must be already allocated. 245 | @return : size of regenerated data (necessarily <= `dstCapacity`), 246 | or an errorCode, which can be tested using FSE_isError() */ 247 | FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); 248 | 249 | /*! 250 | Tutorial : 251 | ---------- 252 | (Note : these functions only decompress FSE-compressed blocks. 253 | If block is uncompressed, use memcpy() instead 254 | If block is a single repeated byte, use memset() instead ) 255 | 256 | The first step is to obtain the normalized frequencies of symbols. 257 | This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). 258 | 'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. 259 | In practice, that means it's necessary to know 'maxSymbolValue' beforehand, 260 | or size the table to handle worst case situations (typically 256). 261 | FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. 262 | The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. 263 | Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. 264 | If there is an error, the function will return an error code, which can be tested using FSE_isError(). 265 | 266 | The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. 267 | This is performed by the function FSE_buildDTable(). 268 | The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). 269 | If there is an error, the function will return an error code, which can be tested using FSE_isError(). 270 | 271 | `FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). 272 | `cSrcSize` must be strictly correct, otherwise decompression will fail. 273 | FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). 274 | If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) 275 | */ 276 | 277 | #endif /* FSE_H */ 278 | 279 | #if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) 280 | #define FSE_H_FSE_STATIC_LINKING_ONLY 281 | 282 | /* *** Dependency *** */ 283 | #include "bitstream.h" 284 | 285 | 286 | /* ***************************************** 287 | * Static allocation 288 | *******************************************/ 289 | /* FSE buffer bounds */ 290 | #define FSE_NCOUNTBOUND 512 291 | #define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) 292 | #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ 293 | 294 | /* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ 295 | #define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2)) 296 | #define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog))) 297 | 298 | /* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ 299 | #define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) 300 | #define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable)) 301 | 302 | 303 | /* ***************************************** 304 | * FSE advanced API 305 | ***************************************** */ 306 | 307 | unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); 308 | /**< same as FSE_optimalTableLog(), which used `minus==2` */ 309 | 310 | /* FSE_compress_wksp() : 311 | * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). 312 | * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. 313 | */ 314 | #define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) 315 | size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); 316 | 317 | size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); 318 | /**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ 319 | 320 | size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); 321 | /**< build a fake FSE_CTable, designed to compress always the same symbolValue */ 322 | 323 | /* FSE_buildCTable_wksp() : 324 | * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). 325 | * `wkspSize` must be >= `(1<= BIT_DStream_completed 464 | 465 | When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. 466 | Checking if DStream has reached its end is performed by : 467 | BIT_endOfDStream(&DStream); 468 | Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. 469 | FSE_endOfDState(&DState); 470 | */ 471 | 472 | 473 | /* ***************************************** 474 | * FSE unsafe API 475 | *******************************************/ 476 | static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); 477 | /* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ 478 | 479 | 480 | /* ***************************************** 481 | * Implementation of inlined functions 482 | *******************************************/ 483 | typedef struct { 484 | int deltaFindState; 485 | U32 deltaNbBits; 486 | } FSE_symbolCompressionTransform; /* total 8 bytes */ 487 | 488 | MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) 489 | { 490 | const void* ptr = ct; 491 | const U16* u16ptr = (const U16*) ptr; 492 | const U32 tableLog = MEM_read16(ptr); 493 | statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; 495 | statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1); 496 | statePtr->stateLog = tableLog; 497 | } 498 | 499 | 500 | /*! FSE_initCState2() : 501 | * Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) 502 | * uses the smallest state value possible, saving the cost of this symbol */ 503 | MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) 504 | { 505 | FSE_initCState(statePtr, ct); 506 | { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; 507 | const U16* stateTable = (const U16*)(statePtr->stateTable); 508 | U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); 509 | statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; 510 | statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; 511 | } 512 | } 513 | 514 | MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol) 515 | { 516 | FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; 517 | const U16* const stateTable = (const U16*)(statePtr->stateTable); 518 | U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); 519 | BIT_addBits(bitC, statePtr->value, nbBitsOut); 520 | statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; 521 | } 522 | 523 | MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) 524 | { 525 | BIT_addBits(bitC, statePtr->value, statePtr->stateLog); 526 | BIT_flushBits(bitC); 527 | } 528 | 529 | 530 | /* FSE_getMaxNbBits() : 531 | * Approximate maximum cost of a symbol, in bits. 532 | * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) 533 | * note 1 : assume symbolValue is valid (<= maxSymbolValue) 534 | * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ 535 | MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) 536 | { 537 | const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; 538 | return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16; 539 | } 540 | 541 | /* FSE_bitCost() : 542 | * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) 543 | * note 1 : assume symbolValue is valid (<= maxSymbolValue) 544 | * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ 545 | MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog) 546 | { 547 | const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; 548 | U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16; 549 | U32 const threshold = (minNbBits+1) << 16; 550 | assert(tableLog < 16); 551 | assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */ 552 | { U32 const tableSize = 1 << tableLog; 553 | U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize); 554 | U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */ 555 | U32 const bitMultiplier = 1 << accuracyLog; 556 | assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold); 557 | assert(normalizedDeltaFromThreshold <= bitMultiplier); 558 | return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold; 559 | } 560 | } 561 | 562 | 563 | /* ====== Decompression ====== */ 564 | 565 | typedef struct { 566 | U16 tableLog; 567 | U16 fastMode; 568 | } FSE_DTableHeader; /* sizeof U32 */ 569 | 570 | typedef struct 571 | { 572 | unsigned short newState; 573 | unsigned char symbol; 574 | unsigned char nbBits; 575 | } FSE_decode_t; /* size == U32 */ 576 | 577 | MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) 578 | { 579 | const void* ptr = dt; 580 | const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; 581 | DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); 582 | BIT_reloadDStream(bitD); 583 | DStatePtr->table = dt + 1; 584 | } 585 | 586 | MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) 587 | { 588 | FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; 589 | return DInfo.symbol; 590 | } 591 | 592 | MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) 593 | { 594 | FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; 595 | U32 const nbBits = DInfo.nbBits; 596 | size_t const lowBits = BIT_readBits(bitD, nbBits); 597 | DStatePtr->state = DInfo.newState + lowBits; 598 | } 599 | 600 | MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) 601 | { 602 | FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; 603 | U32 const nbBits = DInfo.nbBits; 604 | BYTE const symbol = DInfo.symbol; 605 | size_t const lowBits = BIT_readBits(bitD, nbBits); 606 | 607 | DStatePtr->state = DInfo.newState + lowBits; 608 | return symbol; 609 | } 610 | 611 | /*! FSE_decodeSymbolFast() : 612 | unsafe, only works if no symbol has a probability > 50% */ 613 | MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) 614 | { 615 | FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; 616 | U32 const nbBits = DInfo.nbBits; 617 | BYTE const symbol = DInfo.symbol; 618 | size_t const lowBits = BIT_readBitsFast(bitD, nbBits); 619 | 620 | DStatePtr->state = DInfo.newState + lowBits; 621 | return symbol; 622 | } 623 | 624 | MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) 625 | { 626 | return DStatePtr->state == 0; 627 | } 628 | 629 | 630 | 631 | #ifndef FSE_COMMONDEFS_ONLY 632 | 633 | /* ************************************************************** 634 | * Tuning parameters 635 | ****************************************************************/ 636 | /*!MEMORY_USAGE : 637 | * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) 638 | * Increasing memory usage improves compression ratio 639 | * Reduced memory usage can improve speed, due to cache effect 640 | * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ 641 | #ifndef FSE_MAX_MEMORY_USAGE 642 | # define FSE_MAX_MEMORY_USAGE 14 643 | #endif 644 | #ifndef FSE_DEFAULT_MEMORY_USAGE 645 | # define FSE_DEFAULT_MEMORY_USAGE 13 646 | #endif 647 | #if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE) 648 | # error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE" 649 | #endif 650 | 651 | /*!FSE_MAX_SYMBOL_VALUE : 652 | * Maximum symbol value authorized. 653 | * Required for proper stack allocation */ 654 | #ifndef FSE_MAX_SYMBOL_VALUE 655 | # define FSE_MAX_SYMBOL_VALUE 255 656 | #endif 657 | 658 | /* ************************************************************** 659 | * template functions type & suffix 660 | ****************************************************************/ 661 | #define FSE_FUNCTION_TYPE BYTE 662 | #define FSE_FUNCTION_EXTENSION 663 | #define FSE_DECODE_TYPE FSE_decode_t 664 | 665 | 666 | #endif /* !FSE_COMMONDEFS_ONLY */ 667 | 668 | 669 | /* *************************************************************** 670 | * Constants 671 | *****************************************************************/ 672 | #define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) 673 | #define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX 680 | # error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" 681 | #endif 682 | 683 | #define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) 684 | 685 | 686 | #endif /* FSE_STATIC_LINKING_ONLY */ 687 | 688 | 689 | #if defined (__cplusplus) 690 | } 691 | #endif 692 | --------------------------------------------------------------------------------