├── .gitattributes ├── .gitignore ├── CONTRIBUTION.md ├── COPYING ├── ECL_JH_States.h ├── ECL_NanoLZ.c ├── ECL_NanoLZ.h ├── ECL_NanoLZ_schemes_inline.h ├── ECL_ZeroDevourer.c ├── ECL_ZeroDevourer.h ├── ECL_ZeroEater.c ├── ECL_ZeroEater.h ├── ECL_common.c ├── ECL_config.h ├── ECL_utils.h ├── NEWS.md ├── README.md ├── ecl-all-c-included └── ECL_all_c_included.c ├── formats ├── ECL_E_number_format.txt ├── ECL_JH.txt ├── ECL_NanoLZ.txt ├── ECL_ZeroDevourer.txt └── ECL_ZeroEater.txt ├── sample ├── run-msvc2013-helper.bat ├── run-msvc2015-helper.bat ├── sample.cpp ├── unix_clang.sh ├── unix_gcc.sh ├── win_gcc.bat └── win_msvc.bat └── tests ├── ntest ├── ntest.cpp └── ntest.h ├── run-msvc2013-helper.bat ├── run-msvc2015-helper.bat ├── run-msvc2017-x64-helper.bat ├── run-msvc2017-x86-helper.bat ├── run-msvc2019-x64-helper.bat ├── run-msvc2019-x86-helper.bat ├── tests.cpp ├── tests_base_inline.cpp ├── tests_common_inline.hpp ├── tests_datasets_inline.hpp ├── tests_errors_inline.hpp ├── tests_perf_inline.hpp ├── tests_random_data_inline.hpp ├── unix_clang.sh ├── unix_gcc.sh ├── win_gcc.bat └── win_msvc.bat /.gitattributes: -------------------------------------------------------------------------------- 1 | # default 2 | * text=auto 3 | 4 | # code files 5 | *.h text 6 | *.c text 7 | *.hpp text 8 | *.cpp text 9 | *.m text 10 | *.mm text 11 | 12 | # script files 13 | *.py text 14 | *.sh text eol=lf 15 | 16 | # config, docs 17 | *.cfg text=auto 18 | *.txt text=auto 19 | 20 | # qt config file 21 | *.conf text eol=crlf 22 | 23 | # translation files 24 | *.ts text eol=lf 25 | 26 | # platform files 27 | *.sln text eol=crlf 28 | *.bat text eol=crlf 29 | *.rc text eol=lf 30 | *.vcxproj text eol=crlf 31 | *.vcxproj.filters text eol=crlf 32 | 33 | # wix files 34 | *.wxs text eol=crlf 35 | *.wxi text eol=crlf 36 | 37 | # binary files 38 | *.jpg binary 39 | *.jpeg binary 40 | *.png binary 41 | *.ico binary 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.obj 2 | *.o 3 | *.exe 4 | -------------------------------------------------------------------------------- /CONTRIBUTION.md: -------------------------------------------------------------------------------- 1 | If you found a bug - feel free to create an issue for it, or even propose a patch to "contrib" branch. 2 | 3 | If you want to contribute a patch - you're assumed to agree with project license, minor bug fixes are not supposed to change copyrights. 4 | 5 | If you propose a patch with a bug fix - please add appropriate test for it (near other tests). 6 | 7 | Due to project specifics (embedded) - it's likely to be very hard to reproduce some hardware/compiler dependent issues, 8 | also I don't have CI builders for testing regular platforms, so I may be reluctant to apply such updates. 9 | 10 | If you want to create a fork and rework some stuff to optimize for your target platform (or by other reasons) - please update ECL_VERSION_BRANCH macro to be anything but "master", 11 | preferably to clearly identify your fork. 12 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright 2017 - 2018 Evgeniy Evstratov 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation files 5 | (the "Software"), to deal in the Software without restriction, 6 | including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software 9 | is furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 16 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 18 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 | OTHER DEALINGS IN THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /ECL_JH_States.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 - 2018 Evgeniy Evstratov 3 | * 4 | * Permission is hereby granted, free of charge, to any person 5 | * obtaining a copy of this software and associated documentation 6 | * files (the "Software"), to deal in the Software without 7 | * restriction, including without limitation the rights to use, 8 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the 10 | * Software is furnished to do so, subject to the following 11 | * conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be 14 | * included in all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 20 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 21 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 23 | * OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | #ifndef ECL_JH_STATES_H_ 27 | #define ECL_JH_STATES_H_ 28 | 29 | #include "ECL_config.h" 30 | 31 | #ifdef __cplusplus 32 | extern "C" { 33 | #endif 34 | 35 | /* 36 | ECL Jumping Header states for writing and reading. 37 | - lower bits go first 38 | - you can continue calling read/write safely even if state has invalidated, but only if *Init method succeeded 39 | - reading/writing on invalid buffer is safe but result is undefined 40 | */ 41 | typedef struct { 42 | uint8_t* byte; 43 | uint8_t* next; 44 | uint8_t* end; 45 | uint8_t n_bits; 46 | uint8_t is_valid; /* to check after any transaction */ 47 | } ECL_JH_WState; 48 | 49 | typedef struct { 50 | const uint8_t* byte; 51 | const uint8_t* next; 52 | const uint8_t* end; 53 | uint8_t n_bits; 54 | uint8_t is_valid; /* to check after any transaction */ 55 | } ECL_JH_RState; 56 | 57 | 58 | void ECL_JH_WInit(ECL_JH_WState* state, uint8_t* ptr, ECL_usize size, ECL_usize start); /* state's constructor */ 59 | void ECL_JH_RInit(ECL_JH_RState* state, const uint8_t* ptr, ECL_usize size, ECL_usize start); /* -:- */ 60 | void ECL_JH_Write(ECL_JH_WState* state, uint8_t value, uint8_t bits); /* writes 'bits' bits of 'value' to 'state' */ 61 | uint8_t ECL_JH_Read(ECL_JH_RState* state, uint8_t bits); /* returns value of 'bits' size read from 'state' */ 62 | void ECL_JH_WJump(ECL_JH_WState* state, ECL_usize distance); /* moves pointer to next byte at 'distance' if possible */ 63 | void ECL_JH_RJump(ECL_JH_RState* state, ECL_usize distance); /* -:- */ 64 | 65 | 66 | /* E-numbers part */ 67 | void ECL_JH_Write_E2(ECL_JH_WState* state, ECL_usize value); /* writes 'value' to 'state' in E2 number format */ 68 | ECL_usize ECL_JH_Read_E2(ECL_JH_RState* state); /* reads from 'state' a value in E2 number format */ 69 | uint8_t ECL_Evaluate_E2(ECL_usize number); /* returns amount of bits required for coding 'number' in E2 format */ 70 | 71 | void ECL_JH_Write_E4E5(ECL_JH_WState* state, ECL_usize value); /* -:- */ 72 | ECL_usize ECL_JH_Read_E4E5(ECL_JH_RState* state); /* -:- */ 73 | uint8_t ECL_Evaluate_E4E5(ECL_usize number); /* -:- */ 74 | 75 | /* other numbers are declared via macros below */ 76 | 77 | #define ECL_E_NUMBER_DECLARE_SIMPLE(num) \ 78 | void ECL_JH_Write_E##num(ECL_JH_WState* state, ECL_usize value); \ 79 | ECL_usize ECL_JH_Read_E##num(ECL_JH_RState* state); \ 80 | uint8_t ECL_Evaluate_E##num(ECL_usize number); 81 | 82 | ECL_E_NUMBER_DECLARE_SIMPLE(3) 83 | ECL_E_NUMBER_DECLARE_SIMPLE(4) 84 | ECL_E_NUMBER_DECLARE_SIMPLE(5) 85 | ECL_E_NUMBER_DECLARE_SIMPLE(6) 86 | ECL_E_NUMBER_DECLARE_SIMPLE(7) 87 | 88 | #undef ECL_E_NUMBER_DECLARE_SIMPLE 89 | 90 | #define ECL_E_NUMBER_DECLARE_X2(num1, num2) \ 91 | void ECL_JH_Write_E##num1##E##num2(ECL_JH_WState* state, ECL_usize value); \ 92 | ECL_usize ECL_JH_Read_E##num1##E##num2(ECL_JH_RState* state); \ 93 | uint8_t ECL_Evaluate_E##num1##E##num2(ECL_usize number); 94 | 95 | ECL_E_NUMBER_DECLARE_X2(4, 5) 96 | ECL_E_NUMBER_DECLARE_X2(5, 2) 97 | ECL_E_NUMBER_DECLARE_X2(5, 3) 98 | ECL_E_NUMBER_DECLARE_X2(5, 4) 99 | ECL_E_NUMBER_DECLARE_X2(6, 2) 100 | ECL_E_NUMBER_DECLARE_X2(6, 3) 101 | ECL_E_NUMBER_DECLARE_X2(6, 4) 102 | ECL_E_NUMBER_DECLARE_X2(7, 2) 103 | ECL_E_NUMBER_DECLARE_X2(7, 3) 104 | ECL_E_NUMBER_DECLARE_X2(7, 4) 105 | 106 | #undef ECL_E_NUMBER_DECLARE_X2 107 | 108 | 109 | #ifdef __cplusplus 110 | } 111 | #endif 112 | 113 | #endif 114 | -------------------------------------------------------------------------------- /ECL_NanoLZ.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 - 2018 Evgeniy Evstratov 3 | * 4 | * Permission is hereby granted, free of charge, to any person 5 | * obtaining a copy of this software and associated documentation 6 | * files (the "Software"), to deal in the Software without 7 | * restriction, including without limitation the rights to use, 8 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the 10 | * Software is furnished to do so, subject to the following 11 | * conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be 14 | * included in all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 20 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 21 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 23 | * OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | #ifndef ECL_NANO_LZ_ 27 | #define ECL_NANO_LZ_ 28 | 29 | #include "ECL_config.h" 30 | 31 | #include 32 | 33 | /* 34 | Calculates size of buffer to fit compressed version of any data of 'src_size' size. 35 | Formula is for ECL_NANOLZ_SCHEME1 scheme, for other schemes may not give enough size, 36 | but in such case when compression fails - size is already bigger than source size, so compressed stream isn't worth to be used. 37 | */ 38 | #define ECL_NANO_LZ_GET_BOUND(src_size) ((src_size) + 2 + sizeof(ECL_usize) + (sizeof(ECL_usize) / 4)) 39 | 40 | 41 | #ifdef __cplusplus 42 | extern "C" { 43 | #endif 44 | 45 | /* 46 | Declare static counters for tracking statistic on data decompression. 47 | */ 48 | #ifdef ECL_USE_STAT_COUNTERS 49 | #define ECL_NANO_LZ_DECOMPRESSION_OPCODE_PICK_COUNTERS_COUNT (8+1) 50 | extern int ECL_NanoLZ_Decompression_OpcodePickCounters[ECL_NANO_LZ_DECOMPRESSION_OPCODE_PICK_COUNTERS_COUNT]; 51 | #endif 52 | 53 | 54 | /* 55 | Schemes are: 56 | - incompatible between each other, so you have to use same scheme during compression and decompression of your dataset for correct result; 57 | - not included as identifiers in compressed stream in any way (so decompression method has 'scheme' parameter as well); 58 | - intended to be tested with target datasets to determine optimal one (different schemes would provide different compression level and performance); 59 | - can be extended with your own scheme, optimized for your particular data (if you know what you're doing), 60 | in this case it's recommended to assign bigger values for them (e.g. ECL_NANOLZ_PETER_SCHEME = 128,) 61 | in case you plan to use ECL as dynamic library and need binary compatibility. 62 | */ 63 | typedef enum { 64 | #if ECL_NANO_LZ_IS_SCHEME_ENABLED(1) 65 | ECL_NANOLZ_SCHEME1 = 0, /* main scheme, highly optimized for small datasets */ 66 | #endif 67 | #if ECL_NANO_LZ_IS_SCHEME_ENABLED(2) 68 | ECL_NANOLZ_SCHEME2_DEMO = 1, /* demo scheme. small code size, weak compression, high decompression speed - example for writing custom scheme */ 69 | #endif 70 | /* user-defined scheme codes are recommended to start from 128 */ 71 | } ECL_NanoLZ_Scheme; 72 | 73 | 74 | /* macro representing set of all schemes (used for testing) */ 75 | #if (ECL_NANO_LZ_ONLY_SCHEME == 0) 76 | #define ECL_NANO_LZ_SCHEMES_ALL {ECL_NANOLZ_SCHEME1, ECL_NANOLZ_SCHEME2_DEMO} 77 | #elif (ECL_NANO_LZ_ONLY_SCHEME == 1) 78 | #define ECL_NANO_LZ_SCHEMES_ALL {ECL_NANOLZ_SCHEME1} 79 | #else 80 | #define ECL_NANO_LZ_SCHEMES_ALL {ECL_NANOLZ_SCHEME2_DEMO} 81 | #endif 82 | 83 | 84 | /* 85 | The slowest mode, compresses 'src_size' bytes starting at 'src' to destination 'dst' that can hold at most 'dst_size' bytes. 86 | Function returns amount of bytes in resulted compressed stream, or 0 in case of error. 87 | Extra memory usage: 0. 88 | - 'search_limit' is maximum amount of bytes to look back when searching for a match, increasing this parameter can decrease performance dramatically. 89 | - to find enough size for output buffer: dst_size = ECL_NANO_LZ_GET_BOUND(src_size); 90 | See full compress/decompress example usage near decompression function. 91 | */ 92 | ECL_EXPORTED_API ECL_usize ECL_NanoLZ_Compress_slow(ECL_NanoLZ_Scheme scheme, const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size, ECL_usize search_limit); 93 | 94 | 95 | /* 96 | All mid* compressors are the most applicable for embedded. Require temporary buffer (not intersecting src or dst) provided via last parameter: 97 | - mid1, mid1min have 'buf_256' parameter which has to point to a buffer of 256 bytes; 98 | - mid2, mid2min have 'buf_513' (NOTE: not 512) parameter which has to point to a buffer of 513 bytes; 99 | 100 | All mid* algorithms require 'src_size' to not exceed 64k. 101 | 102 | In particular mid1, mid2 algorithms require 'dst_size' to be bigger than 'src_size'/2 (in case you planned to neglect ECL_NANO_LZ_GET_BOUND macro), 103 | also if your data is barely compressible - these two algorithms will work better if you give 'dst_size' = up to "ECL_NANO_LZ_GET_BOUND(src_size) + src_size/2 + 1" 104 | (see ECL_NANO_LZ_MID_GET_BOUND_OPTIMAL). 105 | 106 | mid1min, mid2min are optimized versions of mid1, mid2 with 'search_limit'==1. Provide higher performance and smaller code size (but only basic compression), 107 | these are by fact the fastest NanoLZ compression methods. 108 | */ 109 | ECL_EXPORTED_API ECL_usize ECL_NanoLZ_Compress_mid1(ECL_NanoLZ_Scheme scheme, const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size, ECL_usize search_limit, void* buf_256); 110 | ECL_EXPORTED_API ECL_usize ECL_NanoLZ_Compress_mid2(ECL_NanoLZ_Scheme scheme, const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size, ECL_usize search_limit, void* buf_513); 111 | ECL_EXPORTED_API ECL_usize ECL_NanoLZ_Compress_mid1min(ECL_NanoLZ_Scheme scheme, const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size, void* buf_256); 112 | ECL_EXPORTED_API ECL_usize ECL_NanoLZ_Compress_mid2min(ECL_NanoLZ_Scheme scheme, const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size, void* buf_513); 113 | 114 | /* 115 | Helper macro defining optimal reserved output size for mid* algorithms (except mid*min) 116 | - reserving extra space in output can improve compression ratio of mid* algorithms in cases when data is poorly compressible. 117 | */ 118 | #define ECL_NANO_LZ_MID_GET_BOUND_OPTIMAL(src_size) (ECL_NANO_LZ_GET_BOUND(src_size) + ((src_size)/2) + 1) 119 | 120 | #ifndef ECL_EXCLUDE_HIMEM 121 | 122 | /* 123 | Following macros are for manual allocations of buffers for fast1/fast2 algorithms. 124 | */ 125 | #define ECL_NANO_LZ_GET_FAST1_MAP_BUF_SIZE() (257 * sizeof(ECL_usize)) 126 | #define ECL_NANO_LZ_GET_FAST1_WINDOW_BUF_SIZE(window_size_bits) (((1UL << (window_size_bits)) + 1) * sizeof(ECL_usize)) 127 | /* */ 128 | #define ECL_NANO_LZ_GET_FAST2_MAP_BUF_SIZE() (65537 * sizeof(ECL_usize)) 129 | #define ECL_NANO_LZ_GET_FAST2_WINDOW_BUF_SIZE(window_size_bits) ECL_NANO_LZ_GET_FAST1_WINDOW_BUF_SIZE(window_size_bits) 130 | 131 | /* 132 | Struct with parameters to call fast1/fast2 compression algorithms, has to be initialized with 133 | ECL_NanoLZ_FastParams_Alloc1/ECL_NanoLZ_FastParams_Alloc2 respectively and destroyed with ECL_NanoLZ_FastParams_Destroy. 134 | */ 135 | typedef struct { 136 | void* buf_map; 137 | void* buf_window; 138 | uint32_t window_size_bits; 139 | } ECL_NanoLZ_FastParams; 140 | 141 | /* 142 | TLDR version: 143 | - consumption with *Alloc1 is (258 + (1 << window_size_bits)) * sizeof(ECL_usize) 144 | - consumption with *Alloc2 is (65538 + (1 << window_size_bits)) * sizeof(ECL_usize) 145 | 146 | Initializers for fast1/fast2 compressors, return whether succeeded. 147 | - *Alloc1 allocates buffers with sizes according to ECL_NANO_LZ_GET_FAST1* macros - valid for fast1 algorithm; 148 | - *Alloc2 allocates buffers with sizes according to ECL_NANO_LZ_GET_FAST2* macros - valid for fast2 and fast1 algorithms; 149 | 150 | For maximum efficiency window_size_bits should be = [log2(src_size)] (see ECL_LogRoundUp function). 151 | */ 152 | ECL_EXPORTED_API bool ECL_NanoLZ_FastParams_Alloc1(ECL_NanoLZ_FastParams* p, uint8_t window_size_bits); 153 | ECL_EXPORTED_API bool ECL_NanoLZ_FastParams_Alloc2(ECL_NanoLZ_FastParams* p, uint8_t window_size_bits); 154 | 155 | /* 156 | Deallocates buffers - call after you're done with FastParams. Necessary to call if *Alloc succeeded. 157 | */ 158 | ECL_EXPORTED_API void ECL_NanoLZ_FastParams_Destroy(ECL_NanoLZ_FastParams* p); 159 | 160 | /* 161 | Faster indexed (optimized) compressors using pre-allocated buffers, behave equally to ECL_NanoLZ_Compress_slow with only difference in speed and used memory. 162 | - prefer fast1 algorithm for small datasets (roughly, < 1500 bytes); 163 | - prefer fast2 for bigger datasets; 164 | 165 | Memory consumption of these methods is big for embedded but it would work well if you compress data on powerful (say, PC) side and send it 166 | to embedded hardware where it's decompressed. To compress data on hardware with limited resources you would sacrifice either compression ratio or time. 167 | 168 | Normal 'search_limit' value is typically around 10 - 50. Value of '-1' aims to maximum (and longest) compression - for small datasets should be fine. 169 | 170 | Usage: 171 | MyPODDataStruct my_data; 172 | ECL_usize compressed_size_limit = ECL_NANO_LZ_GET_BOUND(sizeof(my_data)); 173 | uint8_t* compressed_stream = (uint8_t*)malloc( compressed_size_limit ); 174 | ECL_NanoLZ_FastParams fp; 175 | ECL_NanoLZ_FastParams_Alloc2(&fp, 10); 176 | ECL_usize compressed_size = ECL_NanoLZ_Compress_fast2(ECL_NANOLZ_SCHEME1, (const uint8_t*)&my_data, sizeof(my_data), compressed_stream, compressed_size_limit, 20, &fp); 177 | ECL_NanoLZ_FastParams_Destroy(&fp); 178 | */ 179 | ECL_EXPORTED_API ECL_usize ECL_NanoLZ_Compress_fast1(ECL_NanoLZ_Scheme scheme, const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size, ECL_usize search_limit, ECL_NanoLZ_FastParams* p); 180 | ECL_EXPORTED_API ECL_usize ECL_NanoLZ_Compress_fast2(ECL_NanoLZ_Scheme scheme, const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size, ECL_usize search_limit, ECL_NanoLZ_FastParams* p); 181 | 182 | /* 183 | Generic compression method for non-limited environment (not embedded). 184 | Calls one of other compression methods (and allocates appropriate buffers according to input data size), to result in maximum performance. 185 | */ 186 | ECL_EXPORTED_API ECL_usize ECL_NanoLZ_Compress_auto(ECL_NanoLZ_Scheme scheme, const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size, ECL_usize search_limit); 187 | /* 188 | Extended _auto version - allows to specify successfully preallocated FastParams with *Alloc1 and *Alloc2 (prealloc1 and prealloc2 correspondingly). 189 | If a preallocated parameter is NULL - default allocation occurs inside. 190 | Allows: 191 | - to store successfully preallocated FastParams outside and use them for multiple compression runs, thus saving time on malloc/free calls; 192 | - to specify custom window size. 193 | */ 194 | ECL_EXPORTED_API ECL_usize ECL_NanoLZ_Compress_auto_ex(ECL_NanoLZ_Scheme scheme, const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size, ECL_usize search_limit 195 | , ECL_NanoLZ_FastParams* prealloc1, ECL_NanoLZ_FastParams* prealloc2); 196 | 197 | #endif 198 | 199 | /* 200 | Decompresses exactly 'dst_size' bytes to 'dst' from compressed 'src' stream containing 'src_size' bytes. 201 | Function returns: amount of bytes in uncompressed stream (which is 'dst_size') if decompression succeeded, or 0 in case of error. 202 | Extra memory usage: 0. 203 | Usage: 204 | MyPODDataStruct my_data; 205 | ECL_usize compressed_size_limit = ECL_NANO_LZ_GET_BOUND(sizeof(my_data)); 206 | uint8_t* compressed_stream = (uint8_t*)malloc( compressed_size_limit ); 207 | ECL_usize compressed_size = ECL_NanoLZ_Compress_slow(ECL_NANOLZ_SCHEME1, (const uint8_t*)&my_data, sizeof(my_data), compressed_stream, compressed_size_limit, 30); 208 | // <- transferring 'compressed_size' bytes of 'compressed_stream' to receiver side 209 | ECL_usize uncompressed_size = ECL_NanoLZ_Decompress(ECL_NANOLZ_SCHEME1, compressed_stream, compressed_size, (uint8_t*)&my_data, sizeof(my_data)); 210 | if(uncompressed_size != sizeof(my_data)) { 211 | // failed 212 | } 213 | */ 214 | ECL_EXPORTED_API ECL_usize ECL_NanoLZ_Decompress(ECL_NanoLZ_Scheme scheme, const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size); 215 | 216 | #ifdef __cplusplus 217 | } 218 | #endif 219 | 220 | #endif 221 | -------------------------------------------------------------------------------- /ECL_NanoLZ_schemes_inline.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 - 2018 Evgeniy Evstratov 3 | * 4 | * Permission is hereby granted, free of charge, to any person 5 | * obtaining a copy of this software and associated documentation 6 | * files (the "Software"), to deal in the Software without 7 | * restriction, including without limitation the rights to use, 8 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the 10 | * Software is furnished to do so, subject to the following 11 | * conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be 14 | * included in all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 20 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 21 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 23 | * OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | /* schemes inline file */ 27 | 28 | /* scheme 1 coder & decoder -------------------------------------------------------------------------------------- */ 29 | #if ECL_NANO_LZ_IS_SCHEME_ENABLED(1) 30 | 31 | static bool ECL_NanoLZ_Write_Scheme1_copy(ECL_NanoLZ_CompressorState* state) { 32 | ECL_usize estimate_bits, offset_sub; 33 | ECL_ASSERT(state->n_copy >= 2); 34 | 35 | if(state->n_copy == 2) { 36 | if((state->n_new <= 3) && (state->offset <= 8)) { /* opcode 0 */ 37 | /* commented-out lines are replaced with single call: */ 38 | /*ECL_JH_Write(&state->stream, 0, 3); */ 39 | /*ECL_JH_Write(&state->stream, state->n_new, 2); */ 40 | /*ECL_JH_Write(&state->stream, state->offset - 1, 3); */ 41 | ECL_JH_Write(&state->stream, (uint8_t)((state->n_new << 3) | ((state->offset - 1) << 5)), 8); 42 | return true; 43 | } else if((state->n_new <= 8) && (state->offset <= 32)) { /* opcode 1 */ 44 | if(state->n_new) { 45 | /*ECL_JH_Write(&state->stream, 1, 4); */ 46 | /*ECL_JH_Write(&state->stream, state->n_new - 1, 3); */ 47 | ECL_JH_Write(&state->stream, (uint8_t)((state->n_new << 4) + 0xF1), 7); 48 | } else { 49 | ECL_JH_Write(&state->stream, 0x9, 4); 50 | } 51 | ECL_JH_Write(&state->stream, (uint8_t)(state->offset - 1), 5); 52 | return true; 53 | } else if((state->n_new <= 8) && (state->offset <= (32 + 128))) { /* opcode 2 */ 54 | if(state->n_new) { 55 | /*ECL_JH_Write(&state->stream, 2, 4); */ 56 | /*ECL_JH_Write(&state->stream, state->n_new - 1, 3); */ 57 | ECL_JH_Write(&state->stream, (uint8_t)((state->n_new << 4) + 0xF2), 7); 58 | } else { 59 | ECL_JH_Write(&state->stream, 0xA, 4); 60 | } 61 | ECL_JH_Write(&state->stream, (uint8_t)(state->offset - 1 - 32), 7); 62 | return true; 63 | } else { 64 | return false; 65 | } 66 | } else { 67 | offset_sub = 1; 68 | if((state->n_copy <= 10) && (state->n_new <= 8)) { 69 | if(state->offset <= 8) { /* opcode 3 */ 70 | if(state->n_new) { 71 | /*ECL_JH_Write(&state->stream, 3, 4); */ 72 | /*ECL_JH_Write(&state->stream, state->n_new - 1, 3); */ 73 | ECL_JH_Write(&state->stream, (uint8_t)((state->n_new << 4) + 0xF3), 7); 74 | } else { 75 | ECL_JH_Write(&state->stream, 0xB, 4); 76 | } 77 | /*ECL_JH_Write(&state->stream, state->n_copy - 3, 3); */ 78 | /*ECL_JH_Write(&state->stream, state->offset - 1, 3); */ 79 | ECL_JH_Write(&state->stream, (uint8_t)((state->n_copy - 3) | ((state->offset - 1) << 3)), 6); 80 | return true; 81 | } else if(state->offset <= (8 + 32)) { /* opcode 4 */ 82 | if(state->n_new) { 83 | /*ECL_JH_Write(&state->stream, 4, 4); */ 84 | /*ECL_JH_Write(&state->stream, state->n_new - 1, 3); */ 85 | ECL_JH_Write(&state->stream, (uint8_t)((state->n_new << 4) + 0xF4), 7); 86 | } else { 87 | ECL_JH_Write(&state->stream, 0xC, 4); 88 | } 89 | /*ECL_JH_Write(&state->stream, state->n_copy - 3, 3); */ 90 | /*ECL_JH_Write(&state->stream, state->offset - 1 - 8, 5); */ 91 | ECL_JH_Write(&state->stream, (uint8_t)((state->n_copy - 3) | ((state->offset - 1 - 8) << 3)), 8); 92 | return true; 93 | } else if(state->offset <= (8 + 32 + 64)) { /* opcode 5 */ 94 | if(state->n_new) { 95 | /*ECL_JH_Write(&state->stream, 5, 4); */ 96 | /*ECL_JH_Write(&state->stream, state->n_new - 1, 3); */ 97 | ECL_JH_Write(&state->stream, (uint8_t)((state->n_new << 4) + 0xF5), 7); 98 | } else { 99 | ECL_JH_Write(&state->stream, 0xD, 4); 100 | } 101 | ECL_JH_Write(&state->stream, (uint8_t)(state->n_copy - 3), 3); 102 | ECL_JH_Write(&state->stream, (uint8_t)(state->offset - 1 - 8 - 32), 6); 103 | return true; 104 | } else if(state->offset <= (8 + 32 + 64 + 256)) { /* opcode 6 */ 105 | if(state->n_new) { 106 | /*ECL_JH_Write(&state->stream, 6, 4); */ 107 | /*ECL_JH_Write(&state->stream, state->n_new - 1, 3); */ 108 | ECL_JH_Write(&state->stream, (uint8_t)((state->n_new << 4) + 0xF6), 7); 109 | } else { 110 | ECL_JH_Write(&state->stream, 0xE, 4); 111 | } 112 | ECL_JH_Write(&state->stream, (uint8_t)(state->n_copy - 3), 3); 113 | ECL_JH_Write(&state->stream, (uint8_t)(state->offset - 1 - 8 - 32 - 64), 8); 114 | return true; 115 | } else { 116 | offset_sub = 1 + 8 + 32 + 64 + 256; 117 | } 118 | } 119 | } 120 | estimate_bits = (state->n_new ? ECL_Evaluate_E2(state->n_new - 1) : 0) 121 | + ECL_Evaluate_E3(state->n_copy - 2) 122 | + ECL_Evaluate_E6E3(state->offset - offset_sub) 123 | + (3 + 1); 124 | if(estimate_bits <= (state->n_copy << 3)) { 125 | if(! state->n_new) { 126 | ECL_JH_Write(&state->stream, 0x0F, 4); 127 | } else { 128 | ECL_JH_Write(&state->stream, 0x07, 4); 129 | ECL_JH_Write_E2(&state->stream, state->n_new - 1); 130 | } 131 | ECL_JH_Write_E3(&state->stream, state->n_copy - 2); 132 | ECL_JH_Write_E6E3(&state->stream, state->offset - offset_sub); 133 | return true; 134 | } 135 | return false; 136 | } 137 | 138 | static bool ECL_NanoLZ_Write_Scheme1_nocopy(ECL_NanoLZ_CompressorState* state) { 139 | ECL_ASSERT(! state->n_copy); 140 | if(! state->n_new) { 141 | ECL_JH_Write(&state->stream, 0x0F, 8); /* glued {111, 1, (0000 = 0 in E3 format)} */ 142 | /* stream alignment opcode for flow mode (reserved for possible addition of streaming modes) */ 143 | ECL_JH_Write(&state->stream, 0, state->stream.n_bits); 144 | } else { 145 | ECL_JH_Write(&state->stream, 0x07, 4); 146 | ECL_JH_Write_E2(&state->stream, state->n_new - 1); 147 | ECL_JH_Write(&state->stream, 0, 4); /* 0 in E3 format */ 148 | } 149 | return true; 150 | } 151 | 152 | static void ECL_NanoLZ_Read_Scheme1(ECL_NanoLZ_DecompressorState* state) { 153 | uint8_t tmp; 154 | ECL_SCOPED_CONST uint8_t opcode = ECL_JH_Read(&state->stream, 3); 155 | ECL_NANO_LZ_COUNTER_APPEND(opcode, 1) 156 | switch (opcode) { 157 | case 0: 158 | tmp = ECL_JH_Read(&state->stream, 5); 159 | state->n_new = tmp & 0x03; 160 | state->n_copy = 2; 161 | state->offset = (tmp >> 2); 162 | break; 163 | case 1: 164 | state->n_new = ECL_JH_Read(&state->stream, 1) ? 0 : (ECL_JH_Read(&state->stream, 3) + 1); 165 | state->n_copy = 2; 166 | state->offset = ECL_JH_Read(&state->stream, 5); 167 | break; 168 | case 2: 169 | state->n_new = ECL_JH_Read(&state->stream, 1) ? 0 : (ECL_JH_Read(&state->stream, 3) + 1); 170 | state->n_copy = 2; 171 | state->offset = ECL_JH_Read(&state->stream, 7) + 32; 172 | break; 173 | case 3: 174 | state->n_new = ECL_JH_Read(&state->stream, 1) ? 0 : (ECL_JH_Read(&state->stream, 3) + 1); 175 | tmp = ECL_JH_Read(&state->stream, 6); 176 | state->n_copy = (tmp & 0x07) + 3; 177 | state->offset = (tmp >> 3); 178 | break; 179 | case 4: 180 | state->n_new = ECL_JH_Read(&state->stream, 1) ? 0 : (ECL_JH_Read(&state->stream, 3) + 1); 181 | tmp = ECL_JH_Read(&state->stream, 8); 182 | state->n_copy = (tmp & 0x07) + 3; 183 | state->offset = (tmp >> 3) + 8; 184 | break; 185 | case 5: 186 | state->n_new = ECL_JH_Read(&state->stream, 1) ? 0 : (ECL_JH_Read(&state->stream, 3) + 1); 187 | state->n_copy = ECL_JH_Read(&state->stream, 3) + 3; 188 | state->offset = ECL_JH_Read(&state->stream, 6) + 8 + 32; 189 | break; 190 | case 6: 191 | state->n_new = ECL_JH_Read(&state->stream, 1) ? 0 : (ECL_JH_Read(&state->stream, 3) + 1); 192 | state->n_copy = ECL_JH_Read(&state->stream, 3) + 3; 193 | state->offset = ECL_JH_Read(&state->stream, 8) + 8 + 32 + 64; 194 | break; 195 | case 7: 196 | state->n_new = ECL_JH_Read(&state->stream, 1) ? 0 : (ECL_JH_Read_E2(&state->stream) + 1); 197 | state->n_copy = ECL_JH_Read_E3(&state->stream); 198 | if(state->n_copy) { 199 | state->n_copy += 2; 200 | state->offset = ECL_JH_Read_E6E3(&state->stream); 201 | if((state->n_copy <= 10) && (state->n_new <= 8)) { 202 | state->offset += 8 + 32 + 64 + 256; 203 | } 204 | } 205 | break; 206 | } 207 | ++(state->offset); 208 | ECL_NANO_LZ_COUNTER_APPEND(8, state->n_new) 209 | } 210 | 211 | #endif 212 | /* scheme 2 coder & decoder -------------------------------------------------------------------------------------- */ 213 | #if ECL_NANO_LZ_IS_SCHEME_ENABLED(2) 214 | 215 | static bool ECL_NanoLZ_Write_Scheme2_copy(ECL_NanoLZ_CompressorState* state) { 216 | ECL_ASSERT(state->n_copy >= 2); 217 | if(state->n_copy >= 5) { /* ignore smaller match - this weakens compression and increases decompression speed */ 218 | /* we estimated roughly - in some cases it can result in compressed output > original data */ 219 | ECL_JH_Write_E3(&state->stream, state->n_new); 220 | ECL_JH_Write_E3(&state->stream, state->n_copy - 4); 221 | ECL_JH_Write_E7(&state->stream, state->offset - 1); 222 | return true; 223 | } 224 | return false; 225 | } 226 | 227 | static bool ECL_NanoLZ_Write_Scheme2_nocopy(ECL_NanoLZ_CompressorState* state) { 228 | ECL_ASSERT(! state->n_copy); 229 | ECL_JH_Write_E3(&state->stream, state->n_new); 230 | ECL_JH_Write_E3(&state->stream, 0); /* n_copy = 0 */ 231 | return true; 232 | } 233 | 234 | static void ECL_NanoLZ_Read_Scheme2(ECL_NanoLZ_DecompressorState* state) { 235 | state->n_new = ECL_JH_Read_E3(&state->stream); 236 | state->n_copy = ECL_JH_Read_E3(&state->stream); 237 | if(state->n_copy) { /* 0 is 0 and assumes no offset encoded (last block in stream) */ 238 | state->n_copy += 4; 239 | state->offset = ECL_JH_Read_E7(&state->stream) + 1; 240 | } 241 | } 242 | 243 | #endif 244 | /* --------------------------------------------------------------------------------------------------------------- */ 245 | 246 | /* returns a coder that expects only sequences with n_copy >= 2 */ 247 | static ECL_NanoLZ_SchemeCoder ECL_NanoLZ_GetSchemeCoder(ECL_NanoLZ_Scheme scheme) { 248 | switch(scheme) { 249 | #if ECL_NANO_LZ_IS_SCHEME_ENABLED(1) 250 | case ECL_NANOLZ_SCHEME1: return ECL_NanoLZ_Write_Scheme1_copy; 251 | #endif 252 | #if ECL_NANO_LZ_IS_SCHEME_ENABLED(2) 253 | case ECL_NANOLZ_SCHEME2_DEMO: return ECL_NanoLZ_Write_Scheme2_copy; 254 | #endif 255 | default: break; 256 | } 257 | return 0; 258 | } 259 | 260 | /* returns a coder that expects only sequences with n_copy == 0 */ 261 | static ECL_NanoLZ_SchemeCoder ECL_NanoLZ_GetSchemeCoderNoCopy(ECL_NanoLZ_Scheme scheme) { 262 | switch(scheme) { 263 | #if ECL_NANO_LZ_IS_SCHEME_ENABLED(1) 264 | case ECL_NANOLZ_SCHEME1: return ECL_NanoLZ_Write_Scheme1_nocopy; 265 | #endif 266 | #if ECL_NANO_LZ_IS_SCHEME_ENABLED(2) 267 | case ECL_NANOLZ_SCHEME2_DEMO: return ECL_NanoLZ_Write_Scheme2_nocopy; 268 | #endif 269 | default: break; 270 | } 271 | return 0; 272 | } 273 | 274 | /* returns a decoder */ 275 | static ECL_NanoLZ_SchemeDecoder ECL_NanoLZ_GetSchemeDecoder(ECL_NanoLZ_Scheme scheme) { 276 | switch(scheme) { 277 | #if ECL_NANO_LZ_IS_SCHEME_ENABLED(1) 278 | case ECL_NANOLZ_SCHEME1: return ECL_NanoLZ_Read_Scheme1; 279 | #endif 280 | #if ECL_NANO_LZ_IS_SCHEME_ENABLED(2) 281 | case ECL_NANOLZ_SCHEME2_DEMO: return ECL_NanoLZ_Read_Scheme2; 282 | #endif 283 | default: break; 284 | } 285 | return 0; 286 | } 287 | -------------------------------------------------------------------------------- /ECL_ZeroDevourer.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 - 2018 Evgeniy Evstratov 3 | * 4 | * Permission is hereby granted, free of charge, to any person 5 | * obtaining a copy of this software and associated documentation 6 | * files (the "Software"), to deal in the Software without 7 | * restriction, including without limitation the rights to use, 8 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the 10 | * Software is furnished to do so, subject to the following 11 | * conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be 14 | * included in all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 20 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 21 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 23 | * OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | #include "ECL_ZeroDevourer.h" 27 | #include "ECL_JH_States.h" 28 | 29 | #include 30 | 31 | static void ECL_ZeroDevourer_DumpSeq100(ECL_JH_WState* state, const uint8_t* src, ECL_usize cnt_x) { 32 | uint8_t* dst; 33 | ECL_ASSERT((cnt_x >= 1) && (cnt_x <= 4)); 34 | /* next line is equal to "ECL_JH_Write(state, ((cnt_x - 1) << 3) + 0x01, 5);" 35 | and equal to "ECL_JH_Write(state, 0x01, 3); ECL_JH_Write(state, cnt_x - 1, 2);" 36 | */ 37 | ECL_JH_Write(state, (uint8_t)((cnt_x << 3) + 0xF9), 5); 38 | dst = state->next; 39 | ECL_JH_WJump(state, cnt_x); 40 | if(state->is_valid) { 41 | ECL_usize i; 42 | for(i = 0; i < cnt_x; ++i) { 43 | dst[i] = src[i]; 44 | } 45 | } 46 | } 47 | 48 | static void ECL_ZeroDevourer_DumpSeq101(ECL_JH_WState* state, const uint8_t* src, ECL_usize cnt_x) { 49 | uint8_t* dst; 50 | ECL_ASSERT((cnt_x >= 5) && (cnt_x <= 20)); 51 | /* next line is equal to "ECL_JH_Write(state, ((cnt_x - 5) << 3) + 0x03, 7);" 52 | and equal to "ECL_JH_Write(state, 0x03, 3); ECL_JH_Write(state, cnt_x - 5, 4);" 53 | */ 54 | ECL_JH_Write(state, (uint8_t)((cnt_x << 3) + 0xDB), 7); 55 | dst = state->next; 56 | ECL_JH_WJump(state, cnt_x); 57 | if(state->is_valid) { 58 | memcpy(dst, src, cnt_x); 59 | } 60 | } 61 | 62 | static void ECL_ZeroDevourer_DumpSeq110(ECL_JH_WState* state, ECL_usize cnt_0) { 63 | ECL_ASSERT(cnt_0 >= 9); 64 | if(cnt_0 < 25) { 65 | /* next line is equal to "ECL_JH_Write(state, ((cnt_0 - 9) << 3) | 0x05, 8);" 66 | which is valid replacement for 'else' branch for 'cnt_0 < 25' 67 | */ 68 | ECL_JH_Write(state, (uint8_t)((cnt_0 << 3) + 0xBD), 8); 69 | } else { 70 | ECL_JH_Write(state, 0x05, 3); 71 | ECL_JH_Write_E4(state, cnt_0 - 9); 72 | } 73 | } 74 | 75 | static void ECL_ZeroDevourer_DumpSeq111(ECL_JH_WState* state, const uint8_t* src, ECL_usize cnt_x) { 76 | uint8_t* dst; 77 | ECL_ASSERT(cnt_x >= 1); 78 | ECL_JH_Write(state, 0x07, 3); 79 | ECL_JH_Write_E6E3(state, cnt_x - 1); 80 | dst = state->next; 81 | ECL_JH_WJump(state, cnt_x); 82 | if(state->is_valid) { 83 | memcpy(dst, src, cnt_x); 84 | } 85 | } 86 | 87 | static void ECL_ZeroDevourer_DumpZeroGeneric(ECL_JH_WState* state, ECL_usize cnt_0) { 88 | /* cnt_0 > 0 */ 89 | if(cnt_0 >= 9) { 90 | ECL_ZeroDevourer_DumpSeq110(state, cnt_0); 91 | } else { 92 | ECL_JH_Write(state, 0, (uint8_t)cnt_0); 93 | } 94 | } 95 | 96 | static void ECL_ZeroDevourer_DumpGeneric(ECL_JH_WState* state, const uint8_t* src, ECL_usize cnt_x, ECL_usize cnt_0) { 97 | /* cnt_x + cnt_0 > 0 */ 98 | if(! cnt_x) { 99 | ECL_ZeroDevourer_DumpZeroGeneric(state, cnt_0); 100 | } else if(! cnt_0) { 101 | ECL_ZeroDevourer_DumpSeq111(state, src, cnt_x); 102 | } else { 103 | if(cnt_x <= 4) { 104 | ECL_ZeroDevourer_DumpSeq100(state, src, cnt_x); 105 | --cnt_0; 106 | } else if(cnt_x <= 20) { 107 | ECL_ZeroDevourer_DumpSeq101(state, src, cnt_x); 108 | --cnt_0; 109 | } else { 110 | ECL_ZeroDevourer_DumpSeq111(state, src, cnt_x); 111 | } 112 | if(cnt_0) { 113 | ECL_ZeroDevourer_DumpZeroGeneric(state, cnt_0); 114 | } 115 | } 116 | } 117 | 118 | static bool ECL_ZeroDevourer_IsWorth(ECL_usize cnt_x, ECL_usize cnt_0) { 119 | ECL_usize bits_needed; 120 | if(cnt_x <= 20) { 121 | return true; 122 | } else if(cnt_x <= (1U << 9)) { 123 | return cnt_0 >= 2; 124 | } else if(cnt_x <= (1U << 12)) { 125 | return cnt_0 >= 3; 126 | #ifndef ECL_USE_BITNESS_16 127 | } else if(cnt_x <= (1U << 18)) { 128 | #else 129 | } else { 130 | #endif 131 | return cnt_0 >= 4; 132 | } 133 | bits_needed = 3 + ECL_Evaluate_E6E3(cnt_x - 1); 134 | if(cnt_0 < 9) { 135 | bits_needed += cnt_0; 136 | } else if (cnt_0 < 25) { 137 | bits_needed += 8; 138 | } else { 139 | bits_needed += 3 + ECL_Evaluate_E4(cnt_0 - 9); 140 | } 141 | return bits_needed <= (cnt_0 << 3); 142 | } 143 | 144 | ECL_usize ECL_ZeroDevourer_Compress(const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size) { 145 | ECL_JH_WState state; 146 | const uint8_t* first_undone; 147 | const uint8_t* first_x; 148 | const uint8_t* ECL_SCOPED_CONST src_end = src + src_size; 149 | 150 | ECL_JH_WInit(&state, dst, dst_size, 0); 151 | if((! src) || (! src_size) || (! state.is_valid)) { 152 | return 0; 153 | } 154 | first_undone = src; 155 | first_x = src; 156 | while((first_undone < src_end) && (state.is_valid)) { 157 | ECL_usize cnt_x, cnt_0; 158 | const uint8_t* first_0; 159 | for(first_0 = first_x; (first_0 < src_end) && *first_0; ++first_0); /* search for zero from where last search ended */ 160 | 161 | cnt_x = first_0 - first_undone; /* count of non-zeroes in beginning */ 162 | if(first_0 == src_end) { /* complete it (1) */ 163 | ECL_ZeroDevourer_DumpSeq111(&state, first_undone, cnt_x); 164 | break; 165 | } 166 | /* we found zero, find next non-zero */ 167 | for(first_x = first_0; (first_x < src_end) && (! *first_x); ++first_x); 168 | 169 | cnt_0 = first_x - first_0; /* count of zeroes afterwards */ 170 | /* stream looks like {first_undone: [xx..x] first_0: [00..0] first_x: ...} */ 171 | if(ECL_ZeroDevourer_IsWorth(cnt_x, cnt_0)) { 172 | ECL_ZeroDevourer_DumpGeneric(&state, first_undone, cnt_x, cnt_0); 173 | first_undone = first_x; 174 | } 175 | } 176 | if(! state.is_valid) { 177 | return 0; 178 | } 179 | return state.next - dst; 180 | } 181 | 182 | ECL_usize ECL_ZeroDevourer_Decompress(const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size) { 183 | ECL_JH_RState state; 184 | const uint8_t* dst_start = dst; 185 | const uint8_t* dst_end = dst + dst_size; 186 | ECL_JH_RInit(&state, src, src_size, 0); 187 | if((! dst) || (! state.is_valid)) { 188 | return 0; 189 | } 190 | for(;;) { 191 | ECL_usize cnt_x, cnt_0; 192 | ECL_SCOPED_CONST ECL_usize left = dst_end - dst; 193 | if(! left) { 194 | break; 195 | } 196 | if(! ECL_JH_Read(&state, 1)) { 197 | if(state.is_valid) { 198 | *dst = 0; 199 | ++dst; 200 | } else { 201 | return 0; 202 | } 203 | } else { /* wide codes */ 204 | switch (ECL_JH_Read(&state, 2)) { 205 | case 0: /* seq 1.00 */ 206 | cnt_x = ECL_JH_Read(&state, 2) + 1; 207 | { 208 | const uint8_t* ECL_SCOPED_CONST src_block_start = state.next; 209 | ECL_usize i = 0; 210 | ECL_JH_RJump(&state, cnt_x); 211 | if(state.is_valid && (cnt_x < left)) { 212 | dst[cnt_x] = 0; 213 | for(; i < cnt_x; ++i) { 214 | dst[i] = src_block_start[i]; 215 | } 216 | dst += cnt_x + 1; 217 | } else { 218 | return 0; 219 | } 220 | } 221 | break; 222 | case 1: /* seq 1.01 */ 223 | cnt_x = ECL_JH_Read(&state, 4) + 5; 224 | { 225 | const uint8_t* ECL_SCOPED_CONST src_block_start = state.next; 226 | ECL_JH_RJump(&state, cnt_x); 227 | if(state.is_valid && (cnt_x < left)) { 228 | dst[cnt_x] = 0; 229 | memcpy(dst, src_block_start, cnt_x); 230 | dst += cnt_x + 1; 231 | } else { 232 | return 0; 233 | } 234 | } 235 | break; 236 | case 2: /* seq 1.10 */ 237 | cnt_0 = ECL_JH_Read_E4(&state) + 9; 238 | if(state.is_valid && (cnt_0 <= left)) { 239 | memset(dst, 0, cnt_0); 240 | dst += cnt_0; 241 | } else { 242 | return 0; 243 | } 244 | break; 245 | case 3: /* seq 1.11 */ 246 | cnt_x = ECL_JH_Read_E6E3(&state) + 1; 247 | { 248 | const uint8_t* ECL_SCOPED_CONST src_block_start = state.next; 249 | ECL_JH_RJump(&state, cnt_x); 250 | if(state.is_valid && (cnt_x <= left)) { 251 | memcpy(dst, src_block_start, cnt_x); 252 | dst += cnt_x; 253 | } else { 254 | return 0; 255 | } 256 | } 257 | break; 258 | } 259 | } 260 | } 261 | return ((state.next == state.end) && state.is_valid) ? (dst - dst_start) : 0; 262 | } 263 | -------------------------------------------------------------------------------- /ECL_ZeroDevourer.h: -------------------------------------------------------------------------------- 1 | #ifndef ECL_ZERO_DEVOURER_H_ 2 | #define ECL_ZERO_DEVOURER_H_ 3 | 4 | #include "ECL_config.h" 5 | 6 | /* 7 | Calculates size of buffer to fit compressed version of any data of 'src_size' size. 8 | */ 9 | #define ECL_ZERO_DEVOURER_GET_BOUND(src_size) ((src_size) + 2 * sizeof(ECL_usize)) 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | /* 16 | Compresses 'src_size' bytes starting at 'src' to destination 'dst' that can hold at most 'dst_size' bytes. 17 | Function returns amount of bytes in resulted compressed stream, or 0 in case of error. 18 | To find enough size for output buffer: 19 | - use maximum size: dst_size = ECL_ZERO_DEVOURER_GET_BOUND(src_size); 20 | See full compress/decompress example usage near decompression function. 21 | */ 22 | ECL_EXPORTED_API ECL_usize ECL_ZeroDevourer_Compress(const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size); 23 | 24 | /* 25 | Decompresses exactly 'dst_size' bytes to 'dst' from compressed 'src' stream containing 'src_size' bytes. 26 | Function returns: amount of bytes in uncompressed stream (which is 'dst_size') if decompression succeeded, or 0 in case of error. 27 | Usage: 28 | MyPODDataStruct my_data; 29 | const uint8_t* src = (const uint8_t*)&my_data; 30 | ECL_usize src_size = sizeof(my_data); 31 | ECL_usize compressed_size_limit = ECL_ZERO_DEVOURER_GET_BOUND(src_size); 32 | uint8_t* compressed = (uint8_t*)malloc( compressed_size_limit ); 33 | ECL_usize compressed_size = ECL_ZeroDevourer_Compress(src, src_size, compressed, compressed_size_limit); 34 | // ... <- transferring 'compressed_size' bytes of 'compressed' to receiver side 35 | ECL_usize uncompressed_size = ECL_ZeroDevourer_Decompress(compressed, compressed_size, (uint8_t*)&my_data, sizeof(my_data)); 36 | if(uncompressed_size != sizeof(my_data)) { 37 | // failed 38 | } 39 | */ 40 | ECL_EXPORTED_API ECL_usize ECL_ZeroDevourer_Decompress(const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size); 41 | 42 | #ifdef __cplusplus 43 | } 44 | #endif 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /ECL_ZeroEater.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 - 2018 Evgeniy Evstratov 3 | * 4 | * Permission is hereby granted, free of charge, to any person 5 | * obtaining a copy of this software and associated documentation 6 | * files (the "Software"), to deal in the Software without 7 | * restriction, including without limitation the rights to use, 8 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the 10 | * Software is furnished to do so, subject to the following 11 | * conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be 14 | * included in all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 20 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 21 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 23 | * OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | #include "ECL_ZeroEater.h" 27 | 28 | typedef struct { 29 | const uint8_t* src; 30 | uint8_t* dst; 31 | uint8_t* dst_end; 32 | ECL_usize result_size; 33 | } ECL_ZeroEaterComprssorState; 34 | 35 | static void ECL_ZeroEater_DumpSeq1(ECL_ZeroEaterComprssorState* state, ECL_usize count_x) { 36 | ECL_usize full_blocks, part_block_cnt, length; 37 | full_blocks = count_x >> 7; 38 | part_block_cnt = count_x & 0x007F; 39 | length = full_blocks; 40 | if(part_block_cnt) { 41 | ++length; 42 | } 43 | length += count_x; 44 | state->result_size += length; 45 | if((state->dst + length) <= state->dst_end) { 46 | for(; full_blocks > 0; --full_blocks) { 47 | *(state->dst) = 0x7F; /* opcode */ 48 | memcpy(state->dst + 1, state->src, 128); 49 | state->src += 128; 50 | state->dst += 1 + 128; 51 | } 52 | if(part_block_cnt) { 53 | *(state->dst) = (uint8_t)part_block_cnt - 1; /* opcode */ 54 | memcpy(state->dst + 1, state->src, part_block_cnt); 55 | state->src += part_block_cnt; 56 | state->dst += 1 + part_block_cnt; 57 | } 58 | } 59 | } 60 | 61 | static void ECL_ZeroEater_DumpSeq2(ECL_ZeroEaterComprssorState* state, ECL_usize count_0) { 62 | ECL_usize full_blocks, part_block_cnt, length; 63 | full_blocks = count_0 >> 6; 64 | part_block_cnt = count_0 & 0x003F; 65 | length = full_blocks; 66 | if(part_block_cnt) { 67 | ++length; 68 | } 69 | state->result_size += length; 70 | if((state->dst + length) <= state->dst_end) { 71 | for(; full_blocks > 0; --full_blocks) { 72 | *(state->dst) = 0xBF; /* opcode */ 73 | ++(state->dst); 74 | } 75 | if(part_block_cnt) { 76 | *(state->dst) = (uint8_t)part_block_cnt - 0x81; /* subtract 1, subtract extra 0x80 to get 1 in higher bit */ 77 | ++(state->dst); 78 | } 79 | } 80 | } 81 | 82 | static void ECL_ZeroEater_DumpSeq3(ECL_ZeroEaterComprssorState* state, ECL_usize count_x, ECL_usize count_0) { 83 | /* count_x = [1..8]; count_0 = [1..8]; */ 84 | ECL_SCOPED_CONST ECL_usize length = count_x + 1; 85 | state->result_size += length; 86 | if((state->dst + length) <= state->dst_end) { 87 | *(state->dst) = ((uint8_t)count_0 - (uint8_t)0x41) | (((uint8_t)count_x - (uint8_t)1) << 3); /* subtract extra 0x40 to get 11 in higher bits */ 88 | memcpy(state->dst + 1, state->src, count_x); 89 | state->src += count_x; 90 | state->dst += 1 + count_x; 91 | } 92 | } 93 | 94 | static void ECL_ZeroEater_DumpGeneric(ECL_ZeroEaterComprssorState* state, ECL_usize count_x, ECL_usize count_0) { 95 | /* count_x > 0; count_0 > 0 */ 96 | ECL_SCOPED_CONST ECL_usize x_full_blocks = count_x >> 7; 97 | if(x_full_blocks) { 98 | ECL_SCOPED_CONST ECL_usize tmp = count_x & 0x7F; /* last block size */ 99 | ECL_ZeroEater_DumpSeq1(state, count_x - tmp); 100 | count_x = tmp; 101 | } 102 | if(count_x && (count_x < 9) && (count_0 < 9)) { 103 | ECL_ZeroEater_DumpSeq3(state, count_x, count_0); 104 | } else if(count_x && (count_x < 9)) { 105 | ECL_ZeroEater_DumpSeq3(state, count_x, 8); 106 | ECL_ZeroEater_DumpSeq2(state, count_0 - 8); 107 | } else { 108 | if(count_x) { 109 | ECL_ZeroEater_DumpSeq1(state, count_x); 110 | } 111 | ECL_ZeroEater_DumpSeq2(state, count_0); 112 | } 113 | } 114 | 115 | ECL_usize ECL_ZeroEater_Compress(const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size) { 116 | ECL_ZeroEaterComprssorState state; 117 | ECL_usize cnt_x, cnt_0; 118 | const uint8_t* first_x; 119 | const uint8_t* first_0; 120 | const uint8_t* ECL_SCOPED_CONST src_end = src + src_size; 121 | if(! src) { 122 | return 0; 123 | } 124 | if(! dst) { 125 | dst_size = 0; 126 | } 127 | state.src = src; 128 | state.result_size = 0; 129 | state.dst = dst; 130 | state.dst_end = dst ? (dst + dst_size) : dst; 131 | first_x = src; 132 | while(state.src < src_end) { 133 | for(first_0 = first_x; (first_0 < src_end) && *first_0; ++first_0); /* search for zero from where last search ended */ 134 | 135 | cnt_x = first_0 - state.src; /* count of non-zeroes in beginning */ 136 | if(first_0 == src_end) { /* complete it (1) */ 137 | ECL_ZeroEater_DumpSeq1(&state, cnt_x); 138 | break; 139 | } 140 | /* we found zero, find next non-zero */ 141 | for(first_x = first_0; (first_x < src_end) && (! *first_x); ++first_x); 142 | 143 | cnt_0 = first_x - first_0; /* count of zeroes afterwards */ 144 | /* stream looks like {state.src: [xx..x] first_0: [00..0] first_x: ...} */ 145 | if(cnt_x) { /* has non-zero stream too (3) */ 146 | if((cnt_0 == 1) && (cnt_x > 8)) { /* bad deal */ 147 | /* this particular case is the worst, this single zero-byte should be considered as non-zero for better compression */ 148 | /* this data will be processed in next iteration */ 149 | continue; 150 | } 151 | ECL_ZeroEater_DumpGeneric(&state, cnt_x, cnt_0); 152 | state.src = first_x; 153 | } else { /* only zero stream (2) */ 154 | ECL_ZeroEater_DumpSeq2(&state, cnt_0); 155 | state.src = first_x; 156 | } 157 | } 158 | return state.result_size; 159 | } 160 | 161 | ECL_usize ECL_ZeroEater_Decompress(const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size) { 162 | const uint8_t* src_end = src + src_size; 163 | const uint8_t* dst_start = dst; 164 | const uint8_t* dst_end = dst + dst_size; 165 | if((! src) || (! dst)) { /* invalid parameters */ 166 | return 0; 167 | } 168 | for(; src < src_end; ) { 169 | ECL_usize cnt_x, cnt_0; 170 | ECL_SCOPED_CONST uint8_t opcode = *src; 171 | ++src; 172 | if(opcode & 0x80) { /* method 2 or 3 */ 173 | if(opcode & 0x40) { /* method 3 */ 174 | cnt_x = ((opcode >> 3) & 0x07) + 1; 175 | cnt_0 = (opcode & 0x07) + 1; 176 | /* check if fits output buffer and unpack */ 177 | if((cnt_x + cnt_0) > (ECL_usize)(dst_end - dst)) { 178 | break; 179 | } 180 | if(cnt_x > (ECL_usize)(src_end - src)) { 181 | break; /* invalid stream */ 182 | } 183 | memcpy(dst, src, cnt_x); 184 | memset(dst + cnt_x, 0, cnt_0); 185 | dst += cnt_x + cnt_0; 186 | src += cnt_x; 187 | } else { /* method 2 */ 188 | cnt_0 = (opcode & 0x3F) + 1; 189 | if(cnt_0 > (ECL_usize)(dst_end - dst)) { 190 | return 0; 191 | } 192 | memset(dst, 0, cnt_0); 193 | dst += cnt_0; 194 | } 195 | } else { /* method 1 */ 196 | cnt_x = opcode + 1; 197 | if(cnt_x > (ECL_usize)(dst_end - dst)) { 198 | break; 199 | } 200 | if(cnt_x > (ECL_usize)(src_end - src)) { 201 | break; /* invalid stream */ 202 | } 203 | memcpy(dst, src, cnt_x); 204 | dst += cnt_x; 205 | src += cnt_x; 206 | } 207 | } 208 | return (src == src_end) ? (dst - dst_start) : 0; /* ensure all source is consumed */ 209 | } 210 | -------------------------------------------------------------------------------- /ECL_ZeroEater.h: -------------------------------------------------------------------------------- 1 | #ifndef ECL_ZERO_EATER_H_ 2 | #define ECL_ZERO_EATER_H_ 3 | 4 | #include "ECL_config.h" 5 | 6 | #define ECL_ZERO_EATER_GET_BOUND(src_size) ((src_size) + 1 + (src_size)/128) 7 | 8 | #ifdef __cplusplus 9 | extern "C" { 10 | #endif 11 | 12 | /* 13 | Compresses 'src_size' bytes starting at 'src' to destination 'dst' that can hold at most 'dst_size' bytes. 14 | Function returns amount of bytes in resulted compressed stream or 0 if parameters are invalid. 15 | 'dst' is allowed to be NULL for dry run, if 'dst' is not NULL and 'dst_size' is not less than returned value, then 'dst' contains valid output. 16 | To find enough size for output buffer you can: 17 | - use dry run: dst_size = ECL_ZeroEater_Compress(src, src_size, 0, 0); 18 | - use maximum size: dst_size = ECL_ZERO_EATER_GET_BOUND(src_size); 19 | See full compress/decompress example usage near decompression function. 20 | */ 21 | ECL_EXPORTED_API ECL_usize ECL_ZeroEater_Compress(const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size); 22 | 23 | /* 24 | Decompresses full 'src' stream of 'src_size' bytes to destination 'dst' holding at most 'dst_size' uncompressed bytes. 25 | Function returns: amount of bytes in uncompressed stream, or 0 in case of error. 26 | Usage: 27 | MyPODDataStruct my_data; 28 | const uint8_t* src = (const uint8_t*)&my_data; 29 | ECL_usize src_size = sizeof(my_data); 30 | ECL_usize compressed_size_limit = ECL_ZERO_EATER_GET_BOUND(src_size); 31 | uint8_t* compressed = (uint8_t*)malloc( compressed_size_limit ); 32 | ECL_usize compressed_size = ECL_ZeroEater_Compress(src, src_size, compressed, compressed_size_limit); 33 | // <- transferring 'compressed_size' bytes of 'compressed' to receiver side 34 | ECL_usize uncompressed_size = ECL_ZeroEater_Decompress(compressed, compressed_size, (uint8_t*)&my_data, sizeof(my_data)); 35 | if(uncompressed_size != sizeof(my_data)) { 36 | // failed 37 | } 38 | */ 39 | ECL_EXPORTED_API ECL_usize ECL_ZeroEater_Decompress(const uint8_t* src, ECL_usize src_size, uint8_t* dst, ECL_usize dst_size); 40 | 41 | #ifdef __cplusplus 42 | } 43 | #endif 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /ECL_common.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 - 2018 Evgeniy Evstratov 3 | * 4 | * Permission is hereby granted, free of charge, to any person 5 | * obtaining a copy of this software and associated documentation 6 | * files (the "Software"), to deal in the Software without 7 | * restriction, including without limitation the rights to use, 8 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | * copies of the Software, and to permit persons to whom the 10 | * Software is furnished to do so, subject to the following 11 | * conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be 14 | * included in all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 20 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 21 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 23 | * OTHER DEALINGS IN THE SOFTWARE. 24 | */ 25 | 26 | #include "ECL_JH_States.h" 27 | #include "ECL_utils.h" 28 | 29 | static uint8_t ECL_JH_dummy_buffer; /* dummy data storage to simplify some checks */ 30 | static const uint8_t c_bmasks8[] = {0, 1, 3, 7, 15, 31, 63, 127, 255}; 31 | 32 | /* utils part */ 33 | uint32_t ECL_GetSizeBitness(void) { 34 | return ECL_SIZE_TYPE_BITS_COUNT; 35 | } 36 | 37 | uint32_t ECL_GetVersionNumber(void) { 38 | return ECL_VERSION_NUMBER; 39 | } 40 | 41 | #define ECL_STRING_OF_HELPER(x) #x 42 | #define ECL_STRING_OF(x) ECL_STRING_OF_HELPER(x) 43 | 44 | const char* ECL_GetVersionString(void) { 45 | return ECL_STRING_OF(ECL_VERSION_MAJOR.ECL_VERSION_MINOR.ECL_VERSION_PATCH); 46 | } 47 | 48 | #undef ECL_STRING_OF_HELPER 49 | #undef ECL_STRING_OF 50 | 51 | const char* ECL_GetVersionBranch(void) { 52 | return ECL_VERSION_BRANCH; 53 | } 54 | 55 | uint8_t ECL_LogRoundUp(ECL_usize value) { 56 | ECL_usize tmp; 57 | uint8_t result; 58 | if(value < 2) { 59 | return 1; 60 | } 61 | result = sizeof(value) * 8; 62 | tmp = 1; 63 | tmp <<= result - 1; 64 | while(value <= tmp) { 65 | tmp >>= 1; 66 | --result; 67 | } 68 | return result; 69 | } 70 | 71 | uint16_t* ECL_GetAlignedPointer2(uint8_t* ptr) { 72 | ECL_ASSERT(ptr); 73 | return (uint16_t*) ( (((uintptr_t)ptr) & 1) ? (ptr + 1) : ptr); 74 | } 75 | 76 | ECL_usize* ECL_GetAlignedPointerS(uint8_t* ptr) { 77 | ECL_SCOPED_CONST int offset = ((uintptr_t)ptr) & (sizeof(ECL_usize) - 1); 78 | ECL_ASSERT(ptr); 79 | return (ECL_usize*)(offset ? (ptr + sizeof(ECL_usize) - offset) : ptr); 80 | } 81 | 82 | uint8_t* ECL_Helper_WriteE7(uint8_t* data_start, ECL_usize max_bytes, ECL_usize value) { 83 | if(data_start) { 84 | while(max_bytes) { 85 | ECL_SCOPED_CONST uint8_t e = value > 0x7F ? 0x80 : 0; 86 | *data_start = (uint8_t)value | e; 87 | value >>= 7; 88 | ++data_start; 89 | if(! e) { 90 | return data_start; 91 | } 92 | --max_bytes; 93 | } 94 | } 95 | return 0; 96 | } 97 | 98 | const uint8_t* ECL_Helper_ReadE7(const uint8_t* data_start, ECL_usize max_bytes, ECL_usize* output_value) { 99 | if(data_start && output_value) { 100 | uint8_t shift = 0; 101 | *output_value = 0; 102 | while(max_bytes) { 103 | ECL_SCOPED_CONST ECL_usize adding = *data_start & 0x7F; 104 | ECL_SCOPED_CONST uint8_t e = *data_start & 0x80; 105 | *output_value |= adding << shift; 106 | ++data_start; 107 | if(! e) { 108 | if(shift) { 109 | ECL_SCOPED_CONST uint8_t last_allowed_bits = ECL_SIZE_TYPE_BITS_COUNT - shift; 110 | if(( ((ECL_usize)1) << last_allowed_bits ) <= adding) { 111 | break; /* failed - not enough capacity */ 112 | } 113 | } 114 | return data_start; 115 | } 116 | --max_bytes; 117 | shift += 7; 118 | if(shift >= ECL_SIZE_TYPE_BITS_COUNT) { 119 | break; /* failed - not enough capacity */ 120 | } 121 | } 122 | } 123 | return 0; 124 | } 125 | 126 | 127 | /* JH part */ 128 | void ECL_JH_WInit(ECL_JH_WState* state, uint8_t* ptr, ECL_usize size, ECL_usize start) { 129 | uint8_t input_is_valid = 1; 130 | if((! ptr) || (! size)) { 131 | input_is_valid = 0; 132 | ptr = &ECL_JH_dummy_buffer; 133 | size = 1; 134 | } 135 | state->is_valid = input_is_valid * (start < size ? 1 : 0); 136 | state->end = ptr + size; 137 | if(state->is_valid) { 138 | state->byte = ptr + start; 139 | state->next = state->byte + 1; 140 | } else { 141 | state->byte = ptr; 142 | state->next = state->end; /* if invalid - next==end */ 143 | } 144 | state->n_bits = 8; 145 | *(state->byte) = 0; 146 | } 147 | 148 | void ECL_JH_RInit(ECL_JH_RState* state, const uint8_t* ptr, ECL_usize size, ECL_usize start) { 149 | uint8_t input_is_valid = 1; 150 | if((! ptr) || (! size)) { 151 | input_is_valid = 0; 152 | ptr = &ECL_JH_dummy_buffer; 153 | size = 1; 154 | } 155 | state->is_valid = input_is_valid * (start < size ? 1 : 0); 156 | state->end = ptr + size; 157 | if(state->is_valid) { 158 | state->byte = ptr + start; 159 | state->next = state->byte + 1; 160 | } else { 161 | state->byte = ptr; 162 | state->next = state->end; 163 | } 164 | state->n_bits = 8; 165 | } 166 | 167 | void ECL_JH_Write(ECL_JH_WState* state, uint8_t value, uint8_t bits) { 168 | ECL_ASSERT(bits && (bits < 9)); 169 | value &= c_bmasks8[bits]; 170 | if(bits <= state->n_bits) { /* fits easily */ 171 | *(state->byte) |= value << (8 - state->n_bits); 172 | state->n_bits -= bits; 173 | } else if(! state->n_bits) { /* to next */ 174 | if(state->next == state->end) { 175 | state->is_valid = 0; 176 | return; 177 | } 178 | state->byte = state->next; 179 | ++(state->next); 180 | *(state->byte) = value; 181 | state->n_bits = 8 - bits; 182 | } else { /* 2 parts */ 183 | *(state->byte) |= value << (8 - state->n_bits); 184 | if(state->next == state->end) { 185 | state->is_valid = 0; 186 | return; 187 | } 188 | state->byte = state->next; 189 | ++(state->next); 190 | *(state->byte) = value >> state->n_bits; 191 | state->n_bits = 8 - bits + state->n_bits; 192 | } 193 | } 194 | 195 | uint8_t ECL_JH_Read(ECL_JH_RState* state, uint8_t bits) { 196 | uint8_t res; 197 | ECL_ASSERT(bits && (bits < 9)); 198 | if(bits <= state->n_bits) { /* fits easily */ 199 | res = *(state->byte) >> (8 - state->n_bits); 200 | state->n_bits -= bits; 201 | } else if(! state->n_bits) { /* to next */ 202 | if(state->next == state->end) { 203 | state->is_valid = 0; 204 | return 0; 205 | } 206 | state->byte = state->next; 207 | ++(state->next); 208 | res = *(state->byte); 209 | state->n_bits = 8 - bits; 210 | } else { /* 2 parts */ 211 | res = *(state->byte) >> (8 - state->n_bits); 212 | if(state->next == state->end) { 213 | state->is_valid = 0; 214 | return 0; 215 | } 216 | state->byte = state->next; 217 | ++(state->next); 218 | res |= *(state->byte) << state->n_bits; 219 | state->n_bits = 8 - bits + state->n_bits; 220 | } 221 | return res & c_bmasks8[bits]; 222 | } 223 | 224 | void ECL_JH_WJump(ECL_JH_WState* state, ECL_usize distance) { 225 | if((state->next + distance) <= state->end) { 226 | state->next += distance; 227 | } else { 228 | state->next = state->end; 229 | state->is_valid = 0; 230 | } 231 | } 232 | 233 | void ECL_JH_RJump(ECL_JH_RState* state, ECL_usize distance) { 234 | if((state->next + distance) <= state->end) { 235 | state->next += distance; 236 | } else { 237 | state->next = state->end; 238 | state->is_valid = 0; 239 | } 240 | } 241 | 242 | 243 | #ifdef ECL_USE_BRANCHLESS 244 | #define ECL_CALC_E(value, n_bits) \ 245 | (((ECL_usize)((1 << (n_bits)) - 1) - (value)) >> (ECL_SIZE_TYPE_BITS_COUNT - 1 - (n_bits))) & (1 << (n_bits)); 246 | 247 | #else 248 | #define ECL_CALC_E(value, n_bits) \ 249 | ((value) > ((ECL_usize)(1 << (n_bits)) - 1) ? (1 << (n_bits)) : 0) 250 | 251 | #endif 252 | 253 | /* 254 | this should have been done in C++ templates, but unfortunately embedded rather use C. 255 | 0 < num < 8. 256 | */ 257 | #define ECL_E_NUMBER_DEFINE_SIMPLE_IMPL(num) \ 258 | void ECL_JH_Write_E##num(ECL_JH_WState* state, ECL_usize value) { \ 259 | do { \ 260 | ECL_SCOPED_CONST uint8_t e = ECL_CALC_E(value, num); \ 261 | ECL_JH_Write(state, (uint8_t)value | e, num + 1); \ 262 | value >>= num; \ 263 | } while(value); \ 264 | } \ 265 | ECL_usize ECL_JH_Read_E##num(ECL_JH_RState* state) { \ 266 | ECL_usize result; \ 267 | uint8_t shift; \ 268 | shift = 0; \ 269 | result = 0; \ 270 | do { \ 271 | ECL_SCOPED_CONST uint8_t code = ECL_JH_Read(state, (num + 1)); \ 272 | result |= ((ECL_usize)(code & ((1 << num) - 1))) << shift; \ 273 | if(! (code & (1 << num))) { \ 274 | break; \ 275 | } \ 276 | shift += num; \ 277 | } while(shift < ECL_SIZE_TYPE_BITS_COUNT); \ 278 | return result; \ 279 | } \ 280 | uint8_t ECL_Evaluate_E##num(ECL_usize number) { \ 281 | uint8_t result = 0; \ 282 | do { \ 283 | number >>= num; \ 284 | result += num + 1; \ 285 | } while(number); \ 286 | return result; \ 287 | } 288 | 289 | ECL_E_NUMBER_DEFINE_SIMPLE_IMPL(2) 290 | ECL_E_NUMBER_DEFINE_SIMPLE_IMPL(3) 291 | ECL_E_NUMBER_DEFINE_SIMPLE_IMPL(4) 292 | ECL_E_NUMBER_DEFINE_SIMPLE_IMPL(5) 293 | ECL_E_NUMBER_DEFINE_SIMPLE_IMPL(6) 294 | ECL_E_NUMBER_DEFINE_SIMPLE_IMPL(7) 295 | 296 | #undef ECL_E_NUMBER_DEFINE_SIMPLE_IMPL 297 | 298 | /* 299 | 0 < num1 < 8. 300 | 0 < num2 < 8. 301 | */ 302 | #define ECL_E_NUMBER_DEFINE_X2_IMPL(num1, num2) \ 303 | void ECL_JH_Write_E##num1##E##num2(ECL_JH_WState* state, ECL_usize value) { \ 304 | uint8_t e = ECL_CALC_E(value, num1); \ 305 | ECL_JH_Write(state, (uint8_t)value | e, num1 + 1); \ 306 | value >>= num1; \ 307 | while(value) { \ 308 | e = ECL_CALC_E(value, num2); \ 309 | ECL_JH_Write(state, (uint8_t)value | e, num2 + 1); \ 310 | value >>= num2; \ 311 | } \ 312 | } \ 313 | ECL_usize ECL_JH_Read_E##num1##E##num2(ECL_JH_RState* state) { \ 314 | ECL_usize result; \ 315 | uint8_t code, shift; \ 316 | code = ECL_JH_Read(state, num1 + 1); \ 317 | result = code & ((1 << num1) - 1); \ 318 | if(code & (1 << num1)) { \ 319 | shift = num1; \ 320 | do { \ 321 | code = ECL_JH_Read(state, num2 + 1); \ 322 | result |= ((ECL_usize)(code & ((1 << num2) - 1))) << shift; \ 323 | if(! (code & (1 << num2))) { \ 324 | break; \ 325 | } \ 326 | shift += num2; \ 327 | } while(shift < ECL_SIZE_TYPE_BITS_COUNT); \ 328 | } \ 329 | return result; \ 330 | } \ 331 | uint8_t ECL_Evaluate_E##num1##E##num2(ECL_usize number) { \ 332 | uint8_t result = num1 + 1; \ 333 | number >>= num1; \ 334 | while(number) { \ 335 | number >>= num2; \ 336 | result += num2 + 1; \ 337 | } \ 338 | return result; \ 339 | } 340 | 341 | ECL_E_NUMBER_DEFINE_X2_IMPL(4, 5) 342 | ECL_E_NUMBER_DEFINE_X2_IMPL(5, 2) 343 | ECL_E_NUMBER_DEFINE_X2_IMPL(5, 3) 344 | ECL_E_NUMBER_DEFINE_X2_IMPL(5, 4) 345 | ECL_E_NUMBER_DEFINE_X2_IMPL(6, 2) 346 | ECL_E_NUMBER_DEFINE_X2_IMPL(6, 3) 347 | ECL_E_NUMBER_DEFINE_X2_IMPL(6, 4) 348 | ECL_E_NUMBER_DEFINE_X2_IMPL(7, 2) 349 | ECL_E_NUMBER_DEFINE_X2_IMPL(7, 3) 350 | ECL_E_NUMBER_DEFINE_X2_IMPL(7, 4) 351 | 352 | #undef ECL_E_NUMBER_DEFINE_X2_IMPL 353 | 354 | /* cleanup in case of compiling as single file */ 355 | #undef ECL_CALC_E 356 | #undef ECL_CHECK_E_AND_SHIFT 357 | -------------------------------------------------------------------------------- /ECL_config.h: -------------------------------------------------------------------------------- 1 | #ifndef ECL_CONFIG_H_ 2 | #define ECL_CONFIG_H_ 3 | 4 | #include 5 | #include 6 | #include /* memcpy, memset */ 7 | 8 | /* see runtime version in ECL_utils.h */ 9 | #define ECL_VERSION_MAJOR 1 10 | #define ECL_VERSION_MINOR 0 11 | #define ECL_VERSION_PATCH 3 12 | #define ECL_VERSION_NUMBER (ECL_VERSION_MAJOR*10000 + ECL_VERSION_MINOR*100 + ECL_VERSION_PATCH) 13 | 14 | #define ECL_VERSION_BRANCH "master" 15 | 16 | 17 | /* user setup part --------------------------------------------- */ 18 | /* 19 | Add your own definitions here if you don't have available: 20 | - uint8_t 21 | - uint16_t 22 | - uint32_t 23 | */ 24 | 25 | #ifndef ECL_USE_BITNESS_16 26 | #ifndef ECL_USE_BITNESS_32 27 | #ifndef ECL_USE_BITNESS_64 28 | /* none of these is defined with force for compilation, use 32 bits by default */ 29 | #define ECL_USE_BITNESS_32 30 | #endif 31 | #endif 32 | #endif 33 | 34 | /*#define ECL_USE_ASSERT */ 35 | /*#define ECL_USE_BRANCHLESS */ 36 | /*#define ECL_USE_STAT_COUNTERS */ 37 | /*#define ECL_DISABLE_MALLOC */ 38 | /*#define ECL_USE_DLL */ 39 | 40 | 41 | #ifndef ECL_NANO_LZ_ONLY_SCHEME 42 | /* set to 0 to unlock all schemes, set to 1 to have only scheme1, 2 = scheme2 etc. Having single scheme allows compiler to inline for better performance */ 43 | /* default is 1 since most likely you will use only scheme1 and want better performance */ 44 | #define ECL_NANO_LZ_ONLY_SCHEME 1 45 | #endif 46 | 47 | /* non-user part ------------------------------------------ */ 48 | #define ECL_NANO_LZ_IS_SCHEME_ENABLED(scheme_num) ((ECL_NANO_LZ_ONLY_SCHEME == 0) || (ECL_NANO_LZ_ONLY_SCHEME == scheme_num)) 49 | 50 | /* size types --------------------------------------------- */ 51 | #ifdef ECL_USE_BITNESS_16 52 | typedef uint16_t ECL_usize; 53 | typedef int16_t ECL_ssize; 54 | #define ECL_SIZE_TYPE_BITS_COUNT 16 55 | 56 | #elif defined ECL_USE_BITNESS_32 57 | typedef uint32_t ECL_usize; 58 | typedef int32_t ECL_ssize; 59 | #define ECL_SIZE_TYPE_BITS_COUNT 32 60 | 61 | #elif defined ECL_USE_BITNESS_64 62 | typedef uint64_t ECL_usize; 63 | typedef int64_t ECL_ssize; 64 | #define ECL_SIZE_TYPE_BITS_COUNT 64 65 | 66 | #endif 67 | 68 | #define ECL_POINTER_BITS_COUNT (sizeof(void*) * 8) 69 | 70 | /* 71 | Helper paranoid macro to ensure that estimated compression bound doesn't overflow ECL_usize. 72 | Results in bool. Makes sense for ECL_USE_BITNESS_16. 73 | Usage: assert( ECL_VALIDATE_BOUND_OK( ECL_NANO_LZ_GET_BOUND(src_size), src_size ) ); 74 | */ 75 | #define ECL_VALIDATE_BOUND_OK(estimated_max_compressed_size, src_size) (( (ECL_usize)(estimated_max_compressed_size) ) > ( (ECL_usize)(src_size) )) 76 | 77 | 78 | /* asserts --------------------------------------------- */ 79 | #ifdef ECL_USE_ASSERT 80 | #include 81 | #define ECL_ASSERT(expr) assert(expr) 82 | #else 83 | #define ECL_ASSERT(expr) 84 | #endif 85 | 86 | /* malloc/free can be disabled (stubbed). useful if those functions don't compile on some particular platform */ 87 | #ifdef ECL_DISABLE_MALLOC 88 | #define ECL_MEM_ALLOC(size) 0 89 | #define ECL_MEM_FREE(ptr) 90 | #else 91 | #define ECL_MEM_ALLOC(size) malloc(size) 92 | #define ECL_MEM_FREE(ptr) free(ptr) 93 | #endif 94 | 95 | 96 | /* helpful */ 97 | #define ECL_MIN(a, b) ((a) < (b) ? (a) : (b)) 98 | #define ECL_MAX(a, b) ((a) > (b) ? (a) : (b)) 99 | 100 | 101 | /* dyn library macros, etc */ 102 | #ifndef ECL_API_VISIBILITY 103 | #if defined(__GNUC__) && (__GNUC__ >= 4) 104 | #define ECL_API_VISIBILITY __attribute__ ((visibility ("default"))) 105 | #else 106 | #define ECL_API_VISIBILITY 107 | #endif 108 | #endif 109 | 110 | #if defined(ECL_USE_DLL) 111 | #if defined(ECL_DLL_EXPORT) 112 | #define ECL_EXPORTED_API __declspec(dllexport) ECL_API_VISIBILITY 113 | #else 114 | #define ECL_EXPORTED_API __declspec(dllimport) ECL_API_VISIBILITY 115 | #endif 116 | #else 117 | #define ECL_EXPORTED_API ECL_API_VISIBILITY 118 | #endif 119 | 120 | #endif 121 | 122 | 123 | /* compiler quirks */ 124 | #ifdef __XC8 /* more like an example check actually. XC8 compiler */ 125 | #define ECL_SCOPED_CONST /* local const isn't supported */ 126 | #endif 127 | 128 | #ifndef ECL_SCOPED_CONST 129 | #define ECL_SCOPED_CONST const 130 | #endif 131 | -------------------------------------------------------------------------------- /ECL_utils.h: -------------------------------------------------------------------------------- 1 | #ifndef ECL_UTILS_H_ 2 | #define ECL_UTILS_H_ 3 | 4 | #include "ECL_config.h" 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | ECL_EXPORTED_API uint32_t ECL_GetSizeBitness(void); /* returns ECL_SIZE_TYPE_BITS_COUNT */ 11 | ECL_EXPORTED_API uint32_t ECL_GetVersionNumber(void); /* returns ECL_VERSION_NUMBER */ 12 | ECL_EXPORTED_API const char* ECL_GetVersionString(void); /* returns "major.minor.patch" version string */ 13 | ECL_EXPORTED_API const char* ECL_GetVersionBranch(void); /* returns ECL_VERSION_BRANCH */ 14 | 15 | ECL_EXPORTED_API uint8_t ECL_LogRoundUp(ECL_usize value); /* returns [log2(value)]. for 0 returns 1 */ 16 | ECL_EXPORTED_API uint16_t* ECL_GetAlignedPointer2(uint8_t* ptr); /* returns aligned pointer (shifts ptr forward if needed) matching uint16_t alignment */ 17 | ECL_EXPORTED_API ECL_usize* ECL_GetAlignedPointerS(uint8_t* ptr); /* returns aligned pointer (shifts ptr forward if needed) matching ECL_usize alignment */ 18 | 19 | /* 20 | Exported helper functions for storing separate 'size' values in E7 format. 21 | Functions receive pointer to first byte of data (data_start), it's capacity (max_bytes) and value/output_value parameters 22 | to write/read it respectively. 23 | Return pointer to first unused byte, or NULL in case of any error. 24 | */ 25 | ECL_EXPORTED_API uint8_t* ECL_Helper_WriteE7(uint8_t* data_start, ECL_usize max_bytes, ECL_usize value); 26 | ECL_EXPORTED_API const uint8_t* ECL_Helper_ReadE7(const uint8_t* data_start, ECL_usize max_bytes, ECL_usize* output_value); 27 | 28 | #ifdef __cplusplus 29 | } 30 | #endif 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /NEWS.md: -------------------------------------------------------------------------------- 1 | ## Release 1.0.3 2 | - Suppressed some 'loses precision' warnings (explicit conversions are added) 3 | 4 | ## Release 1.0.2 5 | - Added/used ECL_SCOPED_CONST define 6 | - Added handling for ECL_EXCLUDE_HIMEM define (16bit-compilers friendly) 7 | - Minor fix in ZeroDevourer for ECL_USE_BITNESS_16 8 | 9 | ## Release 1.0.1 10 | - Fixed tests compilation for clang 64 bit 11 | - Suppressed rare warning 12 | 13 | ## Release 1.0.0 14 | - NanoLZ generic codec: decompressor, 7 compression modes, 2 *auto compression functions 15 | - NanoLZ schemes: Scheme1, Scheme2(demo) 16 | - ZeroDevourer codec 17 | - ZeroEater codec 18 | - high test coverage for all above 19 | - sample program to compress/decompress files in NanoLZ:Scheme1 format 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | EMBEDDED COMPRESSION LIBRARY 2 | === 3 | 4 | **ECL** aka EMBEDDED COMPRESSION LIBRARY is NOT ONLY for embedded, it is **mostly oriented for small data** and has special optimized **low-memory** modes for restricted environments. 5 | 6 | --- 7 | ### Language: C 8 | ### Platforms: any 9 | ### Endianness: any 10 | ### Library version: 1.0.3 11 | --- 12 | ### Tested on 13 | - Windows 7: msvc2013, msvc2015, gcc 4.8, gcc 7.2 14 | - Mac OS 10.12: clang (Apple LLVM version 8.0.0) 15 | - Embedded ARM Cortex-M3: armcc 5.06 16 | --- 17 | 18 | 19 | ## COMPRESSORS 20 | Some of modes of some compressors use intermediate buffers for compression, they don't use any implicit allocation (unless otherwise specified) - user can easily choose how to allocate buffers. 21 | Every compression method that uses temporary buffer (say, more than 10 bytes) - has it specified in documentation near method declaration. 22 | 23 | ### ECL:NanoLZ - meticulously formatted version of traditional LZ77 algorithm. 24 | - use cases - various, same with other pure LZ algorithms; 25 | - takes advantage of repeated sequences with length of 2 bytes; 26 | - provides API for adjusting compressed data format (see "schemes") to gain better compression ratio for user's datasets (for advanced users); 27 | - can be beneficial for very small amounts of data (e.g. to fit some data into single Bluetooth Low Energy packet); 28 | - compression ratio - middle..high; 29 | - compression ratio limit - roughly infinite; 30 | - compressors complexity - linear..cubic; 31 | - compressors performance - low..high, provides different modes, configurable compression/performance trade off; 32 | - compressors performance for small data payloads - optimized; 33 | - decompressor complexity - linear; 34 | - decompressor performance - middle..high; 35 | - compressors buffer memory consumption - from zero to any, has different modes using 0, 256, 512,.. bytes for temporary buffers; 36 | - decompressor buffer memory consumption - zero; 37 | - static const memory consumption - low; 38 | - stack consumption - normal; 39 | - (TBD if needed) stream modes for processing data by chunks; 40 | 41 | ### ECL:ZeroDevourer - a diff-oriented compressor that takes advantage of zero bytes (even single ones) in your stream. 42 | - use cases - incremental update of a structure FOO where you compress (FOO_before XOR FOO_after) rather than FOO itself, or data with significant amount of zeroes; 43 | - compression ratio - roughly, linearly depends on percentage of zeroes in your data, for target use cases - high; 44 | - compression ratio limit - roughly infinite; 45 | - compressor complexity - linear; 46 | - compressor performance - high (up to gigabyte per second); 47 | - decompressor complexity - linear; 48 | - decompressor performance - high (up to several gigabytes per second); 49 | - compressor buffer memory consumption - zero; 50 | - decompressor buffer memory consumption - zero; 51 | - static const memory consumption - 10 bytes; 52 | - stack consumption - low; 53 | - binary code size - small; 54 | 55 | ### ECL:ZeroEater - a diff-oriented compressor that takes advantage of zero bytes (starting from two bytes in a row) in your stream. 56 | - use cases - incremental update of a structure FOO where you compress (FOO_before XOR FOO_after) rather than FOO itself, or data with significant amount of zeroes; 57 | - compression ratio - similar to `ZeroDevourer` but worse, for target use cases - high; 58 | - compression ratio limit - 64; 59 | - compressor has dry-run mode; 60 | - compressor complexity - linear; 61 | - compressor performance - very high (up to gigabyte per second); 62 | - decompressor complexity - linear; 63 | - decompressor performance - very high (up to several gigabytes per second); 64 | - compressor buffer memory consumption - zero; 65 | - decompressor buffer memory consumption - zero; 66 | - static const memory consumption - zero; 67 | - stack consumption - lowest; 68 | - binary code size - minimum; 69 | - doesn't require "ECL_common.c" file to be compiled; 70 | 71 | 72 | ## FORMATS 73 | Formats of compressors are described in "formats/" dir, there are common features shared between compressors (except `ZeroEater`, which is simple and independent): 74 | - "formats/ECL_JH.txt" - format of Jumping Header, main feature and core of the library; 75 | - "formats/ECL_E_number_format.txt" - format of 'Extensible' numbers; 76 | 77 | 78 | ## EXTRA CONFIGURING 79 | See **ECL_config.h** for details on configuring, mostly controlled by ECL_USE* macros. 80 | - you can explicitly specify bitness of length variables on your consideration (`ECL_USE_BITNESS_16` / 32 / 64 macro), default is 32; 81 | - you can enable/disable branchless optimizations for your consideration (currently inefficient) - see `ECL_USE_BRANCHLESS`; 82 | - you can enable/disable internal asserts (work if system assert works e.g. no `NDEBUG` macro specified) - see `ECL_USE_ASSERT`; 83 | - you can disable malloc/free in case they cause compilation errors on some restricted platforms - see `ECL_DISABLE_MALLOC`; 84 | - you can disable/exclude memory-demanding functions by defining ECL_EXCLUDE_HIMEM (useful for 16bit compilers, e.g. arduino environment) to fix some warnings; 85 | - you can allow all `NanoLZ` schemes or only specific one - to let compiler inline more for better performance - see `ECL_NANO_LZ_ONLY_SCHEME`; 86 | - to use ECL as dynamic library - uncomment define for `ECL_USE_DLL`, to build dynamic library - define also `ECL_DLL_EXPORT`; 87 | - in case you don't have uint*_t types defined in `stdint.h` - define those types there near "user setup part"; 88 | 89 | 90 | ## PERFORMANCE BENCHMARKS 91 | PC benchmarks are performed for *Intel core i5-3570k @ 3.4 GHz / Windows 7 64 bit / 16gb RAM 1600 MHz*. 92 | All benchmarks are performed for ECL version 1.0.0. 93 | Compiled with GCC 7.2.0, options: `-m32 -Wall -Wextra -pedantic -O3`. 94 | - ECL sources are compiled as single file: "ecl-all-c-included/ECL_all_c_included.c" (and for Embedded benchmarks too); 95 | - Speed is in megabytes per second (mb/s); 96 | - For `NanoLZ` used Scheme1 unless otherwise specified; 97 | - ECL is built for 32 bits (`ECL_USE_BITNESS_32`, default option) unless otherwise specified. In most cases 32bit build is the most efficient; 98 | - Compressor parameter (for LZ4_HC - *compressionLevel*, for `NanoLZ` - *search_limit*) is further referenced as *Param*; 99 | - *Ratio* is size of compressed data comparing to size of original data, e.g. compressing 1000 bytes -> 100 bytes corresponds to ratio 0.1. 100 | 101 | Benchmarking small datasets (PC and Embedded environment) 102 | --- 103 | Used samples around 2 kb each. Run in big external cycle: 104 | - main comparison - `NanoLZ` versus [LZ4] (which can also be configured for low-memory); 105 | - for PC: `NanoLZ` demo scheme (Scheme2) - to show that you can achieve different parameters within `NanoLZ`, if needed; 106 | - for PC: [LZ4] in high-compression mode (LZ4_HC) with big memory buffers used; 107 | - for PC: `NanoLZ` with bigger memory buffers used (mid2min, fast1/fast2 using *window_size_bits=11* - further referenced as *Window*) for detailed codec comparison on small data. 108 | 109 | ### PC environment 110 | 111 | | Compressor | Param | Ratio | Compression | Decompression | Compressor memory | 112 | | --------------------------------------------- | ----- | ------- | ------------ | ------------- | ----------------------------- | 113 | | LZ4_compress_default (v1.8.1) | |**0.548**| **625 mb/s**| 2020 mb/s | **256 bytes** | 114 | | **ECL_NanoLZ_Compress_mid1min** | |**0.465**| **249 mb/s**| 335 mb/s | **256 bytes** | 115 | | ECL_NanoLZ_Compress_mid1 | 2 | 0.427 | 114 mb/s | 332 mb/s | **256 bytes** | 116 | | ECL_NanoLZ_Compress_mid1 | 3 | 0.413 | 81 mb/s | 332 mb/s | **256 bytes** | 117 | | ECL_NanoLZ_Compress_mid1 | 4 | 0.4 | 65 mb/s | 332 mb/s | **256 bytes** | 118 | | ECL_NanoLZ_Compress_mid1 | 5 | 0.397 | 56 mb/s | 337 mb/s | **256 bytes** | 119 | | ECL_NanoLZ_Compress_mid1 | 10 | 0.387 | 39 mb/s | 344 mb/s | **256 bytes** | 120 | | ECL_NanoLZ_Compress_mid1 | 50 |**0.377**| 24 mb/s | 355 mb/s | **256 bytes** | 121 | | ECL_NanoLZ_Compress_mid1min **Scheme2 (demo)**| | 0.592 | 228 mb/s | **890 mb/s**| **256 bytes** | 122 | | | | | | | | 123 | | **ECL_NanoLZ_Compress_mid2min** | |**0.465**| **347 mb/s**| 334 mb/s | **513 bytes** | 124 | | ECL_NanoLZ_Compress_fast1 | 20 | 0.383 | 60 mb/s | 340 mb/s | 4612 bytes (**16bit build**) | 125 | | ECL_NanoLZ_Compress_fast1 | 20 | 0.383 | 66 mb/s | 342 mb/s | 9224 bytes | 126 | | **ECL_NanoLZ_Compress_fast2** | 20 | 0.377 | **93 mb/s**| 343 mb/s | 135172 bytes (**16bit build**)| 127 | | **ECL_NanoLZ_Compress_fast2** | 20 | 0.377 | **79 mb/s**| 350 mb/s | 270344 bytes | 128 | | ECL_NanoLZ_Compress_fast2 | 100 |**0.37** | 54 mb/s | 350 mb/s | 135172 bytes (**16bit build**)| 129 | | ECL_NanoLZ_Compress_fast2 | 100 |**0.37** | 50 mb/s | 360 mb/s | 270344 bytes | 130 | | | | | | | | 131 | | LZ4_compress_HC (v1.8.1) | 3 |**0.49** | 87 mb/s | 2230 mb/s | 384 kb | 132 | | LZ4_compress_HC (v1.8.1) | 9 |**0.49** | 59 mb/s | 2230 mb/s | 384 kb | 133 | | LZ4_compress_HC (v1.8.1) | 11 |**0.49** | 16 mb/s | 2230 mb/s | 384 kb | 134 | | LZ4_compress_HC (v1.8.1) | 12 |**0.49** | 10 mb/s | 2230 mb/s | 384 kb | 135 | 136 | On some other small datasets (highly compressible) `NanoLZ`:Scheme1 decompression speed exceeded **1000 mb/s**, 137 | while compression speed of ECL_NanoLZ_Compress_mid2min reached **570 mb/s**. 138 | 139 | ### Embedded environment 140 | - hardware: ARM Cortex-M3 120 MHz; 141 | - compiler: armcc 5.06; 142 | - optimization options: `-O3`. 143 | 144 | | Compressor | Param | Ratio | Compression | Decompression | Compressor memory | 145 | | ------------------------------- | ----- | ------- | ------------ | ------------- | ----------------- | 146 | | LZ4_compress_default (v1.8.0) | |**0.53** |**1.785 mb/s**| 10.12 mb/s | **1024 bytes** | 147 | | **ECL_NanoLZ_Compress_mid2min** | |**0.465**|**1.822 mb/s**| 2.42 mb/s | **513 bytes** | 148 | | ECL_NanoLZ_Compress_mid2 | 2 | 0.427 | 0.865 mb/s | 2.42 mb/s | **513 bytes** | 149 | | ECL_NanoLZ_Compress_mid2 | 3 | 0.413 | 0.718 mb/s | 2.46 mb/s | **513 bytes** | 150 | | ECL_NanoLZ_Compress_mid2 | 4 | 0.4 | 0.636 mb/s | 2.48 mb/s | **513 bytes** | 151 | | ECL_NanoLZ_Compress_mid2 | 5 | 0.397 | 0.584 mb/s | 2.54 mb/s | **513 bytes** | 152 | | ECL_NanoLZ_Compress_mid2 | 10 | 0.387 | 0.455 mb/s | 2.57 mb/s | **513 bytes** | 153 | 154 | [LZ4]: http://www.lz4.org/ 155 | 156 | Benchmarking large datasets (only PC environment) 157 | --- 158 | Big datasets used here to show performance measured on big data without wrapping in external cycle, they also show that `NanoLZ` is inappropriate choice for big data. 159 | On my measurements compression ratio of [LZ4] wins on data bigger than around 25 kb (this bound isn't accurately measured and appropriate statistic isn't provided). 160 | Though, again, **with custom scheme** you are able to achieve different characteristics. 161 | 162 | #### Benchmarks for data samples from [Silesia Corpus] (further referenced as *Silesia*): 12 files of different types, sizes range in 6..51 mb: 163 | 164 | | Compressor | Param | Window | Compression | Decompression | Compressor memory | 165 | | ------------------------- | ----- | ------ | -------------- | --------------- | ----------------- | 166 | | ECL_NanoLZ_Compress_fast2 | 1 | 16 | 57..158 mb/s | 122..310 mb/s | 512kb | 167 | | ECL_NanoLZ_Compress_fast2 | 10 | 16 | 25..98 mb/s | 132..435 mb/s | 512kb | 168 | | ECL_NanoLZ_Compress_fast2 | 100 | 16 | 8..31 mb/s | 164..569 mb/s | 512kb | 169 | | ECL_NanoLZ_Compress_fast2 | 10 | 18 | 22..97 mb/s | 135..436 mb/s | 1.3mb | 170 | | ECL_NanoLZ_Compress_fast2 | 100 | 18 | 5..23 mb/s | 161..586 mb/s | 1.3mb | 171 | | | | | | | | 172 | | ECL_ZeroEater_Compress | | | 518..1446 mb/s | 1353..7000 mb/s | 0 | 173 | | ECL_ZeroDevourer_Compress | | | 341..1456 mb/s | 739..10317 mb/s | 0 | 174 | 175 | Note that prodigious decompression speed of `ZeroEater` and `ZeroDevourer` is achieved on files they cannot compress, see next table for accumulated statistic. 176 | 177 | #### Benchmarks for single *Silesia*.tar file (202 mb): 178 | 179 | | Compressor | Param | Window | Ratio | Compression | Decompression | Compressor memory | 180 | | ------------------------- | ----- | ------ | ------- | ----------- | ------------- | ----------------- | 181 | | memcpy | | | 1.000 | 6300 mb/s | 6300 mb/s | 0 | 182 | | LZ4_compress_default (v1.8.1) | | | 0.479 | 437 mb/s | 1543 mb/s | 16 kb | 183 | | LZ4_compress_HC (v1.8.1) | 3 | | 0.383 | 74.5 mb/s | 1704 mb/s | 384 kb | 184 | | LZ4_compress_HC (v1.8.1) | 9 | |**0.367**| 26.9 mb/s | 1783 mb/s | 384 kb | 185 | | | | | | | | | 186 | | ECL_NanoLZ_Compress_fast2 | 1 | 16 | 0.473 | 92.5 mb/s | 192.7 mb/s | 512 kb | 187 | | ECL_NanoLZ_Compress_fast2 | 10 | 16 | 0.407 | 47.5 mb/s | 221.5 mb/s | 512 kb | 188 | | ECL_NanoLZ_Compress_fast2 | 100 | 16 | 0.39 | 15.3 mb/s | 250 mb/s | 512 kb | 189 | | ECL_NanoLZ_Compress_fast2 | 10 | 18 | 0.405 | 42.7 mb/s | 222 mb/s | 1.3 mb | 190 | | ECL_NanoLZ_Compress_fast2 | 100 | 18 |**0.385**| 10.5 mb/s | 254 mb/s | 1.3 mb | 191 | | | | | | | | | 192 | | ECL_ZeroEater_Compress dry | | | 0.948 | 1136 mb/s | | 0 | 193 | | ECL_ZeroEater_Compress | | | 0.948 |**925 mb/s** |**2940 mb/s** | 0 | 194 | | ECL_ZeroDevourer_Compress | | | 0.935 | 696 mb/s | 1905 mb/s | 0 | 195 | 196 | [Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia 197 | 198 | 199 | ## MULTITHREADING 200 | Codecs don't share any non-const data, so API is thread safe. 201 | 202 | 203 | ## SAMPLE TESTING PROGRAM 204 | There's sample program to compress/decompress in `NanoLZ`:Scheme1 format via command line: see **"sample/"** dir. 205 | It's enough to compile single **"sample/sample.cpp"** file, some building scripts are provided in same dir. 206 | Program is unable to compress too large files. 207 | 208 | 209 | ## USAGE 210 | In general usage is pretty straightforward - you call \*Compress method, you call \*Decompress method - that's it. 211 | - usage samples present in headers of each codec; 212 | - see **"sample/sample.cpp"** example program to encode/decode files; 213 | - you can find examples in tests located in **"tests/"** dir. 214 | 215 | 216 | ## BUILDING 217 | It's enough to build single **"ecl-all-c-included/ECL_all_c_included.c"** file, so do it unless you have reasons to compile minimum amount of code. 218 | If you need to minimize amount of code to compile, then: to have compressor "FOO" available - include "ECL_FOO.h", compile "ECL_common.c" + "ECL_FOO.c". 219 | 220 | If you need a static/dynamic library instead of adding ECL sources to your project - you will have to compile it yourself (no such scripts here). To build dynamic library define `ECL_USE_DLL` and `ECL_DLL_EXPORT` macros, to import it - define only `ECL_USE_DLL` macro. 221 | 222 | Note that building as C rather than C++ results in higher performance due to absence of code generated for exception handling, so simple #including code into some of your C++ source files is not the most efficient option. 223 | 224 | 225 | ## TESTS 226 | - unit tests are in **"tests/"** dir, can be built and launched with scripts in same place. Basically single **"tests/tests.cpp"** file is enough to be compiled; 227 | - scripts in **"tests/"** dir build and run ECL consequently for 16, 32 and 64 bits (bitness of ECL_usize); 228 | - executable test file has optional "depth" parameter on launch: a number between -10 and 1000 (e.g. "tests.exe 50"). It determines tests coverage and time spent on tests (-10 for fast run, 1000 for very deep/slow run); 229 | - sources of tests can be easily embedded into another project and run any amount of times with any parameters from there, see **"tests/tests.cpp"**. 230 | -------------------------------------------------------------------------------- /ecl-all-c-included/ECL_all_c_included.c: -------------------------------------------------------------------------------- 1 | #include "../ECL_common.c" 2 | #include "../ECL_ZeroEater.c" 3 | #include "../ECL_ZeroDevourer.c" 4 | #include "../ECL_NanoLZ.c" 5 | -------------------------------------------------------------------------------- /formats/ECL_E_number_format.txt: -------------------------------------------------------------------------------- 1 | Format of E numbers. 2 | 3 | E-number (e.g. E4, E7E4) is named like that for simplicity. Numbers are unsigned. 4 | E stands for 'extensible'. 5 | Number consists of fixed-length blocks, each block contains 'E' flag as elder bit in block, 6 | this bit determines whether there's next block. Next block contains elder bits of number. 7 | 8 | Example for E4 (each block has 4 data bits): 9 | value = read 5 bits // where 5 is 4 + extension 10 | result = (value & 0x0F) // retrieve 4 lower bits 11 | if value & 0x10: // check extension flag 12 | value = read 5 bits // where 5 is 4 + extension 13 | result = result | ((value & 0x0F) << 4) 14 | if value & 0x10: // check extension flag 15 | ... 16 | 17 | Similar is fair for E2, E3, E5, E7... each block has equal amount of bits. 18 | Numbers having several E in format name (e.g. E7E4) have appropriate block sizes and last block is cycled. 19 | 20 | Example for E7E4 (first block has 7 data bits, all others have 4): 21 | value = read 8 bits // where 8 is 7 + extension 22 | result = (value & 0x7F) // retrieve 7 lower bits 23 | if value & 0x80: // check extension flag 24 | value = read 5 bits // where 5 is 4 + extension 25 | result = result | ((value & 0x0F) << 7) 26 | if value & 0x10: // check extension flag 27 | ... read E4 blocks 28 | 29 | That's it. -------------------------------------------------------------------------------- /formats/ECL_JH.txt: -------------------------------------------------------------------------------- 1 | Jumping Header (JH) is an approach of storing strongly-packed data with small retrieving time. 2 | It assumes 2 types of data being united: 3 | - bit fields of various length 4 | - raw bytes data aligned by byte bound 5 | 6 | JH represents byte-stream with both bit and byte data messed inside and has methods for adding/retrieving bit and byte data: 7 | 1. stream has: 8 | - p_byte - pointer to byte in the stream 9 | - p_next pointer to byte in the stream 10 | - p_end pointer to byte in the stream, followed by last byte 11 | - n_bits counter representing amount of bits unused in byte pointed by p_byte 12 | 13 | 2. initializing stream sets: 14 | - p_byte = pointer to first byte 15 | - p_next = pointer to second byte 16 | - p_end = pointer to byte after last byte 17 | - n_bits = 8 18 | 19 | 3. to read a single bit of data: 20 | if n_bits == 0: 21 | if p_next == p_end: 22 | // error, stream completed 23 | exit 24 | p_byte = p_next // update pointer 25 | n_bits = 8 26 | output = (*p_byte >> (8 - n_bits)) & 0x01 27 | n_bits = n_bits - 1 28 | 29 | 4. when read more than 1 bit of data at time - lower bits logically go first, so reading value A of 2 bits is same to: 30 | A = read_bits(1) | (read_bits(1) << 1); 31 | 32 | 5. to read a sequence of raw bytes: 33 | src_pointer = p_next 34 | p_next = p_next + n_bytes_to_read // jump 35 | copy n_bytes_to_read bytes from src_pointer 36 | 37 | 6. extra note: if you haven't read/written any bits before first jump - first byte is still unused and reserved for bits data, 38 | so first jump cannot be for distance of whole buffer size, at most "sizeof(buffer) - 1". 39 | 40 | 41 | So for example you: 42 | - put 11 bits of data (A) in stream 43 | - put 5 bytes of data (Araw) in stream 44 | - put 7 bits of data (B) in stream 45 | - put 3 bytes of data (Braw) in stream 46 | - put 9 bits of data (C) in stream 47 | Your stream looks like this sequence of bytes (each byte is in square brackets): 48 | [A] [A, B] [Araw] [Araw] [Araw] [Araw] [Araw] [B, C] [Braw] [Braw] [Braw] [C] 49 | 50 | Bit header B is split between bytes stream[1] and stream[7] due to jump over sequence of raw A data. 51 | Bit header C is split between bytes stream[7] and stream[11] due to jump over sequence of raw B data. 52 | 53 | This: 54 | - is possible due to knowledge of where to jump, which is shared between coder and decoder; 55 | - allows to make very compact data streams and take advantage of memcpy for raw sequences aligned by byte bound; 56 | - leaves unused bits in last logical partial byte (which is not necessarily the very last byte), similarly to a stream fully written as bitset; 57 | - requires peculiar handling if infinite stream processing is needed (special opcode of your stream to handle forced byte alignment); 58 | 59 | This is the core of ECL formats. 60 | That's it. -------------------------------------------------------------------------------- /formats/ECL_NanoLZ.txt: -------------------------------------------------------------------------------- 1 | For E2, E3, E6E3 formats refer to ECL_E_number_format.txt. 2 | 3 | First byte of compressed stream is exactly first byte of uncompressed data, further compressed data represent consequent 4 | blocks of format determined by used scheme (ECL_NanoLZ_Scheme enum). 5 | 6 | Stable schemes are: {ECL_NANOLZ_SCHEME1} - they're not to be changed. 7 | 8 | ECL_NANOLZ_SCHEME1 format: TBD (currently I just don't have time to document it). 9 | -------------------------------------------------------------------------------- /formats/ECL_ZeroDevourer.txt: -------------------------------------------------------------------------------- 1 | ZeroDevourer stream format. 2 | Refers to functionality from: 3 | - ECL_JH.txt 4 | - ECL_E_number_format.txt 5 | 6 | 1. Stream consists of independent blocks aligned with JumpingHeader. 7 | 8 | 2. Decompressor logic for each block (error conditions are not checked in this example): 9 | 10 | not_zero = read bit from JumpingHeader 11 | if not_zero == 0: 12 | add zero byte to output stream 13 | continue to next block 14 | else: 15 | opcode = read 2 bits from JumpingHeader 16 | if opcode == 0: 17 | count_x = read 2 bits from JumpingHeader 18 | count_x = count_x + 1 19 | count_0 = 1 20 | else if opcode == 1: 21 | count_x = read 4 bits from JumpingHeader 22 | count_x = count_x + 5 23 | count_0 = 1 24 | else if opcode == 2: 25 | count_x = 0 26 | count_0 = read E4 number from JumpingHeader 27 | count_0 = count_0 + 9 28 | else: // opcode == 3 29 | count_x = read E6E3 number from JumpingHeader 30 | count_x = count_x + 1 31 | count_0 = 0 32 | 33 | src_pointer = JumpingHeader.next 34 | JumpingHeader.jump(count_x) 35 | copy count_x bytes from src_pointer to output 36 | add count_0 zero bytes to output 37 | 38 | That's it. -------------------------------------------------------------------------------- /formats/ECL_ZeroEater.txt: -------------------------------------------------------------------------------- 1 | ZeroEater stream format. 2 | 3 | 1. Stream consists of independent blocks aligned by byte bound. 4 | 5 | 2. Each block starts with a "header" byte and optionally followed by a sequence of raw bytes. 6 | 7 | 3. Decompressor logic for each block (error conditions are not checked in this example): 8 | 9 | header = read byte from stream 10 | if (header & 0x80) == 0: 11 | count_x = (header & 0x7F) + 1 12 | copy count_x bytes from stream to output // stream pointer is adjusted 13 | else if (header & 0x40) == 0: 14 | count_0 = (header & 0x3F) + 1 15 | add count_0 zero bytes to output 16 | else: 17 | count_x = ((header >> 3) & 0x07) + 1 18 | count_0 = (header & 0x07) + 1 19 | copy count_x bytes from stream to output // stream pointer is adjusted 20 | add count_0 zero bytes to output 21 | 22 | That's it. -------------------------------------------------------------------------------- /sample/run-msvc2013-helper.bat: -------------------------------------------------------------------------------- 1 | %comspec% /k ""C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"" x86 -------------------------------------------------------------------------------- /sample/run-msvc2015-helper.bat: -------------------------------------------------------------------------------- 1 | %comspec% /k ""C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"" x86 -------------------------------------------------------------------------------- /sample/sample.cpp: -------------------------------------------------------------------------------- 1 | #include "../ECL_common.c" 2 | #include "../ECL_NanoLZ.c" 3 | #include "../ECL_utils.h" 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | static const auto c_scheme = ECL_NANOLZ_SCHEME1; 11 | 12 | void s_show_usage() { 13 | std::cout << "-- Usage compress: sample c 20 my-src-file my-output-compressed-file" << std::endl; 14 | std::cout << "-- Usage decompress: sample d my-compressed-file my-output-file" << std::endl; 15 | std::cout << " where '20' is serach_limit (compression level), which has to be > 0" << std::endl; 16 | std::cout << " e.g:" << std::endl; 17 | std::cout << " sample c 20 111.txt 111.txt.nlz" << std::endl; 18 | std::cout << " sample d 111.txt.nlz 111.txt.recovered" << std::endl; 19 | std::cout << " will provide '111.txt.nlz' compressed file, and '111.txt.recovered' decompressed file matching original '111.txt'" << std::endl; 20 | std::cout << " OPERANDS ORDER IS STRICT." << std::endl; 21 | } 22 | 23 | typedef std::vector Raw; 24 | 25 | bool s_file_exists(const char* fname) { 26 | std::ifstream ifs(fname, std::ios::binary); 27 | return !!ifs; 28 | } 29 | 30 | Raw s_read_file(const char* fname) { 31 | std::ifstream ifs(fname, std::ios::binary); 32 | if(! ifs) { 33 | return Raw(); 34 | } 35 | ifs.seekg(0, ifs.end); 36 | auto size = ifs.tellg(); 37 | ifs.seekg(0, ifs.beg); 38 | if(size) { 39 | Raw data; 40 | data.resize(size); 41 | if(ifs.read((char*)data.data(), size)) { 42 | return data; 43 | } 44 | } 45 | return Raw(); 46 | } 47 | 48 | Raw s_handle_src(const char* src_fname) { 49 | if(! s_file_exists(src_fname)) { 50 | std::cout << "- error: file '" << src_fname << "' doesn't exist / can't be opened" << std::endl; 51 | return Raw(); 52 | } 53 | auto src = s_read_file(src_fname); 54 | if(! src.size()) { 55 | std::cout << "- error: can't read file '" << src_fname << "' or it's empty" << std::endl; 56 | return Raw(); 57 | } 58 | return src; 59 | } 60 | 61 | bool s_write_file_part(std::ostream& file, const uint8_t* data, size_t size) { 62 | if(data && size) { 63 | if(! file.write((const char*)data, size)) { 64 | std::cout << "- error: can't write file" << std::endl; 65 | return false; 66 | } 67 | } 68 | return true; 69 | } 70 | 71 | bool s_write_file(const char* fname, const uint8_t* hdr, size_t hdr_size, const Raw& data) { 72 | std::ofstream ofs(fname, std::ios::binary); 73 | if(! ofs) { 74 | std::cout << "- error: file '" << fname << "' can't be written" << std::endl; 75 | return false; 76 | } 77 | return s_write_file_part(ofs, hdr, hdr_size) 78 | && s_write_file_part(ofs, data.data(), data.size()); 79 | } 80 | 81 | bool s_try_compress(const char* src_fname, const char* dst_fname, int limit) { 82 | std::cout << "compressing '" << src_fname << "' to '" << dst_fname << "' with limit=" << limit << std::endl; 83 | const auto src = s_handle_src(src_fname); 84 | if(! src.size()) { 85 | return false; 86 | } 87 | Raw output; 88 | const auto enough_size = ECL_NANO_LZ_GET_BOUND(src.size()); 89 | output.resize(enough_size); 90 | const auto comp_size = ECL_NanoLZ_Compress_auto(c_scheme, src.data(), src.size(), output.data(), output.size(), limit); 91 | if((! comp_size) || (comp_size > output.size())) { 92 | std::cout << "- error: an unknown error has occurred, maybe file is too big" << std::endl; 93 | return false; 94 | } 95 | output.resize(comp_size); 96 | // encode original file size in header - encode as E7 number 97 | uint8_t hdr[10]; 98 | const auto hdr_end = ECL_Helper_WriteE7(hdr, sizeof(hdr), src.size()); 99 | if(! hdr_end) { 100 | std::cout << "- error: unknown stream error :|" << std::endl; 101 | return false; 102 | } 103 | const auto hdr_size = hdr_end - hdr; 104 | const auto total_size = comp_size + hdr_size; 105 | // write file data 106 | std::cout << "- successfully compressed: original size = " << src.size() << std::endl; 107 | std::cout << "compressed stream size = " << comp_size 108 | << " (with " << hdr_size << " byte header = " << total_size << ")" << std::endl; 109 | std::cout << "ratio = " << std::fixed << (double(comp_size) / double(src.size())) << std::endl; 110 | return s_write_file(dst_fname, hdr, hdr_size, output); 111 | } 112 | 113 | bool s_try_decompress(const char* src_fname, const char* dst_fname) { 114 | std::cout << "decompressing '" << src_fname << "' to '" << dst_fname << "'" << std::endl; 115 | const auto src = s_handle_src(src_fname); 116 | if(! src.size()) { 117 | return false; 118 | } 119 | ECL_usize original_size; 120 | const auto hdr_end = ECL_Helper_ReadE7(src.data(), src.size(), &original_size); 121 | if(! hdr_end) { 122 | std::cout << "- error: invalid file content" << std::endl; 123 | return false; 124 | } 125 | const auto comp_start = hdr_end; 126 | const auto comp_size = src.data() + src.size() - comp_start; 127 | Raw recovered; 128 | recovered.resize(original_size); 129 | const auto decomp_size = ECL_NanoLZ_Decompress(c_scheme, comp_start, comp_size, recovered.data(), recovered.size()); 130 | if(decomp_size != original_size) { 131 | std::cout << "- error: decompression failed - invalid file content" << std::endl; 132 | return false; 133 | } 134 | std::cout << "- successfully decompressed: size = " << decomp_size << std::endl; 135 | return s_write_file(dst_fname, nullptr, 0, recovered); 136 | } 137 | 138 | bool s_cmp_operand(const char* str, const char* expected) { 139 | return str && expected && (0 == strcmp(str, expected)); 140 | } 141 | 142 | int main(int argc, char* argv[]) { 143 | std::cout << "*** Sample ECL program to compress/decompress with ECL:NanoLZ format ***" << std::endl; 144 | if((argc == 4) && s_cmp_operand(argv[1], "d")) { 145 | if(s_try_decompress(argv[2], argv[3])) { 146 | return 0; 147 | } 148 | std::cout << std::endl; 149 | } else if((argc == 5) && s_cmp_operand(argv[1], "c")) { 150 | int limit = atoi(argv[2]); 151 | if(limit < 1) { 152 | std::cout << "- error: limit has to be > 0" << std::endl; 153 | } else if(s_try_compress(argv[3], argv[4], limit)) { 154 | return 0; 155 | } 156 | std::cout << std::endl; 157 | } 158 | s_show_usage(); 159 | return 0; 160 | } 161 | -------------------------------------------------------------------------------- /sample/unix_clang.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | clang sample.cpp -std=c++11 -lstdc++ -Werror -Wall -pedantic -O3 -o ecl 4 | -------------------------------------------------------------------------------- /sample/unix_gcc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | gcc sample.cpp -std=c++11 -lstdc++ -Werror -Wall -pedantic -O3 -o ecl 4 | -------------------------------------------------------------------------------- /sample/win_gcc.bat: -------------------------------------------------------------------------------- 1 | gcc sample.cpp -m32 -Wall -Wextra -pedantic -O3 -std=c++11 -lstdc++ -o ecl.exe 2 | -------------------------------------------------------------------------------- /sample/win_msvc.bat: -------------------------------------------------------------------------------- 1 | cl sample.cpp /O2 /EHsc /Fe:ecl.exe 2 | -------------------------------------------------------------------------------- /tests/ntest/ntest.cpp: -------------------------------------------------------------------------------- 1 | #include "ntest.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include // strcmp 8 | 9 | namespace NTEST_NAMESPACE_NAME { 10 | 11 | uint64_t GetTimeMicroseconds() { 12 | return std::chrono::duration_cast( 13 | std::chrono::system_clock::now().time_since_epoch()).count(); 14 | } 15 | 16 | typedef std::vector RunnersType; 17 | 18 | static RunnersType& GetRunners() { 19 | static RunnersType runners; 20 | return runners; 21 | } 22 | 23 | TestBase :: TestBase(const char* _name) 24 | : name(_name), result(INIT), time_mcs(0) { 25 | } 26 | 27 | TestBase::Result TestBase :: run(std::ostream& log, int depth) { 28 | struct Guard { 29 | TestBase* const b; 30 | const uint64_t before; 31 | Guard(TestBase* _b) : b(_b), before(GetTimeMicroseconds()) {} 32 | ~Guard() { b->time_mcs = GetTimeMicroseconds() - before; } 33 | } guard(this); 34 | result = INIT; 35 | runInternal(log, depth); 36 | return result; 37 | } 38 | 39 | const char* TestBase :: getName() const { 40 | return name; 41 | } 42 | 43 | uint64_t TestBase :: getDurationMicroseconds() const { 44 | return time_mcs; 45 | } 46 | 47 | void TestBase :: PushRunner(TestBase* test) { 48 | GetRunners().push_back(test); 49 | } 50 | 51 | bool TestBase :: hasntFailed() const { 52 | return result != FAIL; 53 | } 54 | 55 | const char* TestBase :: ResultToStr(Result result) { 56 | switch (result) { 57 | case INIT: return "?"; 58 | case SUCCESS: return "PASS"; 59 | case SKIP: return "SKIPPED"; 60 | case FAIL: return "FAILED"; 61 | case CRASH: return "CRASH"; 62 | }; 63 | return nullptr; 64 | } 65 | 66 | int TestBase :: BoundVMinMax(int v, int min, int max) { 67 | if(min > max) { 68 | throw std::runtime_error("min > max in BoundVMinMax"); 69 | } 70 | if(v < min) { 71 | return min; 72 | } 73 | if(v > max) { 74 | return max; 75 | } 76 | return v; 77 | } 78 | 79 | #define NTEST_STRING_OF_HELPER(x) #x 80 | #define NTEST_STRING_OF(x) NTEST_STRING_OF_HELPER(x) 81 | #define NTEST_NS_STRING NTEST_STRING_OF(NTEST_NAMESPACE_NAME) 82 | 83 | size_t TestBase :: RunTests(std::ostream& log_output, int depth) { 84 | const auto comp = [](const TestBase* left, const TestBase* right) { 85 | return strcmp(left->getName(), right->getName()) < 0; 86 | }; 87 | auto& tests = GetRunners(); 88 | std::sort(tests.begin(), tests.end(), comp); 89 | size_t n_failed = 0; 90 | size_t n_succeeded = 0; 91 | size_t n_skipped = 0; 92 | size_t n_crashed = 0; 93 | log_output << "ntest" << NTEST_VERSION_STRING << " (compiled with namespace '" << NTEST_NS_STRING 94 | << "'): Running tests with depth = " << depth << std::endl; 95 | double total_time = 0; 96 | for(auto runner : tests) { 97 | const auto name = runner->getName(); 98 | Result result = INIT; 99 | try { 100 | result = runner->run(log_output, depth); 101 | } catch (const std::exception& e) { 102 | log_output << "* test " << name << " thrown exception: " << e.what() << std::endl; 103 | result = CRASH; 104 | } catch (...) { 105 | result = CRASH; 106 | } 107 | switch (result) { 108 | case SUCCESS: ++n_succeeded; break; 109 | case SKIP: ++n_skipped; break; 110 | case CRASH: ++n_crashed; // fallthru 111 | default: ++n_failed; break; 112 | } 113 | const auto seconds = (double(runner->getDurationMicroseconds()) / 1000000.); 114 | total_time += seconds; 115 | log_output << '[' << std::setw(7) << std::left << ResultToStr(result) << "] " << std::fixed << seconds << " : " << name << std::endl; 116 | } 117 | log_output << "ntest: Total tests run : " << (n_failed + n_succeeded) << " in " << std::fixed << total_time << " seconds" << std::endl; 118 | log_output << "ntest: Succeeded : " << n_succeeded << std::endl; 119 | log_output << "ntest: Skipped : " << n_skipped << std::endl; 120 | log_output << "ntest: Failed : " << n_failed; 121 | if(n_crashed) { 122 | log_output << " (crashed: " << n_crashed << ")"; 123 | } 124 | log_output << std::endl; 125 | return n_failed; 126 | } 127 | 128 | #undef NTEST_STRING_OF_HELPER 129 | #undef NTEST_STRING_OF 130 | #undef NTEST_NS_STRING 131 | 132 | void TestBase :: skip() { 133 | result = SKIP; 134 | } 135 | 136 | bool TestBase :: approve(bool condition) { 137 | if(result != FAIL) { 138 | result = condition ? SUCCESS : FAIL; 139 | return result == FAIL; 140 | } 141 | return false; 142 | } 143 | 144 | } // end namespace ntest 145 | -------------------------------------------------------------------------------- /tests/ntest/ntest.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | #define NTEST_VERSION_STRING "2.3.1" 6 | 7 | // allow to override namespace to have ability of having multiple thirdparties with ntest easily launched within single project 8 | #ifndef NTEST_NAMESPACE_NAME 9 | #define NTEST_NAMESPACE_NAME ntest 10 | #endif 11 | 12 | namespace NTEST_NAMESPACE_NAME { 13 | 14 | // a stub to force adding semicolons after macro 15 | inline void ntest_noop() {} 16 | 17 | uint64_t GetTimeMicroseconds(); 18 | 19 | class TestBase { 20 | public: 21 | enum Result { 22 | INIT, 23 | SUCCESS, 24 | SKIP, 25 | FAIL, 26 | CRASH, 27 | }; 28 | 29 | TestBase(const char* _name); 30 | Result run(std::ostream& log, int depth); 31 | const char* getName() const; 32 | uint64_t getDurationMicroseconds() const; 33 | bool isFailed() const { return result == FAIL; }; 34 | 35 | static const char* ResultToStr(Result result); 36 | static int BoundVMinMax(int v, int min, int max); // returns value 'v' bounded to [min..max]. min <= max 37 | static size_t RunTests(std::ostream& log_output, int depth); // returns amount of failed tests 38 | 39 | protected: 40 | static void PushRunner(TestBase*); 41 | bool hasntFailed() const; 42 | 43 | void skip(); 44 | bool approve(bool condition); // don't use 'assert' name to not conflict with C-library. returns whether failed now first time 45 | virtual void runInternal(std::ostream&, int) = 0; 46 | 47 | private: 48 | const char* name; 49 | Result result; 50 | uint64_t time_mcs; 51 | }; 52 | 53 | } //end ntest 54 | 55 | #define NTEST_SUPPRESS_UNUSED (void)log; (void)depth; NTEST_NAMESPACE_NAME::ntest_noop() 56 | 57 | #define NTEST_REQUIRE_DEPTH_ABOVE(value) if(depth <= (value)) { skip(); return; } NTEST_NAMESPACE_NAME::ntest_noop() 58 | 59 | #define NTEST(test_name) \ 60 | class test_name : public NTEST_NAMESPACE_NAME::TestBase { \ 61 | public: \ 62 | test_name() : NTEST_NAMESPACE_NAME::TestBase(#test_name) { \ 63 | PushRunner(this); \ 64 | }; \ 65 | protected: \ 66 | void runInternal(std::ostream& log, int depth) override; \ 67 | }; \ 68 | static test_name test_name##_instance; \ 69 | void test_name :: runInternal(std::ostream& log, int depth) 70 | -------------------------------------------------------------------------------- /tests/run-msvc2013-helper.bat: -------------------------------------------------------------------------------- 1 | %comspec% /k ""C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"" x86 -------------------------------------------------------------------------------- /tests/run-msvc2015-helper.bat: -------------------------------------------------------------------------------- 1 | %comspec% /k ""C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"" x86 -------------------------------------------------------------------------------- /tests/run-msvc2017-x64-helper.bat: -------------------------------------------------------------------------------- 1 | %comspec% /k ""C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat"" x64 -------------------------------------------------------------------------------- /tests/run-msvc2017-x86-helper.bat: -------------------------------------------------------------------------------- 1 | %comspec% /k ""C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat"" x86 -------------------------------------------------------------------------------- /tests/run-msvc2019-x64-helper.bat: -------------------------------------------------------------------------------- 1 | %comspec% /k ""C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat"" x64 -------------------------------------------------------------------------------- /tests/run-msvc2019-x86-helper.bat: -------------------------------------------------------------------------------- 1 | %comspec% /k ""C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat"" x86 -------------------------------------------------------------------------------- /tests/tests.cpp: -------------------------------------------------------------------------------- 1 | // optional define 2 | #define NTEST_NAMESPACE_NAME ntest_for_ecl 3 | 4 | #include "tests_base_inline.cpp" 5 | 6 | int main(int argc, char* argv[]) { 7 | int depth = 0; 8 | if(argc == 2) { 9 | depth = atoi(argv[1]); 10 | } 11 | return NTEST_NAMESPACE_NAME::TestBase::RunTests(std::cout, depth); 12 | } 13 | -------------------------------------------------------------------------------- /tests/tests_base_inline.cpp: -------------------------------------------------------------------------------- 1 | #include "ntest/ntest.cpp" 2 | #include "../ECL_JH_States.h" 3 | #include "../ECL_ZeroEater.h" 4 | #include "../ECL_ZeroDevourer.h" 5 | #include "../ECL_NanoLZ.h" 6 | #include "../ECL_utils.h" 7 | 8 | #ifndef ECL_BUILD_AS_C 9 | #include "../ECL_common.c" 10 | #include "../ECL_ZeroEater.c" 11 | #include "../ECL_ZeroDevourer.c" 12 | #include "../ECL_NanoLZ.c" 13 | #endif 14 | 15 | #include 16 | 17 | // auxiliary macro and methods for testing 18 | #define ECL_TEST_ASSERT(expr) approve((bool)(expr)) 19 | #define ECL_TEST_COMPARE(val1, val2) \ 20 | { \ 21 | if((val1) != (val2)) { \ 22 | if(hasntFailed()) { \ 23 | log << "VAL1: " << std::hex << (val1) << " != VAL2: " << (val2) << " "; \ 24 | } \ 25 | approve(false); \ 26 | } else { \ 27 | approve(true); \ 28 | } \ 29 | } 30 | 31 | #define ECL_TEST_MAGIC_RESIZE(vector_name, capacity) \ 32 | vector_name.resize(capacity + 1); \ 33 | vector_name[capacity] = 0x39; NTEST_NAMESPACE_NAME::ntest_noop() 34 | 35 | #define ECL_TEST_MAGIC_VALIDATE(vector_name) \ 36 | ECL_TEST_ASSERT(vector_name[vector_name.size() - 1] == 0x39) 37 | 38 | 39 | 40 | static void ECL_TEST_LogRawData(std::ostream& log, const std::vector& v) { 41 | const size_t max_size = 50; 42 | auto sz = std::min(max_size, v.size()); 43 | log << '{' << v.size() << '}'; 44 | log << '['; 45 | for(size_t i = 0; i < sz; ++i) { 46 | if(i) { 47 | log << ", "; 48 | } 49 | log << std::hex << int(v[i]); 50 | } 51 | if(v.size() > sz) { 52 | log << ", ..."; 53 | } 54 | log << ']' << std::endl; 55 | } 56 | 57 | #include "tests_common_inline.hpp" 58 | #include "tests_datasets_inline.hpp" 59 | #include "tests_random_data_inline.hpp" 60 | #include "tests_perf_inline.hpp" 61 | #include "tests_errors_inline.hpp" 62 | 63 | NTEST(test_version) { 64 | NTEST_SUPPRESS_UNUSED; 65 | log << " ECL Size Bitness: " << ECL_GetSizeBitness() << std::endl; 66 | log << " ECL Version: " << ECL_GetVersionNumber() << std::endl; 67 | log << " ECL Version String: " << ECL_GetVersionString() << std::endl; 68 | log << " ECL Version Branch: " << ECL_GetVersionBranch() << std::endl; 69 | approve(true); 70 | } 71 | -------------------------------------------------------------------------------- /tests/tests_common_inline.hpp: -------------------------------------------------------------------------------- 1 | #include "../ECL_JH_States.h" 2 | #include "../ECL_utils.h" 3 | #include "ntest/ntest.h" 4 | 5 | #include 6 | 7 | #define ECL_TEST_E_NEXT_VALUE(value) (((value) * 128) / 127 + 1) 8 | 9 | #define ECL_TEST_E_NUMBER_GENERIC(write_func, read_func, eval_func) \ 10 | { \ 11 | const uint64_t limit = 0x0FFFFFFFFFFFFFFLL & ((1ULL << (ECL_SIZE_TYPE_BITS_COUNT - 1)) - 1ULL); \ 12 | ECL_JH_WState wstate; \ 13 | ECL_JH_RState rstate; \ 14 | ECL_usize n_bits = 0; \ 15 | for(uint64_t value = 0; value < limit; value = ECL_TEST_E_NEXT_VALUE(value)) { \ 16 | n_bits += eval_func(value); \ 17 | } \ 18 | auto n_bytes = (n_bits / 8) + 1; \ 19 | auto data = (uint8_t*)malloc(n_bytes); \ 20 | ECL_TEST_ASSERT(data); \ 21 | const auto n_repeats = (BoundVMinMax(depth + 10, 0, 1010) + 2); \ 22 | for(int i = 0; i < n_repeats; ++i) { \ 23 | ECL_JH_WInit(&wstate, data, n_bytes, 0); \ 24 | for(uint64_t value = 0; value < limit; value = ECL_TEST_E_NEXT_VALUE(value)) { \ 25 | write_func(&wstate, value); \ 26 | } \ 27 | ECL_TEST_ASSERT(wstate.is_valid); \ 28 | ECL_JH_RInit(&rstate, data, n_bytes, 0); \ 29 | for(uint64_t value = 0; value < limit; value = ECL_TEST_E_NEXT_VALUE(value)) { \ 30 | auto retrieved = read_func(&rstate); \ 31 | ECL_TEST_COMPARE((ECL_usize)value, retrieved); \ 32 | } \ 33 | ECL_TEST_ASSERT(rstate.is_valid); \ 34 | ECL_TEST_ASSERT(wstate.byte == rstate.byte); \ 35 | ECL_TEST_ASSERT(wstate.next == rstate.next); \ 36 | ECL_TEST_ASSERT(wstate.n_bits == rstate.n_bits); \ 37 | } \ 38 | free(data); \ 39 | } 40 | 41 | NTEST(test_number_E7_helpers) { 42 | NTEST_SUPPRESS_UNUSED; 43 | { 44 | const uint64_t limit = 0x0FFFFFFFFFFFFFFLL & ((1ULL << (ECL_SIZE_TYPE_BITS_COUNT - 1)) - 1ULL); 45 | ECL_usize n_bits = 0; 46 | for(uint64_t value = 0; value < limit; value = ECL_TEST_E_NEXT_VALUE(value)) { 47 | n_bits += ECL_Evaluate_E7(value); 48 | } 49 | const auto n_bytes = (n_bits / 8) + 1; 50 | auto data = (uint8_t*)malloc(n_bytes); 51 | const auto data_end = data + n_bytes; 52 | ECL_TEST_ASSERT(data); 53 | const auto n_repeats = (BoundVMinMax(depth + 10, 0, 1010) + 2); 54 | for(int i = 0; i < n_repeats; ++i) { 55 | uint8_t* next_wr = data; 56 | for(uint64_t value = 0; value < limit; value = ECL_TEST_E_NEXT_VALUE(value)) { 57 | next_wr = ECL_Helper_WriteE7(next_wr, data_end - next_wr, value); 58 | ECL_TEST_ASSERT(next_wr); 59 | } 60 | auto finished_wr = next_wr; 61 | // read back 62 | const uint8_t* next_rd = data; 63 | for(uint64_t value = 0; value < limit; value = ECL_TEST_E_NEXT_VALUE(value)) { 64 | ECL_usize retrieved = 0; 65 | next_rd = ECL_Helper_ReadE7(next_rd, data_end - next_rd, &retrieved); 66 | ECL_TEST_ASSERT(next_rd); 67 | ECL_TEST_COMPARE((ECL_usize)value, retrieved); 68 | } 69 | auto finished_rd = next_rd; 70 | // 71 | ECL_TEST_ASSERT(finished_wr <= data_end); 72 | ECL_TEST_COMPARE((uintptr_t)finished_wr, (uintptr_t)finished_rd); 73 | } 74 | free(data); 75 | } 76 | } 77 | 78 | NTEST(test_number_E7) { 79 | NTEST_SUPPRESS_UNUSED; 80 | NTEST_REQUIRE_DEPTH_ABOVE(0); 81 | ECL_TEST_E_NUMBER_GENERIC(ECL_JH_Write_E7, ECL_JH_Read_E7, ECL_Evaluate_E7); 82 | } 83 | 84 | NTEST(test_number_E6) { 85 | NTEST_SUPPRESS_UNUSED; 86 | NTEST_REQUIRE_DEPTH_ABOVE(0); 87 | ECL_TEST_E_NUMBER_GENERIC(ECL_JH_Write_E6, ECL_JH_Read_E6, ECL_Evaluate_E6); 88 | } 89 | 90 | NTEST(test_number_E5) { 91 | NTEST_SUPPRESS_UNUSED; 92 | NTEST_REQUIRE_DEPTH_ABOVE(-1); 93 | ECL_TEST_E_NUMBER_GENERIC(ECL_JH_Write_E5, ECL_JH_Read_E5, ECL_Evaluate_E5); 94 | } 95 | 96 | NTEST(test_number_E4) { 97 | NTEST_SUPPRESS_UNUSED; 98 | ECL_TEST_E_NUMBER_GENERIC(ECL_JH_Write_E4, ECL_JH_Read_E4, ECL_Evaluate_E4); 99 | } 100 | 101 | NTEST(test_number_E3) { 102 | NTEST_REQUIRE_DEPTH_ABOVE(-1); 103 | NTEST_SUPPRESS_UNUSED; 104 | ECL_TEST_E_NUMBER_GENERIC(ECL_JH_Write_E3, ECL_JH_Read_E3, ECL_Evaluate_E3); 105 | } 106 | 107 | NTEST(test_number_E2) { 108 | NTEST_SUPPRESS_UNUSED; 109 | ECL_TEST_E_NUMBER_GENERIC(ECL_JH_Write_E2, ECL_JH_Read_E2, ECL_Evaluate_E2); 110 | } 111 | 112 | NTEST(test_number_E7E4) { 113 | NTEST_SUPPRESS_UNUSED; 114 | NTEST_REQUIRE_DEPTH_ABOVE(0); 115 | ECL_TEST_E_NUMBER_GENERIC(ECL_JH_Write_E7E4, ECL_JH_Read_E7E4, ECL_Evaluate_E7E4); 116 | } 117 | 118 | NTEST(test_number_E6E3) { 119 | NTEST_SUPPRESS_UNUSED; 120 | ECL_TEST_E_NUMBER_GENERIC(ECL_JH_Write_E6E3, ECL_JH_Read_E6E3, ECL_Evaluate_E6E3); 121 | } 122 | 123 | NTEST(test_number_E4E5) { 124 | NTEST_SUPPRESS_UNUSED; 125 | NTEST_REQUIRE_DEPTH_ABOVE(-1); 126 | ECL_TEST_E_NUMBER_GENERIC(ECL_JH_Write_E4E5, ECL_JH_Read_E4E5, ECL_Evaluate_E4E5); 127 | } 128 | 129 | #undef ECL_TEST_E_NEXT_VALUE 130 | 131 | NTEST(test_JH_generic) { 132 | NTEST_SUPPRESS_UNUSED; 133 | uint8_t bit_counts[] = {1,2,3,4,4,5,6,7,8}; 134 | uint8_t data[5]; 135 | ECL_JH_WState wstate; 136 | ECL_JH_RState rstate; 137 | do { 138 | ECL_JH_WInit(&wstate, data, 5, 0); 139 | ECL_JH_RInit(&rstate, data, 5, 0); 140 | uint8_t value = 0; 141 | for(auto n_bits : bit_counts) { 142 | ECL_JH_Write(&wstate, value, n_bits); 143 | value = ~value; 144 | } 145 | value = 0; 146 | for(auto n_bits : bit_counts) { 147 | auto retrieved = ECL_JH_Read(&rstate, n_bits); 148 | uint8_t expected = value & ((1 << n_bits) - 1); 149 | ECL_TEST_ASSERT(retrieved == expected); 150 | value = ~value; 151 | } 152 | ECL_TEST_ASSERT(wstate.is_valid); 153 | ECL_TEST_ASSERT(rstate.is_valid); 154 | ECL_TEST_ASSERT(wstate.byte == rstate.byte); 155 | ECL_TEST_ASSERT(wstate.next == rstate.next); 156 | ECL_TEST_ASSERT(wstate.n_bits == rstate.n_bits); 157 | } while(std::next_permutation(std::begin(bit_counts), std::end(bit_counts))); 158 | } 159 | 160 | NTEST(test_JH_writing_failures) { 161 | NTEST_SUPPRESS_UNUSED; 162 | uint8_t data[6]; 163 | ECL_JH_WState state; 164 | { 165 | data[1] = 0xFF; 166 | ECL_JH_WInit(&state, data, 1, 0); 167 | ECL_TEST_ASSERT(state.is_valid); 168 | ECL_JH_Write(&state, 0, 8); 169 | ECL_TEST_ASSERT(state.is_valid); 170 | ECL_JH_Write(&state, 0, 1); 171 | ECL_TEST_ASSERT(! state.is_valid); 172 | ECL_JH_Write(&state, 0, 1); 173 | ECL_TEST_ASSERT(! state.is_valid); 174 | ECL_TEST_ASSERT(data[1] == uint8_t(0xFF)); 175 | } 176 | { 177 | data[1] = 0xFF; 178 | ECL_JH_WInit(&state, data, 1, 0); 179 | ECL_TEST_ASSERT(state.is_valid); 180 | ECL_JH_Write(&state, 0, 7); 181 | ECL_TEST_ASSERT(state.is_valid); 182 | ECL_JH_Write(&state, 0, 1); 183 | ECL_TEST_ASSERT(state.is_valid); 184 | ECL_JH_Write(&state, 0, 1); 185 | ECL_TEST_ASSERT(! state.is_valid); 186 | ECL_JH_Write(&state, 0, 1); 187 | ECL_TEST_ASSERT(! state.is_valid); 188 | ECL_TEST_ASSERT(data[1] == uint8_t(0xFF)); 189 | } 190 | { 191 | data[2] = 0xFF; 192 | ECL_JH_WInit(&state, data, 2, 0); 193 | ECL_TEST_ASSERT(state.is_valid); 194 | ECL_JH_Write(&state, 0, 8); 195 | ECL_TEST_ASSERT(state.is_valid); 196 | ECL_JH_Write(&state, 0, 8); 197 | ECL_TEST_ASSERT(state.is_valid); 198 | ECL_JH_Write(&state, 0, 1); 199 | ECL_TEST_ASSERT(! state.is_valid); 200 | ECL_JH_Write(&state, 0, 1); 201 | ECL_TEST_ASSERT(! state.is_valid); 202 | ECL_TEST_ASSERT(data[2] == uint8_t(0xFF)); 203 | } 204 | { 205 | data[2] = 0xFF; 206 | ECL_JH_WInit(&state, data, 2, 0); 207 | ECL_TEST_ASSERT(state.is_valid); 208 | ECL_JH_Write(&state, 0, 7); 209 | ECL_TEST_ASSERT(state.is_valid); 210 | ECL_JH_Write(&state, 0, 7); 211 | ECL_TEST_ASSERT(state.is_valid); 212 | ECL_JH_Write(&state, 0, 2); 213 | ECL_TEST_ASSERT(state.is_valid); 214 | ECL_JH_Write(&state, 0, 1); 215 | ECL_TEST_ASSERT(! state.is_valid); 216 | ECL_JH_Write(&state, 0, 1); 217 | ECL_TEST_ASSERT(! state.is_valid); 218 | ECL_TEST_ASSERT(data[2] == uint8_t(0xFF)); 219 | } 220 | { 221 | data[2] = 0xFF; 222 | ECL_JH_WInit(&state, data, 2, 0); 223 | ECL_TEST_ASSERT(state.is_valid); 224 | ECL_JH_Write(&state, 0, 7); 225 | ECL_TEST_ASSERT(state.is_valid); 226 | ECL_JH_Write(&state, 0, 7); 227 | ECL_TEST_ASSERT(state.is_valid); 228 | ECL_JH_Write(&state, 0, 3); 229 | ECL_TEST_ASSERT(! state.is_valid); 230 | ECL_JH_Write(&state, 0, 3); 231 | ECL_TEST_ASSERT(! state.is_valid); 232 | ECL_TEST_ASSERT(data[2] == uint8_t(0xFF)); 233 | } 234 | { 235 | data[2] = 0xFF; 236 | ECL_JH_WInit(&state, data, 2, 0); 237 | ECL_TEST_ASSERT(state.is_valid); 238 | ECL_JH_Write(&state, 0, 7); 239 | ECL_TEST_ASSERT(state.is_valid); 240 | ECL_JH_Write(&state, 0, 7); 241 | ECL_TEST_ASSERT(state.is_valid); 242 | ECL_JH_Write(&state, 0, 8); 243 | ECL_TEST_ASSERT(! state.is_valid); 244 | ECL_JH_Write(&state, 0, 8); 245 | ECL_TEST_ASSERT(! state.is_valid); 246 | ECL_TEST_ASSERT(data[2] == uint8_t(0xFF)); 247 | } 248 | { 249 | data[5] = 0xFF; 250 | ECL_JH_WInit(&state, data, 5, 0); 251 | ECL_TEST_ASSERT(state.is_valid); 252 | ECL_JH_Write(&state, 0, 5); 253 | ECL_TEST_ASSERT(state.is_valid); 254 | ECL_JH_WJump(&state, 4); 255 | ECL_TEST_ASSERT(state.is_valid); 256 | ECL_JH_Write(&state, 0, 2); 257 | ECL_TEST_ASSERT(state.is_valid); 258 | ECL_JH_Write(&state, 0, 2); 259 | ECL_TEST_ASSERT(! state.is_valid); 260 | ECL_JH_Write(&state, 0, 2); 261 | ECL_TEST_ASSERT(! state.is_valid); 262 | ECL_TEST_ASSERT(data[5] == uint8_t(0xFF)); 263 | } 264 | { 265 | data[5] = 0xFF; 266 | ECL_JH_WInit(&state, data, 5, 0); 267 | ECL_TEST_ASSERT(state.is_valid); 268 | ECL_JH_Write(&state, 0, 5); 269 | ECL_TEST_ASSERT(state.is_valid); 270 | ECL_JH_WJump(&state, 3); 271 | ECL_TEST_ASSERT(state.is_valid); 272 | ECL_JH_Write(&state, 0, 7); 273 | ECL_TEST_ASSERT(state.is_valid); 274 | ECL_JH_Write(&state, 0, 4); 275 | ECL_TEST_ASSERT(state.is_valid); 276 | ECL_JH_Write(&state, 0, 1); 277 | ECL_TEST_ASSERT(! state.is_valid); 278 | ECL_JH_Write(&state, 0, 1); 279 | ECL_TEST_ASSERT(! state.is_valid); 280 | ECL_TEST_ASSERT(data[5] == uint8_t(0xFF)); 281 | } 282 | { 283 | data[5] = 0xFF; 284 | ECL_JH_WInit(&state, data, 5, 0); 285 | ECL_TEST_ASSERT(state.is_valid); 286 | ECL_JH_WJump(&state, 4); 287 | ECL_TEST_ASSERT(state.is_valid); 288 | ECL_JH_WJump(&state, 1); 289 | ECL_TEST_ASSERT(! state.is_valid); 290 | ECL_JH_Write(&state, 0, 8); 291 | ECL_TEST_ASSERT(! state.is_valid); 292 | ECL_TEST_ASSERT(data[5] == uint8_t(0xFF)); 293 | } 294 | { 295 | data[5] = 0xFF; 296 | ECL_JH_WInit(&state, data, 5, 0); 297 | ECL_TEST_ASSERT(state.is_valid); 298 | ECL_JH_WJump(&state, 5); 299 | ECL_TEST_ASSERT(! state.is_valid); 300 | ECL_JH_WJump(&state, 1); 301 | ECL_TEST_ASSERT(! state.is_valid); 302 | ECL_JH_Write(&state, 0, 8); 303 | ECL_TEST_ASSERT(! state.is_valid); 304 | ECL_TEST_ASSERT(data[5] == uint8_t(0xFF)); 305 | } 306 | } 307 | 308 | NTEST(test_JH_reading_failures) { 309 | NTEST_SUPPRESS_UNUSED; 310 | uint8_t data[6]; 311 | ECL_JH_RState state; 312 | { 313 | memset(data, 0, sizeof(data)); 314 | data[1] = 0xFF; 315 | ECL_JH_RInit(&state, data, 1, 0); 316 | ECL_TEST_ASSERT(state.is_valid); 317 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 8)); 318 | ECL_TEST_ASSERT(state.is_valid); 319 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 1)); 320 | ECL_TEST_ASSERT(! state.is_valid); 321 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 1)); 322 | ECL_TEST_ASSERT(! state.is_valid); 323 | } 324 | { 325 | memset(data, 0, sizeof(data)); 326 | data[1] = 0xFF; 327 | ECL_JH_RInit(&state, data, 1, 0); 328 | ECL_TEST_ASSERT(state.is_valid); 329 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 7)); 330 | ECL_TEST_ASSERT(state.is_valid); 331 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 1)); 332 | ECL_TEST_ASSERT(state.is_valid); 333 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 1)); 334 | ECL_TEST_ASSERT(! state.is_valid); 335 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 1)); 336 | ECL_TEST_ASSERT(! state.is_valid); 337 | } 338 | { 339 | memset(data, 0, sizeof(data)); 340 | data[2] = 0xFF; 341 | ECL_JH_RInit(&state, data, 2, 0); 342 | ECL_TEST_ASSERT(state.is_valid); 343 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 8)); 344 | ECL_TEST_ASSERT(state.is_valid); 345 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 8)); 346 | ECL_TEST_ASSERT(state.is_valid); 347 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 1)); 348 | ECL_TEST_ASSERT(! state.is_valid); 349 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 1)); 350 | ECL_TEST_ASSERT(! state.is_valid); 351 | } 352 | { 353 | memset(data, 0, sizeof(data)); 354 | data[2] = 0xFF; 355 | ECL_JH_RInit(&state, data, 2, 0); 356 | ECL_TEST_ASSERT(state.is_valid); 357 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 7)); 358 | ECL_TEST_ASSERT(state.is_valid); 359 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 7)); 360 | ECL_TEST_ASSERT(state.is_valid); 361 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 2)); 362 | ECL_TEST_ASSERT(state.is_valid); 363 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 1)); 364 | ECL_TEST_ASSERT(! state.is_valid); 365 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 1)); 366 | ECL_TEST_ASSERT(! state.is_valid); 367 | } 368 | { 369 | memset(data, 0, sizeof(data)); 370 | data[2] = 0xFF; 371 | ECL_JH_RInit(&state, data, 2, 0); 372 | ECL_TEST_ASSERT(state.is_valid); 373 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 7)); 374 | ECL_TEST_ASSERT(state.is_valid); 375 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 7)); 376 | ECL_TEST_ASSERT(state.is_valid); 377 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 3)); 378 | ECL_TEST_ASSERT(! state.is_valid); 379 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 3)); 380 | ECL_TEST_ASSERT(! state.is_valid); 381 | } 382 | { 383 | memset(data, 0, sizeof(data)); 384 | data[2] = 0xFF; 385 | ECL_JH_RInit(&state, data, 2, 0); 386 | ECL_TEST_ASSERT(state.is_valid); 387 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 7)); 388 | ECL_TEST_ASSERT(state.is_valid); 389 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 7)); 390 | ECL_TEST_ASSERT(state.is_valid); 391 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 8)); 392 | ECL_TEST_ASSERT(! state.is_valid); 393 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 8)); 394 | ECL_TEST_ASSERT(! state.is_valid); 395 | } 396 | { 397 | memset(data, 0, sizeof(data)); 398 | data[5] = 0xFF; 399 | ECL_JH_RInit(&state, data, 5, 0); 400 | ECL_TEST_ASSERT(state.is_valid); 401 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 5)); 402 | ECL_TEST_ASSERT(state.is_valid); 403 | ECL_JH_RJump(&state, 4); 404 | ECL_TEST_ASSERT(state.is_valid); 405 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 2)); 406 | ECL_TEST_ASSERT(state.is_valid); 407 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 2)); 408 | ECL_TEST_ASSERT(! state.is_valid); 409 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 2)); 410 | ECL_TEST_ASSERT(! state.is_valid); 411 | } 412 | { 413 | memset(data, 0, sizeof(data)); 414 | data[5] = 0xFF; 415 | ECL_JH_RInit(&state, data, 5, 0); 416 | ECL_TEST_ASSERT(state.is_valid); 417 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 5)); 418 | ECL_TEST_ASSERT(state.is_valid); 419 | ECL_JH_RJump(&state, 3); 420 | ECL_TEST_ASSERT(state.is_valid); 421 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 7)); 422 | ECL_TEST_ASSERT(state.is_valid); 423 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 4)); 424 | ECL_TEST_ASSERT(state.is_valid); 425 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 1)); 426 | ECL_TEST_ASSERT(! state.is_valid); 427 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 1)); 428 | ECL_TEST_ASSERT(! state.is_valid); 429 | } 430 | { 431 | memset(data, 0, sizeof(data)); 432 | data[5] = 0xFF; 433 | ECL_JH_RInit(&state, data, 5, 0); 434 | ECL_TEST_ASSERT(state.is_valid); 435 | ECL_JH_RJump(&state, 4); 436 | ECL_TEST_ASSERT(state.is_valid); 437 | ECL_JH_RJump(&state, 1); 438 | ECL_TEST_ASSERT(! state.is_valid); 439 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 8)); 440 | ECL_TEST_ASSERT(! state.is_valid); 441 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 8)); 442 | ECL_TEST_ASSERT(! state.is_valid); 443 | } 444 | { 445 | memset(data, 0, sizeof(data)); 446 | data[5] = 0xFF; 447 | ECL_JH_RInit(&state, data, 5, 0); 448 | ECL_TEST_ASSERT(state.is_valid); 449 | ECL_JH_RJump(&state, 5); 450 | ECL_TEST_ASSERT(! state.is_valid); 451 | ECL_JH_RJump(&state, 1); 452 | ECL_TEST_ASSERT(! state.is_valid); 453 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 8)); 454 | ECL_TEST_ASSERT(! state.is_valid); 455 | ECL_TEST_ASSERT(0 == ECL_JH_Read(&state, 8)); 456 | ECL_TEST_ASSERT(! state.is_valid); 457 | } 458 | } 459 | 460 | 461 | NTEST(test_string_constants) { 462 | NTEST_SUPPRESS_UNUSED; 463 | // test some weird stuff used in other tests for comfortability 464 | approve(char(3) == "\x3\x5"[0]); 465 | approve(char(5) == "\x3\x5"[1]); 466 | approve(char(5) == "\x0\x5"[1]); 467 | approve(char(0x70) == "\x70\x75"[0]); 468 | approve(char(0x75) == "\x70\x75"[1]); 469 | } 470 | 471 | NTEST(test_ECL_LogSize) { 472 | NTEST_SUPPRESS_UNUSED; 473 | ECL_TEST_COMPARE(ECL_LogRoundUp(0), 1); 474 | ECL_TEST_COMPARE(ECL_LogRoundUp(1), 1); 475 | ECL_TEST_COMPARE(ECL_LogRoundUp(2), 1); 476 | ECL_TEST_COMPARE(ECL_LogRoundUp(3), 2); 477 | ECL_TEST_COMPARE(ECL_LogRoundUp(4), 2); 478 | ECL_TEST_COMPARE(ECL_LogRoundUp(5), 3); 479 | ECL_TEST_COMPARE(ECL_LogRoundUp(8), 3); 480 | ECL_TEST_COMPARE(ECL_LogRoundUp(9), 4); 481 | ECL_TEST_COMPARE(ECL_LogRoundUp(16), 4); 482 | ECL_TEST_COMPARE(ECL_LogRoundUp(17), 5); 483 | ECL_TEST_COMPARE(ECL_LogRoundUp(32), 5); 484 | ECL_TEST_COMPARE(ECL_LogRoundUp(33), 6); 485 | ECL_TEST_COMPARE(ECL_LogRoundUp(64), 6); 486 | ECL_TEST_COMPARE(ECL_LogRoundUp(65), 7); 487 | ECL_TEST_COMPARE(ECL_LogRoundUp(128), 7); 488 | ECL_TEST_COMPARE(ECL_LogRoundUp(129), 8); 489 | } 490 | 491 | NTEST(test_ECL_GetAlignedPointer2) { 492 | NTEST_SUPPRESS_UNUSED; 493 | typedef uint16_t Ty; 494 | const int type_size = sizeof(Ty); 495 | const int buf_size = 20; 496 | uint8_t tmp[buf_size]; 497 | const int shift = ((uintptr_t)tmp) & 1; 498 | auto ptr = tmp + shift; 499 | const auto first = (Ty*)(ptr); 500 | const auto next = (Ty*)(ptr + type_size); 501 | ECL_TEST_COMPARE(ECL_GetAlignedPointer2(ptr), first); 502 | ECL_TEST_COMPARE(ECL_GetAlignedPointer2(ptr + 1), next); 503 | ECL_TEST_COMPARE(ECL_GetAlignedPointer2(ptr + 2), next); 504 | for(int i = 0; i < (buf_size - type_size); ++i) { 505 | auto p = ECL_GetAlignedPointer2(tmp + i); 506 | approve((uintptr_t(p) & 1) == 0); 507 | } 508 | } 509 | 510 | NTEST(test_ECL_GetAlignedPointerS) { 511 | NTEST_SUPPRESS_UNUSED; 512 | typedef ECL_usize Ty; 513 | const int type_size = sizeof(Ty); 514 | const int buf_size = 20; 515 | uint8_t tmp[buf_size]; 516 | const int shift = sizeof(Ty) - (((uintptr_t)tmp) & (type_size - 1)); 517 | auto ptr = tmp + shift; 518 | const auto first = (Ty*)(ptr); 519 | const auto next = (Ty*)(ptr + type_size); 520 | ECL_TEST_COMPARE(ECL_GetAlignedPointerS(ptr), first); 521 | ECL_TEST_COMPARE(ECL_GetAlignedPointerS(ptr + 1), next); 522 | ECL_TEST_COMPARE(ECL_GetAlignedPointerS(ptr + 2), next); 523 | if(sizeof(Ty) > 2) { 524 | ECL_TEST_COMPARE(ECL_GetAlignedPointerS(ptr + 3), next); 525 | ECL_TEST_COMPARE(ECL_GetAlignedPointerS(ptr + 4), next); 526 | if(sizeof(Ty) > 4) { 527 | ECL_TEST_COMPARE(ECL_GetAlignedPointerS(ptr + 5), next); 528 | ECL_TEST_COMPARE(ECL_GetAlignedPointerS(ptr + 6), next); 529 | ECL_TEST_COMPARE(ECL_GetAlignedPointerS(ptr + 7), next); 530 | ECL_TEST_COMPARE(ECL_GetAlignedPointerS(ptr + 8), next); 531 | } 532 | } 533 | for(int i = 0; i < (buf_size - type_size); ++i) { 534 | auto p = ECL_GetAlignedPointerS(tmp + i); 535 | approve((uintptr_t(p) & (type_size - 1)) == 0); 536 | } 537 | } 538 | -------------------------------------------------------------------------------- /tests/tests_datasets_inline.hpp: -------------------------------------------------------------------------------- 1 | #include "../ECL_ZeroEater.h" 2 | #include "../ECL_ZeroDevourer.h" 3 | #include "../ECL_NanoLZ.h" 4 | #include "ntest/ntest.h" 5 | 6 | #include 7 | #include 8 | 9 | struct ECLDatasetRecord { 10 | const char* ptr; 11 | int length; 12 | }; 13 | 14 | typedef std::vector DatasetRecords; 15 | static DatasetRecords& GetDatasetRecords() { 16 | static DatasetRecords recs; 17 | return recs; 18 | } 19 | 20 | #define ECL_TEST_APPEND_DATASET(the_string) \ 21 | GetDatasetRecords().push_back({the_string, sizeof(the_string) - 1}); \ 22 | 23 | struct ECLTestDatasetsInitializer { 24 | ECLTestDatasetsInitializer() { 25 | // common datasets (basically for Zero-oriented algorithms) 26 | ECL_TEST_APPEND_DATASET("\x0"); 27 | ECL_TEST_APPEND_DATASET("\x1"); 28 | ECL_TEST_APPEND_DATASET("\x1\x1"); 29 | ECL_TEST_APPEND_DATASET("\x0\x0"); 30 | ECL_TEST_APPEND_DATASET("\x0\x0\x0"); 31 | ECL_TEST_APPEND_DATASET("\x0\x0\x0\x0"); 32 | ECL_TEST_APPEND_DATASET("\x0\x0\x0\x0\x0"); 33 | ECL_TEST_APPEND_DATASET("\x0\x1\x0\x0\x0"); 34 | ECL_TEST_APPEND_DATASET("\x0\x0\x1\x0\x0"); 35 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x0"); 36 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1"); 37 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x0"); 38 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0"); 39 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x1"); 40 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x1"); 41 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x1"); 42 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x0\x1"); 43 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x1"); 44 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x0\x1"); 45 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x0\x0\x1"); 46 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x0\x0\x0\x1"); 47 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x0\x0\x0\x0\x1"); 48 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x1"); 49 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x1"); 50 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x1"); 51 | ECL_TEST_APPEND_DATASET("\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x1\x0"); 52 | ECL_TEST_APPEND_DATASET("\x0\x0\x0\x0\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x1\x0"); 53 | ECL_TEST_APPEND_DATASET("\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x1\x1\x0\x1\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x1\x0"); 54 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1"); 55 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x0"); 56 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x0\x1"); 57 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x0\x0\x1"); 58 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x1\x1\x1\x1"); 59 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 60 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 61 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x0"); 62 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x0\x1"); 63 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x0\x0\x1"); 64 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x0\x0\x1\x0"); 65 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x0\x0\x1\x0\x1"); 66 | // LZ datasets 67 | ECL_TEST_APPEND_DATASET("\x1\x2\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 68 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 69 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 70 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 71 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x6\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 72 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x6\x7\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 73 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x6\x7\x8\x1\x1\x1\x1\x1\x1\x1\x1"); 74 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x6\x7\x8\x9\x1\x1\x1\x1\x1\x1\x1"); 75 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\x1\x1\x1\x1\x1\x1"); 76 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\x1\x1\x1\x1\x1"); 77 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\xC\x1\x1\x1\x1"); 78 | // similar 1 79 | ECL_TEST_APPEND_DATASET("\x1\x1\x2\x3\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 80 | ECL_TEST_APPEND_DATASET("\x1\x1\x2\x3\x4\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 81 | ECL_TEST_APPEND_DATASET("\x1\x1\x2\x3\x4\x5\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 82 | ECL_TEST_APPEND_DATASET("\x1\x1\x2\x3\x4\x5\x6\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 83 | ECL_TEST_APPEND_DATASET("\x1\x1\x2\x3\x4\x5\x6\x7\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 84 | ECL_TEST_APPEND_DATASET("\x1\x1\x2\x3\x4\x5\x6\x7\x8\x1\x1\x1\x1\x1\x1\x1\x1"); 85 | ECL_TEST_APPEND_DATASET("\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\x1\x1\x1\x1\x1\x1\x1"); 86 | ECL_TEST_APPEND_DATASET("\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\x1\x1\x1\x1\x1\x1"); 87 | ECL_TEST_APPEND_DATASET("\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\x1\x1\x1\x1\x1"); 88 | ECL_TEST_APPEND_DATASET("\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\xC\x1\x1\x1\x1"); 89 | // similar 2 90 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x2\x3\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 91 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x2\x3\x4\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 92 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x2\x3\x4\x5\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 93 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x2\x3\x4\x5\x6\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 94 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x2\x3\x4\x5\x6\x7\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 95 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x2\x3\x4\x5\x6\x7\x8\x1\x1\x1\x1\x1\x1\x1\x1"); 96 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\x1\x1\x1\x1\x1\x1\x1"); 97 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\x1\x1\x1\x1\x1\x1"); 98 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\x1\x1\x1\x1\x1"); 99 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\xC\x1\x1\x1\x1"); 100 | // similar 3 101 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x2\x3\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 102 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x2\x3\x4\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 103 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x2\x3\x4\x5\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 104 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x2\x3\x4\x5\x6\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 105 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x2\x3\x4\x5\x6\x7\x1\x1\x1\x1\x1\x1\x1\x1\x1"); 106 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x2\x3\x4\x5\x6\x7\x8\x1\x1\x1\x1\x1\x1\x1\x1"); 107 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\x1\x1\x1\x1\x1\x1\x1"); 108 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\x1\x1\x1\x1\x1\x1"); 109 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\x1\x1\x1\x1\x1"); 110 | ECL_TEST_APPEND_DATASET("\x1\x1\x1\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\xC\x1\x1\x1\x1"); 111 | // 112 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\xC\x1\x4\x5\x6\x7\x1\x1\x1"); 113 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\xC\x1\x1\x4\x5\x6\x7\x1\x1"); 114 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\xC\x1\x1\x1\x4\x5\x6\x7\x1"); 115 | ECL_TEST_APPEND_DATASET("\x1\x2\x3\x4\x5\x6\x7\x8\x9\xA\xB\xC\x1\x1\x1\x1\x4\x5\x6\x7"); 116 | } 117 | }; 118 | ECLTestDatasetsInitializer ECLTestDatasetsInitializer_instance; 119 | 120 | #undef ECL_TEST_APPEND_DATASET 121 | 122 | NTEST(test_ZeroEater_datasets) { 123 | NTEST_SUPPRESS_UNUSED; 124 | std::vector tmp; 125 | std::vector tmp_output; 126 | for(auto& rec : GetDatasetRecords()) { 127 | auto src_data = (const uint8_t*)rec.ptr; 128 | ECL_usize src_size = rec.length; 129 | approve(src_data); 130 | approve(src_size); 131 | 132 | auto enough_size = ECL_ZERO_EATER_GET_BOUND(src_size); 133 | tmp.resize(enough_size); 134 | auto comp_size = ECL_ZeroEater_Compress(src_data, src_size, tmp.data(), enough_size); 135 | approve(comp_size == ECL_ZeroEater_Compress(src_data, src_size, nullptr, 0)); 136 | 137 | tmp_output.resize(src_size); 138 | auto decomp_size = ECL_ZeroEater_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size); 139 | approve(decomp_size == src_size); 140 | approve(0 == memcmp(src_data, tmp_output.data(), src_size)); 141 | } 142 | } 143 | 144 | NTEST(test_ZeroDevourer_datasets) { 145 | NTEST_SUPPRESS_UNUSED; 146 | std::vector tmp; 147 | std::vector tmp_output; 148 | for(auto& rec : GetDatasetRecords()) { 149 | auto src_data = (const uint8_t*)rec.ptr; 150 | ECL_usize src_size = rec.length; 151 | approve(src_data); 152 | approve(src_size); 153 | 154 | auto enough_size = ECL_ZERO_DEVOURER_GET_BOUND(src_size); 155 | tmp.resize(enough_size); 156 | auto comp_size = ECL_ZeroDevourer_Compress(src_data, src_size, tmp.data(), enough_size); 157 | 158 | tmp_output.resize(src_size); 159 | auto decomp_size = ECL_ZeroDevourer_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size); 160 | approve(decomp_size == src_size); 161 | approve(0 == memcmp(src_data, tmp_output.data(), src_size)); 162 | } 163 | } 164 | 165 | NTEST(test_NanoLZ_slow_datasets) { 166 | NTEST_SUPPRESS_UNUSED; 167 | std::vector tmp; 168 | std::vector tmp_output; 169 | for(auto& rec : GetDatasetRecords()) { 170 | auto src_data = (const uint8_t*)rec.ptr; 171 | ECL_usize src_size = rec.length; 172 | approve(src_data); 173 | approve(src_size); 174 | 175 | auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 176 | tmp.resize(enough_size); 177 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 178 | auto comp_size = ECL_NanoLZ_Compress_slow(scheme, src_data, src_size, tmp.data(), enough_size, -1); 179 | tmp_output.resize(src_size); 180 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 181 | approve(decomp_size == src_size); 182 | approve(0 == memcmp(src_data, tmp_output.data(), src_size)); 183 | } 184 | } 185 | } 186 | 187 | NTEST(test_NanoLZ_fast1_datasets) { 188 | NTEST_SUPPRESS_UNUSED; 189 | std::vector tmp; 190 | std::vector tmp_output; 191 | 192 | ECL_NanoLZ_FastParams fp; 193 | ECL_NanoLZ_FastParams_Alloc1(&fp, 10); 194 | for(auto& rec : GetDatasetRecords()) { 195 | auto src_data = (const uint8_t*)rec.ptr; 196 | ECL_usize src_size = rec.length; 197 | approve(src_data); 198 | approve(src_size); 199 | 200 | auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 201 | tmp.resize(enough_size); 202 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 203 | auto comp_size = ECL_NanoLZ_Compress_fast1(scheme, src_data, src_size, tmp.data(), enough_size, -1, &fp); 204 | tmp_output.resize(src_size); 205 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 206 | approve(decomp_size == src_size); 207 | approve(0 == memcmp(src_data, tmp_output.data(), src_size)); 208 | } 209 | } 210 | ECL_NanoLZ_FastParams_Destroy(&fp); 211 | } 212 | 213 | NTEST(test_NanoLZ_fast2_datasets) { 214 | NTEST_SUPPRESS_UNUSED; 215 | std::vector tmp; 216 | std::vector tmp_output; 217 | 218 | ECL_NanoLZ_FastParams fp; 219 | ECL_NanoLZ_FastParams_Alloc2(&fp, 10); 220 | for(auto& rec : GetDatasetRecords()) { 221 | auto src_data = (const uint8_t*)rec.ptr; 222 | ECL_usize src_size = rec.length; 223 | approve(src_data); 224 | approve(src_size); 225 | 226 | auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 227 | tmp.resize(enough_size); 228 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 229 | auto comp_size = ECL_NanoLZ_Compress_fast2(scheme, src_data, src_size, tmp.data(), enough_size, -1, &fp); 230 | tmp_output.resize(src_size); 231 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 232 | approve(decomp_size == src_size); 233 | approve(0 == memcmp(src_data, tmp_output.data(), src_size)); 234 | } 235 | } 236 | ECL_NanoLZ_FastParams_Destroy(&fp); 237 | } 238 | 239 | NTEST(test_NanoLZ_mid1_datasets) { 240 | NTEST_SUPPRESS_UNUSED; 241 | std::vector tmp; 242 | std::vector tmp_output; 243 | uint8_t buf_x[256]; 244 | const int search_limits[] = {1, 2, 5, 10, -1}; 245 | 246 | for(auto& rec : GetDatasetRecords()) { 247 | auto src_data = (const uint8_t*)rec.ptr; 248 | ECL_usize src_size = rec.length; 249 | approve(src_data); 250 | approve(src_size); 251 | 252 | auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 253 | ECL_TEST_MAGIC_RESIZE(tmp, enough_size); 254 | for(auto limit : search_limits) { 255 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 256 | auto comp_size = ECL_NanoLZ_Compress_mid1(scheme, src_data, src_size, tmp.data(), enough_size, limit, buf_x); 257 | tmp_output.resize(src_size); 258 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 259 | approve(decomp_size == src_size); 260 | approve(0 == memcmp(src_data, tmp_output.data(), src_size)); 261 | ECL_TEST_MAGIC_VALIDATE(tmp); 262 | } 263 | } 264 | } 265 | } 266 | 267 | NTEST(test_NanoLZ_mid2_datasets) { 268 | NTEST_SUPPRESS_UNUSED; 269 | std::vector tmp; 270 | std::vector tmp_output; 271 | uint8_t buf_x[513]; 272 | const int search_limits[] = {1, 2, 5, 10, -1}; 273 | 274 | for(auto& rec : GetDatasetRecords()) { 275 | auto src_data = (const uint8_t*)rec.ptr; 276 | ECL_usize src_size = rec.length; 277 | approve(src_data); 278 | approve(src_size); 279 | 280 | auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 281 | ECL_TEST_MAGIC_RESIZE(tmp, enough_size); 282 | for(auto limit : search_limits) { 283 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 284 | auto comp_size = ECL_NanoLZ_Compress_mid2(scheme, src_data, src_size, tmp.data(), enough_size, limit, buf_x); 285 | tmp_output.resize(src_size); 286 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 287 | approve(decomp_size == src_size); 288 | approve(0 == memcmp(src_data, tmp_output.data(), src_size)); 289 | ECL_TEST_MAGIC_VALIDATE(tmp); 290 | } 291 | } 292 | } 293 | } 294 | 295 | ECL_usize ECL_Test_NanoLZ_CompressWith(ECL_NanoLZ_Scheme scheme, const std::vector& src, std::vector& preallocated_output, int mode, ECL_NanoLZ_FastParams& preallocated_params) { 296 | ECL_usize comp_size = 0; 297 | if(mode == 0) { 298 | comp_size = ECL_NanoLZ_Compress_slow(scheme, 299 | src.data(), src.size(), 300 | preallocated_output.data(), preallocated_output.size(), 301 | -1); 302 | } else if(mode == 1) { 303 | comp_size = ECL_NanoLZ_Compress_fast1(scheme, 304 | src.data(), src.size(), 305 | preallocated_output.data(), preallocated_output.size(), 306 | -1, &preallocated_params); 307 | } else if(mode == 2) { 308 | comp_size = ECL_NanoLZ_Compress_fast2(scheme, 309 | src.data(), src.size(), 310 | preallocated_output.data(), preallocated_output.size(), 311 | -1, &preallocated_params); 312 | } 313 | preallocated_output.resize(comp_size); 314 | return comp_size; 315 | } 316 | 317 | bool ECL_Test_NanoLZ_OnLinearGenericData(std::ostream& log, int mode, ECL_NanoLZ_FastParams& preallocated_params) { 318 | (void)log; 319 | std::vector src; 320 | std::vector tmp; 321 | std::vector tmp_output; 322 | 323 | const int prefixes[] = {0, 1, 2, 10}; 324 | const int suffixes[] = {0, 1, 2, 10}; 325 | const int counts[] = {2, 3, 4, 5, 7, 8, 11}; 326 | const int new_ones[] = {0, 1, 2, 3, 4, 6, 8, 10, 18}; 327 | const int min_dist = 0; 328 | const int max_dist = 800; 329 | 330 | const int n_reserve = 1000; 331 | src.reserve(n_reserve); 332 | tmp.reserve(n_reserve); 333 | tmp_output.reserve(n_reserve); 334 | for(auto prefix_size : prefixes) { 335 | for(auto suffix_size : suffixes) { 336 | for(auto match_size : counts) { 337 | for(auto n_new : new_ones) { 338 | for(int dist = min_dist; dist < max_dist; ++dist) { 339 | // generate data 340 | const ECL_usize src_size = prefix_size + match_size*2 + dist + suffix_size; 341 | src.resize(src_size); 342 | auto next_random_value = match_size; 343 | auto next_ptr = src.data(); 344 | std::iota(next_ptr, next_ptr + prefix_size, next_random_value); // --- prefix 345 | next_ptr += prefix_size; 346 | next_random_value += prefix_size; 347 | std::iota(next_ptr, next_ptr + match_size, 0); // --- first string 348 | next_ptr += match_size; 349 | { // fill the gap 350 | auto n_generate = dist; 351 | if(n_new < dist) { // first, add a block that will hopefully get glued 352 | auto n_repeated = dist - n_new; 353 | std::fill(next_ptr, next_ptr + n_repeated, next_random_value); 354 | next_ptr += n_repeated; 355 | ++next_random_value; 356 | n_generate -= n_repeated; 357 | } 358 | std::iota(next_ptr, next_ptr + n_generate, next_random_value); 359 | next_ptr += n_generate; 360 | next_random_value += n_generate; 361 | } 362 | std::iota(next_ptr, next_ptr + match_size, 0); // --- second string - match 363 | next_ptr += match_size; 364 | std::iota(next_ptr, next_ptr + suffix_size, next_random_value); // --- suffix 365 | next_ptr += suffix_size; 366 | if(next_ptr != (src.data() + src_size)) { 367 | return false; 368 | } 369 | if(next_random_value > 255) { 370 | return false; // oops 371 | } 372 | 373 | // test 374 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 375 | tmp.resize(ECL_NANO_LZ_GET_BOUND(src_size)); 376 | ECL_usize comp_size = ECL_Test_NanoLZ_CompressWith(scheme, src, tmp, mode, preallocated_params); 377 | if(! comp_size) { 378 | return false; 379 | } 380 | tmp_output.resize(src_size); 381 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 382 | if(decomp_size != src_size) { 383 | return false; 384 | } 385 | if(memcmp(src.data(), tmp_output.data(), src_size)) { 386 | return false; 387 | } 388 | } 389 | } 390 | } 391 | } 392 | } 393 | } 394 | return true; 395 | } 396 | 397 | NTEST(test_NanoLZ_fast1_generic_datasets) { // use only for fast1 version, as it's optimal for target data sizes 398 | NTEST_SUPPRESS_UNUSED; 399 | ECL_NanoLZ_FastParams fp; 400 | ECL_NanoLZ_FastParams_Alloc1(&fp, 10); 401 | approve(ECL_Test_NanoLZ_OnLinearGenericData(log, 1, fp)); 402 | ECL_NanoLZ_FastParams_Destroy(&fp); 403 | } 404 | 405 | bool ECL_Test_NanoLZ_AllCompressorsAreEqual(const std::vector& src, std::ostream& log) { 406 | // checks that all compression modes with maximum depth of search end up with equal binary output 407 | using Vec = std::vector; 408 | ECL_NanoLZ_FastParams fp1; 409 | ECL_NanoLZ_FastParams_Alloc1(&fp1, 15); 410 | ECL_NanoLZ_FastParams fp2; 411 | ECL_NanoLZ_FastParams_Alloc2(&fp2, 15); 412 | bool result = true; 413 | 414 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 415 | Vec last; 416 | for(int mode = 0; mode < 3; ++mode) { 417 | Vec tmp; 418 | tmp.resize(ECL_NANO_LZ_GET_BOUND(src.size())); 419 | if(mode == 1) { 420 | ECL_Test_NanoLZ_CompressWith(scheme, src, tmp, mode, fp1); 421 | } else if(mode == 2) { 422 | ECL_Test_NanoLZ_CompressWith(scheme, src, tmp, mode, fp2); 423 | } else { 424 | ECL_Test_NanoLZ_CompressWith(scheme, src, tmp, mode, fp1); 425 | } 426 | if((! last.empty()) && (last != tmp)) { 427 | log << " mode=" << mode << " mismatches previous. " << last.size() << " -- " << tmp.size() << std::endl; 428 | result = false; 429 | break; 430 | } 431 | last = std::move(tmp); 432 | } 433 | if(! result) { 434 | break; 435 | } 436 | Vec decompressed; 437 | decompressed.resize(src.size()); 438 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, last.data(), last.size(), decompressed.data(), decompressed.size()); 439 | if(decomp_size != decompressed.size()) { 440 | result = false; 441 | break; 442 | } 443 | if(src != decompressed) { 444 | result = false; 445 | break; 446 | } 447 | } 448 | 449 | ECL_NanoLZ_FastParams_Destroy(&fp1); 450 | ECL_NanoLZ_FastParams_Destroy(&fp2); 451 | return result; 452 | } 453 | 454 | NTEST(test_NanoLZ_check_modes_equal_result) { 455 | NTEST_SUPPRESS_UNUSED; 456 | std::vector src; 457 | const int n_sets = 2 * (BoundVMinMax(depth + 10, 0, 1010) + 1); 458 | const int max_size = 10000; 459 | const int min_size = 1; 460 | const uint8_t masks[] = {0x3F, 0x07, 0x03, 0x01}; 461 | 462 | src.reserve(max_size); 463 | for(int i = 0; i < n_sets; ++i) { 464 | const auto src_size = (rand() % (max_size - min_size)) + min_size; 465 | src.clear(); 466 | src.resize(src_size); 467 | for(int j = 0; j < src_size; ++j) { 468 | src[j] = rand(); 469 | } 470 | 471 | for(auto mask : masks) { 472 | for(int j = 0; j < src_size; ++j) { 473 | src[j] &= mask; 474 | } 475 | approve( ECL_Test_NanoLZ_AllCompressorsAreEqual(src, log) ); 476 | } 477 | } 478 | } 479 | -------------------------------------------------------------------------------- /tests/tests_errors_inline.hpp: -------------------------------------------------------------------------------- 1 | #include "../ECL_ZeroEater.h" 2 | #include "../ECL_ZeroDevourer.h" 3 | #include "../ECL_NanoLZ.h" 4 | #include "ntest/ntest.h" 5 | 6 | #include 7 | 8 | NTEST(test_ZeroEater_cut_stream) { 9 | NTEST_SUPPRESS_UNUSED; 10 | std::vector src; 11 | std::vector tmp; 12 | std::vector tmp_output; 13 | const uint8_t masks[] = {0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01}; 14 | 15 | const ECL_usize src_size = 2000; 16 | src.resize(src_size); 17 | for(ECL_usize j = 0; j < src_size; ++j) { 18 | src[j] = rand(); 19 | } 20 | for(auto mask : masks) { 21 | for(ECL_usize j = 0; j < src_size; ++j) { 22 | src[j] &= mask; 23 | } 24 | const auto enough_size = ECL_ZERO_EATER_GET_BOUND(src_size); 25 | ECL_TEST_MAGIC_RESIZE(tmp, enough_size); 26 | auto comp_size = ECL_ZeroEater_Compress(src.data(), src_size, tmp.data(), enough_size); 27 | tmp_output.resize(src_size); 28 | auto decomp_size = ECL_ZeroEater_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size); 29 | approve(decomp_size == src_size); 30 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 31 | ECL_TEST_MAGIC_VALIDATE(tmp); 32 | // test cut stream - decompression has to fail correctly 33 | for(ECL_usize i = 1; i <= comp_size; ++i) { 34 | auto decomp_size = ECL_ZeroEater_Decompress(tmp.data(), comp_size - i, tmp_output.data(), src_size); 35 | approve(decomp_size < src_size); 36 | } 37 | } 38 | } 39 | 40 | NTEST(test_ZeroDevourer_cut_stream) { 41 | NTEST_SUPPRESS_UNUSED; 42 | std::vector src; 43 | std::vector tmp; 44 | std::vector tmp_output; 45 | const uint8_t masks[] = {0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01}; 46 | 47 | const ECL_usize src_size = 2000; 48 | src.resize(src_size); 49 | for(ECL_usize j = 0; j < src_size; ++j) { 50 | src[j] = rand(); 51 | } 52 | for(auto mask : masks) { 53 | for(ECL_usize j = 0; j < src_size; ++j) { 54 | src[j] &= mask; 55 | } 56 | const auto enough_size = ECL_ZERO_DEVOURER_GET_BOUND(src_size); 57 | ECL_TEST_MAGIC_RESIZE(tmp, enough_size); 58 | auto comp_size = ECL_ZeroDevourer_Compress(src.data(), src_size, tmp.data(), enough_size); 59 | tmp_output.resize(src_size); 60 | auto decomp_size = ECL_ZeroDevourer_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size); 61 | approve(decomp_size == src_size); 62 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 63 | ECL_TEST_MAGIC_VALIDATE(tmp); 64 | // test cut stream - decompression has to fail correctly 65 | for(ECL_usize i = 1; i <= comp_size; ++i) { 66 | auto decomp_size = ECL_ZeroDevourer_Decompress(tmp.data(), comp_size - i, tmp_output.data(), src_size); 67 | approve(! decomp_size); 68 | } 69 | } 70 | } 71 | 72 | NTEST(test_NanoLZ_cut_stream) { 73 | NTEST_SUPPRESS_UNUSED; 74 | std::vector src; 75 | std::vector tmp; 76 | std::vector tmp_output; 77 | const uint8_t masks[] = {0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01}; 78 | const int search_limits[] = {1, 2, 5, 10, -1}; 79 | 80 | const ECL_usize src_size = 2000; 81 | src.resize(src_size); 82 | for(ECL_usize j = 0; j < src_size; ++j) { 83 | src[j] = rand(); 84 | } 85 | ECL_NanoLZ_FastParams fp; 86 | ECL_NanoLZ_FastParams_Alloc1(&fp, 11); 87 | for(auto mask : masks) { 88 | for(ECL_usize j = 0; j < src_size; ++j) { 89 | src[j] &= mask; 90 | } 91 | const auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 92 | ECL_TEST_MAGIC_RESIZE(tmp, enough_size); 93 | for(auto limit : search_limits) { 94 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 95 | auto comp_size = ECL_NanoLZ_Compress_fast1(scheme, src.data(), src_size, tmp.data(), enough_size, limit, &fp); 96 | tmp_output.resize(src_size); 97 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 98 | approve(decomp_size == src_size); 99 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 100 | ECL_TEST_MAGIC_VALIDATE(tmp); 101 | // test cut stream - decompression has to fail correctly 102 | for(ECL_usize i = 1; i <= comp_size; ++i) { 103 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size - i, tmp_output.data(), src_size); 104 | approve(! decomp_size); 105 | } 106 | } 107 | } 108 | } 109 | ECL_NanoLZ_FastParams_Destroy(&fp); 110 | } 111 | 112 | NTEST(test_ZeroDevourer_insufficient_dst_compr) { 113 | NTEST_SUPPRESS_UNUSED; 114 | // ZeroEater compressor behaves differently for tested scenario so no such test provided 115 | std::vector src; 116 | std::vector tmp; 117 | std::vector tmp_output; 118 | const uint8_t masks[] = {0xFF, 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01}; 119 | const ECL_usize src_size = 8000; 120 | 121 | src.resize(src_size); 122 | for(ECL_usize j = 0; j < src_size; ++j) { 123 | src[j] = rand(); 124 | } 125 | for(auto mask : masks) { 126 | for(ECL_usize j = 0; j < src_size; ++j) { 127 | src[j] &= mask; 128 | } 129 | const auto enough_size = ECL_ZERO_DEVOURER_GET_BOUND(src_size); 130 | tmp.resize(enough_size); 131 | auto comp_size = ECL_ZeroDevourer_Compress(src.data(), src_size, tmp.data(), enough_size); 132 | tmp_output.resize(src_size); 133 | auto decomp_size = ECL_ZeroDevourer_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size); 134 | approve(decomp_size == src_size); 135 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 136 | // test with insufficient output buffer - compression has to fail correctly 137 | for(ECL_usize i = 1; i <= comp_size; ++i) { 138 | const auto output_size = comp_size - i; 139 | ECL_TEST_MAGIC_RESIZE(tmp, output_size); 140 | auto sz = ECL_ZeroDevourer_Compress(src.data(), src_size, tmp.data(), output_size); 141 | approve(! sz); 142 | ECL_TEST_MAGIC_VALIDATE(tmp); 143 | } 144 | } 145 | } 146 | 147 | NTEST(test_NanoLZ_insufficient_dst_compr) { 148 | NTEST_SUPPRESS_UNUSED; 149 | std::vector src; 150 | std::vector tmp; 151 | std::vector tmp_output; 152 | std::vector masks; 153 | const auto dep = BoundVMinMax(depth + 10, 0, 100); 154 | if(dep > 50) { 155 | masks = {0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01}; 156 | } else if(dep > 20) { 157 | masks = {0x3F, 0x0F, 0x07, 0x03, 0x01}; 158 | } else if(dep > 5) { 159 | masks = {0x3F, 0x0F, 0x03, 0x01}; 160 | } else if(dep > 0) { 161 | masks = {0x3F, 0x03, 0x01}; 162 | } else if(dep > -5) { 163 | masks = {0x3F, 0x03}; 164 | } else { 165 | masks = {0x3F}; 166 | } 167 | const int search_limits[] = {1, 5, -1}; 168 | const ECL_usize src_size = 2000; 169 | 170 | src.resize(src_size); 171 | for(ECL_usize j = 0; j < src_size; ++j) { 172 | src[j] = rand(); 173 | } 174 | ECL_NanoLZ_FastParams fp; 175 | ECL_NanoLZ_FastParams_Alloc1(&fp, 11); 176 | for(auto mask : masks) { 177 | for(ECL_usize j = 0; j < src_size; ++j) { 178 | src[j] &= mask; 179 | } 180 | const auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 181 | tmp.resize(enough_size); 182 | for(auto limit : search_limits) { 183 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 184 | auto comp_size = ECL_NanoLZ_Compress_fast1(scheme, src.data(), src_size, tmp.data(), enough_size, limit, &fp); 185 | tmp_output.resize(src_size); 186 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 187 | approve(decomp_size == src_size); 188 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 189 | // test with insufficient output buffer - compression has to fail correctly 190 | for(ECL_usize i = 1; i <= comp_size; ++i) { 191 | const auto output_size = comp_size - i; 192 | ECL_TEST_MAGIC_RESIZE(tmp, output_size); 193 | auto sz = ECL_NanoLZ_Compress_fast1(scheme, src.data(), src_size, tmp.data(), output_size, limit, &fp); 194 | approve(! sz); 195 | ECL_TEST_MAGIC_VALIDATE(tmp); 196 | } 197 | } 198 | } 199 | } 200 | ECL_NanoLZ_FastParams_Destroy(&fp); 201 | } 202 | 203 | NTEST(test_ZeroEater_insufficient_dst_decompr) { 204 | NTEST_SUPPRESS_UNUSED; 205 | std::vector src; 206 | std::vector tmp; 207 | std::vector tmp_output; 208 | const uint8_t masks[] = {0xFF, 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01}; 209 | const ECL_usize src_size = 8000; 210 | 211 | src.resize(src_size); 212 | for(ECL_usize j = 0; j < src_size; ++j) { 213 | src[j] = rand(); 214 | } 215 | for(auto mask : masks) { 216 | for(ECL_usize j = 0; j < src_size; ++j) { 217 | src[j] &= mask; 218 | } 219 | const auto enough_size = ECL_ZERO_EATER_GET_BOUND(src_size); 220 | tmp.resize(enough_size); 221 | auto comp_size = ECL_ZeroEater_Compress(src.data(), src_size, tmp.data(), enough_size); 222 | tmp_output.resize(src_size); 223 | auto decomp_size = ECL_ZeroEater_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size); 224 | approve(decomp_size == src_size); 225 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 226 | // test with insufficient output buffer - compression has to fail correctly 227 | for(ECL_usize i = 1; i <= src_size; ++i) { 228 | const auto output_size = src_size - i; 229 | ECL_TEST_MAGIC_RESIZE(tmp_output, output_size); 230 | auto sz = ECL_ZeroEater_Decompress(tmp.data(), comp_size, tmp_output.data(), output_size); 231 | approve(! sz); 232 | ECL_TEST_MAGIC_VALIDATE(tmp_output); 233 | } 234 | } 235 | } 236 | 237 | NTEST(test_ZeroDevourer_insufficient_dst_decompr) { 238 | NTEST_SUPPRESS_UNUSED; 239 | std::vector src; 240 | std::vector tmp; 241 | std::vector tmp_output; 242 | const uint8_t masks[] = {0xFF, 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01}; 243 | const ECL_usize src_size = 8000; 244 | 245 | src.resize(src_size); 246 | for(ECL_usize j = 0; j < src_size; ++j) { 247 | src[j] = rand(); 248 | } 249 | for(auto mask : masks) { 250 | for(ECL_usize j = 0; j < src_size; ++j) { 251 | src[j] &= mask; 252 | } 253 | const auto enough_size = ECL_ZERO_DEVOURER_GET_BOUND(src_size); 254 | tmp.resize(enough_size); 255 | auto comp_size = ECL_ZeroDevourer_Compress(src.data(), src_size, tmp.data(), enough_size); 256 | tmp_output.resize(src_size); 257 | auto decomp_size = ECL_ZeroDevourer_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size); 258 | approve(decomp_size == src_size); 259 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 260 | // test with insufficient output buffer - compression has to fail correctly 261 | for(ECL_usize i = 1; i <= src_size; ++i) { 262 | const auto output_size = src_size - i; 263 | ECL_TEST_MAGIC_RESIZE(tmp_output, output_size); 264 | auto sz = ECL_ZeroDevourer_Decompress(tmp.data(), comp_size, tmp_output.data(), output_size); 265 | if(sz) { // if succeeded - can be only a coincidence: output_size decompressed but last compressed byte is not fully consumed 266 | approve(sz == output_size); 267 | approve(0 == memcmp(src.data(), tmp_output.data(), output_size)); 268 | } 269 | ECL_TEST_MAGIC_VALIDATE(tmp_output); 270 | } 271 | } 272 | } 273 | 274 | NTEST(test_NanoLZ_insufficient_dst_decompr) { 275 | NTEST_SUPPRESS_UNUSED; 276 | std::vector src; 277 | std::vector tmp; 278 | std::vector tmp_output; 279 | const uint8_t masks[] = {0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01}; 280 | const int search_limits[] = {1, 2, 5, -1}; 281 | const ECL_usize src_size = 2000; 282 | 283 | src.resize(src_size); 284 | for(ECL_usize j = 0; j < src_size; ++j) { 285 | src[j] = rand(); 286 | } 287 | ECL_NanoLZ_FastParams fp; 288 | ECL_NanoLZ_FastParams_Alloc1(&fp, 11); 289 | for(auto mask : masks) { 290 | for(ECL_usize j = 0; j < src_size; ++j) { 291 | src[j] &= mask; 292 | } 293 | const auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 294 | tmp.resize(enough_size); 295 | for(auto limit : search_limits) { 296 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 297 | auto comp_size = ECL_NanoLZ_Compress_fast1(scheme, src.data(), src_size, tmp.data(), enough_size, limit, &fp); 298 | tmp_output.resize(src_size); 299 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 300 | approve(decomp_size == src_size); 301 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 302 | // test with insufficient output buffer - compression has to fail correctly 303 | for(ECL_usize i = 1; i <= src_size; ++i) { 304 | const auto output_size = src_size - i; 305 | ECL_TEST_MAGIC_RESIZE(tmp_output, output_size); 306 | auto sz = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), output_size); 307 | if(sz) { // if succeeded - can be only a coincidence: output_size decompressed but last compressed byte is not fully consumed 308 | approve(sz == output_size); 309 | approve(0 == memcmp(src.data(), tmp_output.data(), output_size)); 310 | } 311 | ECL_TEST_MAGIC_VALIDATE(tmp_output); 312 | } 313 | } 314 | } 315 | } 316 | ECL_NanoLZ_FastParams_Destroy(&fp); 317 | } 318 | -------------------------------------------------------------------------------- /tests/tests_perf_inline.hpp: -------------------------------------------------------------------------------- 1 | #include "../ECL_ZeroEater.h" 2 | #include "../ECL_ZeroDevourer.h" 3 | #include "ntest/ntest.h" 4 | 5 | #include 6 | 7 | static const uint8_t ECL_test_perf_data_byte_mask = 0x0F; 8 | static const int ECL_test_perf_data_block_size = 50000; 9 | static const int ECL_test_perf_data_repeats = 2000; 10 | 11 | NTEST(test_perf_ZeroDevourer_compressor) { 12 | NTEST_SUPPRESS_UNUSED; 13 | std::vector src; 14 | std::vector tmp; 15 | std::vector tmp_output; 16 | const auto src_size = ECL_test_perf_data_block_size; 17 | src.resize(src_size); 18 | for(int j = 0; j < src_size; ++j) { 19 | src[j] = rand() & ECL_test_perf_data_byte_mask; 20 | } 21 | auto enough_size = ECL_ZERO_DEVOURER_GET_BOUND(src_size); 22 | tmp.resize(enough_size); 23 | tmp_output.resize(src_size); 24 | ECL_usize comp_size; 25 | for(int i = 0; i < ECL_test_perf_data_repeats; ++i) { 26 | comp_size = ECL_ZeroDevourer_Compress(src.data(), src_size, tmp.data(), enough_size); 27 | } 28 | approve(src_size == ECL_ZeroDevourer_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size)); 29 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 30 | } 31 | 32 | NTEST(test_perf_ZeroDevourer_decompressor) { 33 | NTEST_SUPPRESS_UNUSED; 34 | std::vector src; 35 | std::vector tmp; 36 | std::vector tmp_output; 37 | const auto src_size = ECL_test_perf_data_block_size; 38 | src.resize(src_size); 39 | for(int j = 0; j < src_size; ++j) { 40 | src[j] = rand() & ECL_test_perf_data_byte_mask; 41 | } 42 | auto enough_size = ECL_ZERO_DEVOURER_GET_BOUND(src_size); 43 | tmp.resize(enough_size); 44 | tmp_output.resize(src_size); 45 | auto comp_size = ECL_ZeroDevourer_Compress(src.data(), src_size, tmp.data(), enough_size); 46 | for(int i = 0; i < ECL_test_perf_data_repeats; ++i) { 47 | approve(src_size == ECL_ZeroDevourer_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size)); 48 | } 49 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 50 | } 51 | 52 | NTEST(test_perf_ZeroEater_compressor) { 53 | NTEST_SUPPRESS_UNUSED; 54 | std::vector src; 55 | std::vector tmp; 56 | std::vector tmp_output; 57 | const auto src_size = ECL_test_perf_data_block_size; 58 | src.resize(src_size); 59 | for(int j = 0; j < src_size; ++j) { 60 | src[j] = rand() & ECL_test_perf_data_byte_mask; 61 | } 62 | auto enough_size = ECL_ZERO_EATER_GET_BOUND(src_size); 63 | tmp.resize(enough_size); 64 | tmp_output.resize(src_size); 65 | ECL_usize comp_size; 66 | for(int i = 0; i < ECL_test_perf_data_repeats; ++i) { 67 | comp_size = ECL_ZeroEater_Compress(src.data(), src_size, tmp.data(), enough_size); 68 | } 69 | approve(src_size == ECL_ZeroEater_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size)); 70 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 71 | } 72 | 73 | NTEST(test_perf_ZeroEater_decompressor) { 74 | NTEST_SUPPRESS_UNUSED; 75 | std::vector src; 76 | std::vector tmp; 77 | std::vector tmp_output; 78 | const auto src_size = ECL_test_perf_data_block_size; 79 | src.resize(src_size); 80 | for(int j = 0; j < src_size; ++j) { 81 | src[j] = rand() & ECL_test_perf_data_byte_mask; 82 | } 83 | auto enough_size = ECL_ZERO_EATER_GET_BOUND(src_size); 84 | tmp.resize(enough_size); 85 | tmp_output.resize(src_size); 86 | auto comp_size = ECL_ZeroEater_Compress(src.data(), src_size, tmp.data(), enough_size); 87 | for(int i = 0; i < ECL_test_perf_data_repeats; ++i) { 88 | approve(src_size == ECL_ZeroEater_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size)); 89 | } 90 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 91 | } 92 | -------------------------------------------------------------------------------- /tests/tests_random_data_inline.hpp: -------------------------------------------------------------------------------- 1 | #include "../ECL_ZeroEater.h" 2 | #include "../ECL_ZeroDevourer.h" 3 | #include "../ECL_NanoLZ.h" 4 | #include "ntest/ntest.h" 5 | 6 | #include 7 | 8 | NTEST(test_ZeroEater_random_data) { 9 | NTEST_SUPPRESS_UNUSED; 10 | std::vector src; 11 | std::vector tmp; 12 | std::vector tmp_output; 13 | const int n_sets = 100 * (BoundVMinMax(depth + 10, 0, 1010) + 1); 14 | const int max_size = 60000; 15 | const int min_size = 1; 16 | const uint8_t masks[] = {0x3F, 0x07, 0x03, 0x01}; 17 | for(int i = 0; i < n_sets; ++i) { 18 | const ECL_usize src_size = (rand() % (max_size - min_size)) + min_size; 19 | src.resize(src_size); 20 | for(ECL_usize j = 0; j < src_size; ++j) { 21 | src[j] = rand(); 22 | } 23 | 24 | for(auto mask : masks) { 25 | for(ECL_usize j = 0; j < src_size; ++j) { 26 | src[j] &= mask; 27 | } 28 | auto enough_size = ECL_ZERO_EATER_GET_BOUND(src_size); 29 | tmp.resize(enough_size); 30 | auto comp_size = ECL_ZeroEater_Compress(src.data(), src_size, tmp.data(), enough_size); 31 | approve(comp_size == ECL_ZeroEater_Compress(src.data(), src_size, nullptr, 0)); 32 | 33 | tmp_output.resize(src_size); 34 | auto decomp_size = ECL_ZeroEater_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size); 35 | approve(decomp_size == src_size); 36 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 37 | } 38 | } 39 | } 40 | 41 | NTEST(test_ZeroDevourer_random_data) { 42 | NTEST_SUPPRESS_UNUSED; 43 | std::vector src; 44 | std::vector tmp; 45 | std::vector tmp_output; 46 | const int n_sets = 100 * (BoundVMinMax(depth + 10, 0, 1010) + 1); 47 | const int max_size = 60000; 48 | const int min_size = 1; 49 | const uint8_t masks[] = {0x3F, 0x07, 0x03, 0x01}; 50 | for(int i = 0; i < n_sets; ++i) { 51 | const ECL_usize src_size = (rand() % (max_size - min_size)) + min_size; 52 | src.resize(src_size); 53 | for(ECL_usize j = 0; j < src_size; ++j) { 54 | src[j] = rand(); 55 | } 56 | 57 | for(auto mask : masks) { 58 | for(ECL_usize j = 0; j < src_size; ++j) { 59 | src[j] &= mask; 60 | } 61 | auto enough_size = ECL_ZERO_DEVOURER_GET_BOUND(src_size); 62 | tmp.resize(enough_size); 63 | auto comp_size = ECL_ZeroDevourer_Compress(src.data(), src_size, tmp.data(), enough_size); 64 | 65 | tmp_output.resize(src_size); 66 | auto decomp_size = ECL_ZeroDevourer_Decompress(tmp.data(), comp_size, tmp_output.data(), src_size); 67 | approve(decomp_size == src_size); 68 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 69 | } 70 | } 71 | } 72 | 73 | NTEST(test_NanoLZ_slow_random_data) { 74 | NTEST_SUPPRESS_UNUSED; 75 | std::vector src; 76 | std::vector tmp; 77 | std::vector tmp_output; 78 | const int n_sets = 100 * (BoundVMinMax(depth + 10, 0, 1010) + 1); 79 | const int max_size = 2000; 80 | const int min_size = 1; 81 | const uint8_t masks[] = {0x3F, 0x07, 0x03, 0x01}; 82 | 83 | src.reserve(max_size); 84 | for(int i = 0; i < n_sets; ++i) { 85 | const ECL_usize src_size = (rand() % (max_size - min_size)) + min_size; 86 | src.clear(); 87 | src.resize(src_size); 88 | for(ECL_usize j = 0; j < src_size; ++j) { 89 | src[j] = rand(); 90 | } 91 | 92 | for(auto mask : masks) { 93 | for(ECL_usize j = 0; j < src_size; ++j) { 94 | src[j] &= mask; 95 | } 96 | auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 97 | tmp.resize(enough_size); 98 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 99 | auto comp_size = ECL_NanoLZ_Compress_slow(scheme, src.data(), src_size, tmp.data(), enough_size, -1); 100 | tmp_output.resize(src_size); 101 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 102 | approve(decomp_size == src_size); 103 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 104 | } 105 | } 106 | } 107 | } 108 | 109 | NTEST(test_NanoLZ_mid1_random_data) { 110 | NTEST_SUPPRESS_UNUSED; 111 | std::vector src; 112 | std::vector tmp; 113 | std::vector tmp_output; 114 | const int n_sets = 10 * (BoundVMinMax(depth + 10, 0, 1010) + 1); 115 | const int max_size = 2000; 116 | const int min_size = 1; 117 | const uint8_t masks[] = {0x3F, 0x07, 0x03, 0x01}; 118 | const int search_limits[] = {1, 2, 5, 10, -1}; 119 | uint8_t buf_x[256]; 120 | 121 | src.reserve(max_size); 122 | for(int i = 0; i < n_sets; ++i) { 123 | const ECL_usize src_size = (rand() % (max_size - min_size)) + min_size; 124 | src.clear(); 125 | src.resize(src_size); 126 | for(ECL_usize j = 0; j < src_size; ++j) { 127 | src[j] = rand(); 128 | } 129 | 130 | for(auto mask : masks) { 131 | for(ECL_usize j = 0; j < src_size; ++j) { 132 | src[j] &= mask; 133 | } 134 | auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 135 | ECL_TEST_MAGIC_RESIZE(tmp, enough_size); 136 | for(auto limit : search_limits) { 137 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 138 | auto comp_size = ECL_NanoLZ_Compress_mid1(scheme, src.data(), src_size, tmp.data(), enough_size, limit, buf_x); 139 | tmp_output.resize(src_size); 140 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 141 | approve(decomp_size == src_size); 142 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 143 | ECL_TEST_MAGIC_VALIDATE(tmp); 144 | } 145 | } 146 | } 147 | } 148 | } 149 | 150 | NTEST(test_NanoLZ_mid2_random_data) { 151 | NTEST_SUPPRESS_UNUSED; 152 | std::vector src; 153 | std::vector tmp; 154 | std::vector tmp_output; 155 | const int n_sets = 10 * (BoundVMinMax(depth + 10, 0, 1010) + 1); 156 | const int max_size = 2000; 157 | const int min_size = 1; 158 | const uint8_t masks[] = {0x3F, 0x07, 0x03, 0x01}; 159 | const int search_limits[] = {1, 2, 5, 10, -1}; 160 | uint8_t buf_x[513]; 161 | 162 | src.reserve(max_size); 163 | for(int i = 0; i < n_sets; ++i) { 164 | const ECL_usize src_size = (rand() % (max_size - min_size)) + min_size; 165 | src.clear(); 166 | src.resize(src_size); 167 | for(ECL_usize j = 0; j < src_size; ++j) { 168 | src[j] = rand(); 169 | } 170 | 171 | for(auto mask : masks) { 172 | for(ECL_usize j = 0; j < src_size; ++j) { 173 | src[j] &= mask; 174 | } 175 | auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 176 | ECL_TEST_MAGIC_RESIZE(tmp, enough_size); 177 | for(auto limit : search_limits) { 178 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 179 | auto comp_size = ECL_NanoLZ_Compress_mid2(scheme, src.data(), src_size, tmp.data(), enough_size, limit, buf_x); 180 | tmp_output.resize(src_size); 181 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 182 | approve(decomp_size == src_size); 183 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 184 | ECL_TEST_MAGIC_VALIDATE(tmp); 185 | } 186 | } 187 | } 188 | } 189 | } 190 | 191 | NTEST(test_NanoLZ_mid1min_random_data) { 192 | NTEST_SUPPRESS_UNUSED; 193 | std::vector src; 194 | std::vector tmp; 195 | std::vector tmp_output; 196 | const int n_sets = 10 * (BoundVMinMax(depth + 10, 0, 1010) + 1); 197 | const int max_size = 2000; 198 | const int min_size = 1; 199 | const uint8_t masks[] = {0x3F, 0x07, 0x03, 0x01}; 200 | uint8_t buf_x[256]; 201 | 202 | src.reserve(max_size); 203 | for(int i = 0; i < n_sets; ++i) { 204 | const ECL_usize src_size = (rand() % (max_size - min_size)) + min_size; 205 | src.clear(); 206 | src.resize(src_size); 207 | for(ECL_usize j = 0; j < src_size; ++j) { 208 | src[j] = rand(); 209 | } 210 | 211 | for(auto mask : masks) { 212 | for(ECL_usize j = 0; j < src_size; ++j) { 213 | src[j] &= mask; 214 | } 215 | auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 216 | ECL_TEST_MAGIC_RESIZE(tmp, enough_size); 217 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 218 | auto comp_size = ECL_NanoLZ_Compress_mid1min(scheme, src.data(), src_size, tmp.data(), enough_size, buf_x); 219 | tmp_output.resize(src_size); 220 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 221 | approve(decomp_size == src_size); 222 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 223 | ECL_TEST_MAGIC_VALIDATE(tmp); 224 | } 225 | } 226 | } 227 | } 228 | 229 | NTEST(test_NanoLZ_mid2min_random_data) { 230 | NTEST_SUPPRESS_UNUSED; 231 | std::vector src; 232 | std::vector tmp; 233 | std::vector tmp_output; 234 | const int n_sets = 10 * (BoundVMinMax(depth + 10, 0, 1010) + 1); 235 | const int max_size = 2000; 236 | const int min_size = 1; 237 | const uint8_t masks[] = {0x3F, 0x07, 0x03, 0x01}; 238 | uint8_t buf_x[513]; 239 | 240 | src.reserve(max_size); 241 | for(int i = 0; i < n_sets; ++i) { 242 | const ECL_usize src_size = (rand() % (max_size - min_size)) + min_size; 243 | src.clear(); 244 | src.resize(src_size); 245 | for(ECL_usize j = 0; j < src_size; ++j) { 246 | src[j] = rand(); 247 | } 248 | 249 | for(auto mask : masks) { 250 | for(ECL_usize j = 0; j < src_size; ++j) { 251 | src[j] &= mask; 252 | } 253 | auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 254 | ECL_TEST_MAGIC_RESIZE(tmp, enough_size); 255 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 256 | auto comp_size = ECL_NanoLZ_Compress_mid2min(scheme, src.data(), src_size, tmp.data(), enough_size, buf_x); 257 | tmp_output.resize(src_size); 258 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 259 | approve(decomp_size == src_size); 260 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 261 | ECL_TEST_MAGIC_VALIDATE(tmp); 262 | } 263 | } 264 | } 265 | } 266 | 267 | NTEST(test_NanoLZ_auto_random_data) { 268 | NTEST_SUPPRESS_UNUSED; 269 | std::vector src; 270 | std::vector tmp; 271 | std::vector tmp_output; 272 | const int n_sets = 100 * (BoundVMinMax(depth + 10, 0, 1010) + 1); 273 | const int max_size = 2000; 274 | const int min_size = 1; 275 | const uint8_t masks[] = {0x3F, 0x07, 0x03, 0x01}; 276 | 277 | src.reserve(max_size); 278 | for(int i = 0; i < n_sets; ++i) { 279 | const ECL_usize src_size = (rand() % (max_size - min_size)) + min_size; 280 | src.clear(); 281 | src.resize(src_size); 282 | for(ECL_usize j = 0; j < src_size; ++j) { 283 | src[j] = rand(); 284 | } 285 | 286 | for(auto mask : masks) { 287 | for(ECL_usize j = 0; j < src_size; ++j) { 288 | src[j] &= mask; 289 | } 290 | auto enough_size = ECL_NANO_LZ_GET_BOUND(src_size); 291 | tmp.resize(enough_size); 292 | for(auto scheme : ECL_NANO_LZ_SCHEMES_ALL) { 293 | auto comp_size = ECL_NanoLZ_Compress_auto(scheme, src.data(), src_size, tmp.data(), enough_size, -1); 294 | tmp_output.resize(src_size); 295 | auto decomp_size = ECL_NanoLZ_Decompress(scheme, tmp.data(), comp_size, tmp_output.data(), src_size); 296 | approve(decomp_size == src_size); 297 | approve(0 == memcmp(src.data(), tmp_output.data(), src_size)); 298 | } 299 | } 300 | } 301 | } 302 | -------------------------------------------------------------------------------- /tests/unix_clang.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | depth=$1 3 | 4 | main_src="../ecl-all-c-included/ECL_all_c_included.c" 5 | test_src="tests.cpp" 6 | arch="-m32" 7 | opts="$arch -Wall -Wextra -pedantic -O3 -DECL_BUILD_AS_C -DECL_USE_ASSERT" 8 | cpp_opts="-std=c++11" 9 | linker_opts="$arch -lstdc++" 10 | cc="clang" 11 | cxx="clang++" 12 | link=$cxx 13 | 14 | # param 1 = bitness 15 | get_opts() { 16 | echo "$opts -DECL_USE_BITNESS_$1" 17 | } 18 | 19 | compile_run() { 20 | set +e 21 | rm *.o 22 | set -e 23 | $cc $main_src $(get_opts $1) -c 24 | $cxx $test_src $(get_opts $1) $cpp_opts -c 25 | $link $linker_opts *.o -o a.out 26 | echo ./a.out $depth 27 | } 28 | 29 | set -x 30 | $(compile_run 16) 31 | $(compile_run 32) 32 | $(compile_run 64) 33 | -------------------------------------------------------------------------------- /tests/unix_gcc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | depth=$1 3 | 4 | main_src="../ecl-all-c-included/ECL_all_c_included.c" 5 | test_src="tests.cpp" 6 | arch="-m32" 7 | opts="$arch -Wall -Wextra -pedantic -O3 -DECL_BUILD_AS_C -DECL_USE_ASSERT" 8 | cpp_opts="-std=c++11" 9 | linker_opts="$arch -lstdc++" 10 | cc="gcc" 11 | cxx="g++" 12 | link="gcc" 13 | 14 | # param 1 = bitness 15 | get_opts() { 16 | echo "$opts -DECL_USE_BITNESS_$1" 17 | } 18 | 19 | compile_run() { 20 | set +e 21 | rm *.o 22 | set -e 23 | $cc $main_src $(get_opts $1) -c 24 | $cxx $test_src $(get_opts $1) $cpp_opts -c 25 | $link $linker_opts *.o -o a.out 26 | echo ./a.out $depth 27 | } 28 | 29 | set -x 30 | $(compile_run 16) 31 | $(compile_run 32) 32 | $(compile_run 64) 33 | -------------------------------------------------------------------------------- /tests/win_gcc.bat: -------------------------------------------------------------------------------- 1 | set "compile=gcc tests.cpp ../ecl-all-c-included/ECL_all_c_included.c -m32 -Wall -Wextra -pedantic -O3 -std=c++11 -lstdc++ -o a.exe -DECL_BUILD_AS_C -DECL_USE_ASSERT -DECL_USE_BITNESS" 2 | set "run_tests=a.exe %1" 3 | del a.exe && %compile%_16 && %run_tests% && %compile%_32 && %run_tests% && %compile%_64 && %run_tests% && pause 4 | -------------------------------------------------------------------------------- /tests/win_msvc.bat: -------------------------------------------------------------------------------- 1 | set "compile=cl tests.cpp ../ecl-all-c-included/ECL_all_c_included.c /O2 /EHsc /DECL_BUILD_AS_C /DECL_USE_ASSERT /Fe:tests.exe /DECL_USE_BITNESS" 2 | set "run_tests=tests.exe %1" 3 | del tests.exe && %compile%_16 && %run_tests% && %compile%_32 && %run_tests% && %compile%_64 && %run_tests% && pause 4 | --------------------------------------------------------------------------------