├── ZpngApp.exe ├── output.png ├── ZpngTester.exe ├── .gitignore ├── proj ├── ZpngApp.vcxproj.filters ├── ZpngTester.vcxproj.filters ├── ZpngTester.vcxproj.user ├── ZpngApp.vcxproj.user ├── zpng.sln ├── zpng.vcxproj.filters ├── ZpngTester.vcxproj ├── ZpngApp.vcxproj └── zpng.vcxproj ├── zstd ├── zstd_fast.h ├── zstd_double_fast.h ├── LICENSE ├── zstd_opt.h ├── threading.c ├── pool.h ├── error_private.h ├── divsufsort.h ├── zstd_lazy.h ├── error_private.c ├── zstd_common.c ├── zstd_errors.h ├── compiler.h ├── zstd_ldm.h ├── threading.h ├── cpu.h ├── zstdmt_compress.h ├── entropy_common.c ├── pool.c ├── zstd_internal.h ├── zdict.h ├── zstd_fast.c ├── fse_decompress.c ├── mem.h └── xxhash.h ├── LICENSE ├── CMakeLists.txt ├── README.md ├── zpng.h └── apps ├── zpng_app.cpp └── zpng_test.cpp /ZpngApp.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catid/Zpng/HEAD/ZpngApp.exe -------------------------------------------------------------------------------- /output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catid/Zpng/HEAD/output.png -------------------------------------------------------------------------------- /ZpngTester.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/catid/Zpng/HEAD/ZpngTester.exe -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | -------------------------------------------------------------------------------- /proj/ZpngApp.vcxproj.filters: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /proj/ZpngTester.vcxproj.filters: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /proj/ZpngTester.vcxproj.user: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | $(SolutionDir)../ 5 | WindowsLocalDebugger 6 | 7 | 8 | $(SolutionDir)../ 9 | WindowsLocalDebugger 10 | 11 | 12 | $(SolutionDir)../ 13 | WindowsLocalDebugger 14 | 15 | 16 | $(SolutionDir)../ 17 | WindowsLocalDebugger 18 | 19 | -------------------------------------------------------------------------------- /zstd/zstd_fast.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_FAST_H 12 | #define ZSTD_FAST_H 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | #include "mem.h" /* U32 */ 19 | #include "zstd_compress_internal.h" 20 | 21 | void ZSTD_fillHashTable(ZSTD_matchState_t* ms, 22 | ZSTD_compressionParameters const* cParams, 23 | void const* end, ZSTD_dictTableLoadMethod_e dtlm); 24 | size_t ZSTD_compressBlock_fast( 25 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 26 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 27 | size_t ZSTD_compressBlock_fast_extDict( 28 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 29 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 30 | 31 | #if defined (__cplusplus) 32 | } 33 | #endif 34 | 35 | #endif /* ZSTD_FAST_H */ 36 | -------------------------------------------------------------------------------- /zstd/zstd_double_fast.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_DOUBLE_FAST_H 12 | #define ZSTD_DOUBLE_FAST_H 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | #include "mem.h" /* U32 */ 19 | #include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */ 20 | 21 | void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, 22 | ZSTD_compressionParameters const* cParams, 23 | void const* end, ZSTD_dictTableLoadMethod_e dtlm); 24 | size_t ZSTD_compressBlock_doubleFast( 25 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 26 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 27 | size_t ZSTD_compressBlock_doubleFast_extDict( 28 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 29 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 30 | 31 | 32 | #if defined (__cplusplus) 33 | } 34 | #endif 35 | 36 | #endif /* ZSTD_DOUBLE_FAST_H */ 37 | -------------------------------------------------------------------------------- /proj/ZpngApp.vcxproj.user: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | $(SolutionDir)../ 5 | WindowsLocalDebugger 6 | -c images/braid/killsign.png test.zpng 7 | 8 | 9 | $(SolutionDir)../ 10 | WindowsLocalDebugger 11 | -c images/braid/killsign.png test.zpng 12 | 13 | 14 | $(SolutionDir)../ 15 | WindowsLocalDebugger 16 | -c images/braid/killsign.png test.zpng 17 | 18 | 19 | $(SolutionDir)../ 20 | WindowsLocalDebugger 21 | -c images/braid/killsign.png test.zpng 22 | 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Christopher A. Taylor 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /zstd/LICENSE: -------------------------------------------------------------------------------- 1 | BSD License 2 | 3 | For Zstandard software 4 | 5 | Copyright (c) 2016-present, Facebook, Inc. All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, 8 | are permitted provided that the following conditions are met: 9 | 10 | * Redistributions of source code must retain the above copyright notice, this 11 | list of conditions and the following disclaimer. 12 | 13 | * Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | 17 | * Neither the name Facebook nor the names of its contributors may be used to 18 | endorse or promote products derived from this software without specific 19 | prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 25 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 26 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 28 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /zstd/zstd_opt.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_OPT_H 12 | #define ZSTD_OPT_H 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | #include "zstd_compress_internal.h" 19 | 20 | void ZSTD_updateTree( 21 | ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, 22 | const BYTE* ip, const BYTE* iend); /* used in ZSTD_loadDictionaryContent() */ 23 | 24 | size_t ZSTD_compressBlock_btopt( 25 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 26 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 27 | size_t ZSTD_compressBlock_btultra( 28 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 29 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 30 | 31 | size_t ZSTD_compressBlock_btopt_extDict( 32 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 33 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 34 | size_t ZSTD_compressBlock_btultra_extDict( 35 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 36 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 37 | 38 | #if defined (__cplusplus) 39 | } 40 | #endif 41 | 42 | #endif /* ZSTD_OPT_H */ 43 | -------------------------------------------------------------------------------- /zstd/threading.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016 Tino Reichardt 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * 9 | * You can contact the author at: 10 | * - zstdmt source repository: https://github.com/mcmilk/zstdmt 11 | */ 12 | 13 | /** 14 | * This file will hold wrapper for systems, which do not support pthreads 15 | */ 16 | 17 | /* create fake symbol to avoid empty trnaslation unit warning */ 18 | int g_ZSTD_threading_useles_symbol; 19 | 20 | #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) 21 | 22 | /** 23 | * Windows minimalist Pthread Wrapper, based on : 24 | * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html 25 | */ 26 | 27 | 28 | /* === Dependencies === */ 29 | #include 30 | #include 31 | #include "threading.h" 32 | 33 | 34 | /* === Implementation === */ 35 | 36 | static unsigned __stdcall worker(void *arg) 37 | { 38 | ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg; 39 | thread->arg = thread->start_routine(thread->arg); 40 | return 0; 41 | } 42 | 43 | int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, 44 | void* (*start_routine) (void*), void* arg) 45 | { 46 | (void)unused; 47 | thread->arg = arg; 48 | thread->start_routine = start_routine; 49 | thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL); 50 | 51 | if (!thread->handle) 52 | return errno; 53 | else 54 | return 0; 55 | } 56 | 57 | int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr) 58 | { 59 | DWORD result; 60 | 61 | if (!thread.handle) return 0; 62 | 63 | result = WaitForSingleObject(thread.handle, INFINITE); 64 | switch (result) { 65 | case WAIT_OBJECT_0: 66 | if (value_ptr) *value_ptr = thread.arg; 67 | return 0; 68 | case WAIT_ABANDONED: 69 | return EINVAL; 70 | default: 71 | return GetLastError(); 72 | } 73 | } 74 | 75 | #endif /* ZSTD_MULTITHREAD */ 76 | -------------------------------------------------------------------------------- /zstd/pool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef POOL_H 12 | #define POOL_H 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | 19 | #include /* size_t */ 20 | #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ 21 | #include "zstd.h" 22 | 23 | typedef struct POOL_ctx_s POOL_ctx; 24 | 25 | /*! POOL_create() : 26 | * Create a thread pool with at most `numThreads` threads. 27 | * `numThreads` must be at least 1. 28 | * The maximum number of queued jobs before blocking is `queueSize`. 29 | * @return : POOL_ctx pointer on success, else NULL. 30 | */ 31 | POOL_ctx* POOL_create(size_t numThreads, size_t queueSize); 32 | 33 | POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem); 34 | 35 | /*! POOL_free() : 36 | Free a thread pool returned by POOL_create(). 37 | */ 38 | void POOL_free(POOL_ctx* ctx); 39 | 40 | /*! POOL_sizeof() : 41 | return memory usage of pool returned by POOL_create(). 42 | */ 43 | size_t POOL_sizeof(POOL_ctx* ctx); 44 | 45 | /*! POOL_function : 46 | The function type that can be added to a thread pool. 47 | */ 48 | typedef void (*POOL_function)(void*); 49 | /*! POOL_add_function : 50 | The function type for a generic thread pool add function. 51 | */ 52 | typedef void (*POOL_add_function)(void*, POOL_function, void*); 53 | 54 | /*! POOL_add() : 55 | Add the job `function(opaque)` to the thread pool. `ctx` must be valid. 56 | Possibly blocks until there is room in the queue. 57 | Note : The function may be executed asynchronously, so `opaque` must live until the function has been completed. 58 | */ 59 | void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque); 60 | 61 | 62 | /*! POOL_tryAdd() : 63 | Add the job `function(opaque)` to the thread pool if a worker is available. 64 | return immediately otherwise. 65 | @return : 1 if successful, 0 if not. 66 | */ 67 | int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque); 68 | 69 | 70 | #if defined (__cplusplus) 71 | } 72 | #endif 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.5) 2 | project(ccat) 3 | 4 | set(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "" FORCE) 5 | 6 | if(NOT CMAKE_BUILD_TYPE) 7 | set(CMAKE_BUILD_TYPE Release) 8 | endif() 9 | 10 | if(MSVC) 11 | else() 12 | set(CMAKE_CXX_FLAGS "-Wall -Wextra") 13 | set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -march=native") 14 | set(CMAKE_CXX_FLAGS_RELEASE "-O3 -march=native") 15 | endif() 16 | 17 | set(CMAKE_CXX_STANDARD 11) 18 | 19 | set(THREADS_PREFER_PTHREAD_FLAG ON) 20 | find_package(Threads REQUIRED) 21 | 22 | # Zstd library source files 23 | set(ZSTD_LIB_SRCFILES 24 | zstd/bitstream.h 25 | zstd/compiler.h 26 | zstd/cover.c 27 | zstd/cpu.h 28 | zstd/divsufsort.c 29 | zstd/divsufsort.h 30 | zstd/entropy_common.c 31 | zstd/error_private.c 32 | zstd/error_private.h 33 | zstd/fse.h 34 | zstd/fse_compress.c 35 | zstd/fse_decompress.c 36 | zstd/huf.h 37 | zstd/huf_compress.c 38 | zstd/huf_decompress.c 39 | zstd/mem.h 40 | zstd/pool.c 41 | zstd/pool.h 42 | zstd/threading.c 43 | zstd/threading.h 44 | zstd/xxhash.c 45 | zstd/xxhash.h 46 | zstd/zdict.c 47 | zstd/zdict.h 48 | zstd/zstd.h 49 | zstd/zstdmt_compress.c 50 | zstd/zstdmt_compress.h 51 | zstd/zstd_common.c 52 | zstd/zstd_compress.c 53 | zstd/zstd_compress_internal.h 54 | zstd/zstd_decompress.c 55 | zstd/zstd_double_fast.c 56 | zstd/zstd_double_fast.h 57 | zstd/zstd_errors.h 58 | zstd/zstd_fast.c 59 | zstd/zstd_fast.h 60 | zstd/zstd_internal.h 61 | zstd/zstd_lazy.c 62 | zstd/zstd_lazy.h 63 | zstd/zstd_ldm.c 64 | zstd/zstd_ldm.h 65 | zstd/zstd_opt.c 66 | zstd/zstd_opt.h 67 | ) 68 | 69 | # Zpng library source files 70 | set(ZPNG_LIB_SRCFILES 71 | zpng.cpp 72 | zpng.h 73 | ) 74 | 75 | # Zpng unit tester 76 | set(ZPNG_TEST_SRCFILES 77 | apps/zpng_test.cpp 78 | ) 79 | 80 | # Zpng app 81 | set(ZPNG_APP_SRCFILES 82 | apps/zpng_app.cpp 83 | ) 84 | 85 | add_library(zpnglib ${ZPNG_LIB_SRCFILES} ${ZSTD_LIB_SRCFILES}) 86 | 87 | add_executable(unit_test ${ZPNG_TEST_SRCFILES}) 88 | target_link_libraries(unit_test zpnglib Threads::Threads) 89 | 90 | add_executable(zpng ${ZPNG_APP_SRCFILES}) 91 | target_link_libraries(zpng zpnglib Threads::Threads) 92 | -------------------------------------------------------------------------------- /zstd/error_private.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* Note : this module is expected to remain private, do not expose it */ 12 | 13 | #ifndef ERROR_H_MODULE 14 | #define ERROR_H_MODULE 15 | 16 | #if defined (__cplusplus) 17 | extern "C" { 18 | #endif 19 | 20 | 21 | /* **************************************** 22 | * Dependencies 23 | ******************************************/ 24 | #include /* size_t */ 25 | #include "zstd_errors.h" /* enum list */ 26 | 27 | 28 | /* **************************************** 29 | * Compiler-specific 30 | ******************************************/ 31 | #if defined(__GNUC__) 32 | # define ERR_STATIC static __attribute__((unused)) 33 | #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 34 | # define ERR_STATIC static inline 35 | #elif defined(_MSC_VER) 36 | # define ERR_STATIC static __inline 37 | #else 38 | # define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ 39 | #endif 40 | 41 | 42 | /*-**************************************** 43 | * Customization (error_public.h) 44 | ******************************************/ 45 | typedef ZSTD_ErrorCode ERR_enum; 46 | #define PREFIX(name) ZSTD_error_##name 47 | 48 | 49 | /*-**************************************** 50 | * Error codes handling 51 | ******************************************/ 52 | #undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ 53 | #define ERROR(name) ZSTD_ERROR(name) 54 | #define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) 55 | 56 | ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } 57 | 58 | ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } 59 | 60 | 61 | /*-**************************************** 62 | * Error Strings 63 | ******************************************/ 64 | 65 | const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ 66 | 67 | ERR_STATIC const char* ERR_getErrorName(size_t code) 68 | { 69 | return ERR_getErrorString(ERR_getErrorCode(code)); 70 | } 71 | 72 | #if defined (__cplusplus) 73 | } 74 | #endif 75 | 76 | #endif /* ERROR_H_MODULE */ 77 | -------------------------------------------------------------------------------- /zstd/divsufsort.h: -------------------------------------------------------------------------------- 1 | /* 2 | * divsufsort.h for libdivsufsort-lite 3 | * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person 6 | * obtaining a copy of this software and associated documentation 7 | * files (the "Software"), to deal in the Software without 8 | * restriction, including without limitation the rights to use, 9 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the 11 | * Software is furnished to do so, subject to the following 12 | * conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be 15 | * included in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 19 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 21 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | * OTHER DEALINGS IN THE SOFTWARE. 25 | */ 26 | 27 | #ifndef _DIVSUFSORT_H 28 | #define _DIVSUFSORT_H 1 29 | 30 | #ifdef __cplusplus 31 | extern "C" { 32 | #endif /* __cplusplus */ 33 | 34 | 35 | /*- Prototypes -*/ 36 | 37 | /** 38 | * Constructs the suffix array of a given string. 39 | * @param T [0..n-1] The input string. 40 | * @param SA [0..n-1] The output array of suffixes. 41 | * @param n The length of the given string. 42 | * @param openMP enables OpenMP optimization. 43 | * @return 0 if no error occurred, -1 or -2 otherwise. 44 | */ 45 | int 46 | divsufsort(const unsigned char *T, int *SA, int n, int openMP); 47 | 48 | /** 49 | * Constructs the burrows-wheeler transformed string of a given string. 50 | * @param T [0..n-1] The input string. 51 | * @param U [0..n-1] The output string. (can be T) 52 | * @param A [0..n-1] The temporary array. (can be NULL) 53 | * @param n The length of the given string. 54 | * @param num_indexes The length of secondary indexes array. (can be NULL) 55 | * @param indexes The secondary indexes array. (can be NULL) 56 | * @param openMP enables OpenMP optimization. 57 | * @return The primary index if no error occurred, -1 or -2 otherwise. 58 | */ 59 | int 60 | divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP); 61 | 62 | 63 | #ifdef __cplusplus 64 | } /* extern "C" */ 65 | #endif /* __cplusplus */ 66 | 67 | #endif /* _DIVSUFSORT_H */ 68 | -------------------------------------------------------------------------------- /zstd/zstd_lazy.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_LAZY_H 12 | #define ZSTD_LAZY_H 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | #include "zstd_compress_internal.h" 19 | 20 | U32 ZSTD_insertAndFindFirstIndex( 21 | ZSTD_matchState_t* ms, ZSTD_compressionParameters const* cParams, 22 | const BYTE* ip); 23 | 24 | void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). pre-emptively increase value of ZSTD_DUBT_UNSORTED_MARK */ 25 | 26 | size_t ZSTD_compressBlock_btlazy2( 27 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 28 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 29 | size_t ZSTD_compressBlock_lazy2( 30 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 31 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 32 | size_t ZSTD_compressBlock_lazy( 33 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 34 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 35 | size_t ZSTD_compressBlock_greedy( 36 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 37 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 38 | 39 | size_t ZSTD_compressBlock_greedy_extDict( 40 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 41 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 42 | size_t ZSTD_compressBlock_lazy_extDict( 43 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 44 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 45 | size_t ZSTD_compressBlock_lazy2_extDict( 46 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 47 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 48 | size_t ZSTD_compressBlock_btlazy2_extDict( 49 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 50 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize); 51 | 52 | #if defined (__cplusplus) 53 | } 54 | #endif 55 | 56 | #endif /* ZSTD_LAZY_H */ 57 | -------------------------------------------------------------------------------- /zstd/error_private.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* The purpose of this file is to have a single list of error strings embedded in binary */ 12 | 13 | #include "error_private.h" 14 | 15 | const char* ERR_getErrorString(ERR_enum code) 16 | { 17 | static const char* const notErrorCode = "Unspecified error code"; 18 | switch( code ) 19 | { 20 | case PREFIX(no_error): return "No error detected"; 21 | case PREFIX(GENERIC): return "Error (generic)"; 22 | case PREFIX(prefix_unknown): return "Unknown frame descriptor"; 23 | case PREFIX(version_unsupported): return "Version not supported"; 24 | case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; 25 | case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; 26 | case PREFIX(corruption_detected): return "Corrupted block detected"; 27 | case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; 28 | case PREFIX(parameter_unsupported): return "Unsupported parameter"; 29 | case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; 30 | case PREFIX(init_missing): return "Context should be init first"; 31 | case PREFIX(memory_allocation): return "Allocation error : not enough memory"; 32 | case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough"; 33 | case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; 34 | case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; 35 | case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; 36 | case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; 37 | case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; 38 | case PREFIX(dictionary_wrong): return "Dictionary mismatch"; 39 | case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; 40 | case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; 41 | case PREFIX(srcSize_wrong): return "Src size is incorrect"; 42 | /* following error codes are not stable and may be removed or changed in a future version */ 43 | case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; 44 | case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; 45 | case PREFIX(maxCode): 46 | default: return notErrorCode; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /proj/zpng.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio 14 4 | VisualStudioVersion = 14.0.25420.1 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zpng", "zpng.vcxproj", "{6218A984-DB56-41D8-A2B9-3FFEFED36A69}" 7 | EndProject 8 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ZpngApp", "ZpngApp.vcxproj", "{36C2D12E-C734-49B0-8528-E57B3CF37909}" 9 | EndProject 10 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ZpngTester", "ZpngTester.vcxproj", "{3D09F90D-738F-44F6-A846-859DF5C1F35D}" 11 | EndProject 12 | Global 13 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 14 | Debug|x64 = Debug|x64 15 | Debug|x86 = Debug|x86 16 | Release|x64 = Release|x64 17 | Release|x86 = Release|x86 18 | EndGlobalSection 19 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 20 | {6218A984-DB56-41D8-A2B9-3FFEFED36A69}.Debug|x64.ActiveCfg = Debug|x64 21 | {6218A984-DB56-41D8-A2B9-3FFEFED36A69}.Debug|x64.Build.0 = Debug|x64 22 | {6218A984-DB56-41D8-A2B9-3FFEFED36A69}.Debug|x86.ActiveCfg = Debug|Win32 23 | {6218A984-DB56-41D8-A2B9-3FFEFED36A69}.Debug|x86.Build.0 = Debug|Win32 24 | {6218A984-DB56-41D8-A2B9-3FFEFED36A69}.Release|x64.ActiveCfg = Release|x64 25 | {6218A984-DB56-41D8-A2B9-3FFEFED36A69}.Release|x64.Build.0 = Release|x64 26 | {6218A984-DB56-41D8-A2B9-3FFEFED36A69}.Release|x86.ActiveCfg = Release|Win32 27 | {6218A984-DB56-41D8-A2B9-3FFEFED36A69}.Release|x86.Build.0 = Release|Win32 28 | {36C2D12E-C734-49B0-8528-E57B3CF37909}.Debug|x64.ActiveCfg = Debug|x64 29 | {36C2D12E-C734-49B0-8528-E57B3CF37909}.Debug|x64.Build.0 = Debug|x64 30 | {36C2D12E-C734-49B0-8528-E57B3CF37909}.Debug|x86.ActiveCfg = Debug|Win32 31 | {36C2D12E-C734-49B0-8528-E57B3CF37909}.Debug|x86.Build.0 = Debug|Win32 32 | {36C2D12E-C734-49B0-8528-E57B3CF37909}.Release|x64.ActiveCfg = Release|x64 33 | {36C2D12E-C734-49B0-8528-E57B3CF37909}.Release|x64.Build.0 = Release|x64 34 | {36C2D12E-C734-49B0-8528-E57B3CF37909}.Release|x86.ActiveCfg = Release|Win32 35 | {36C2D12E-C734-49B0-8528-E57B3CF37909}.Release|x86.Build.0 = Release|Win32 36 | {3D09F90D-738F-44F6-A846-859DF5C1F35D}.Debug|x64.ActiveCfg = Debug|x64 37 | {3D09F90D-738F-44F6-A846-859DF5C1F35D}.Debug|x64.Build.0 = Debug|x64 38 | {3D09F90D-738F-44F6-A846-859DF5C1F35D}.Debug|x86.ActiveCfg = Debug|Win32 39 | {3D09F90D-738F-44F6-A846-859DF5C1F35D}.Debug|x86.Build.0 = Debug|Win32 40 | {3D09F90D-738F-44F6-A846-859DF5C1F35D}.Release|x64.ActiveCfg = Release|x64 41 | {3D09F90D-738F-44F6-A846-859DF5C1F35D}.Release|x64.Build.0 = Release|x64 42 | {3D09F90D-738F-44F6-A846-859DF5C1F35D}.Release|x86.ActiveCfg = Release|Win32 43 | {3D09F90D-738F-44F6-A846-859DF5C1F35D}.Release|x86.Build.0 = Release|Win32 44 | EndGlobalSection 45 | GlobalSection(SolutionProperties) = preSolution 46 | HideSolutionNode = FALSE 47 | EndGlobalSection 48 | EndGlobal 49 | -------------------------------------------------------------------------------- /zstd/zstd_common.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | 12 | 13 | /*-************************************* 14 | * Dependencies 15 | ***************************************/ 16 | #include /* malloc, calloc, free */ 17 | #include /* memset */ 18 | #include "error_private.h" 19 | #include "zstd_internal.h" 20 | 21 | 22 | /*-**************************************** 23 | * Version 24 | ******************************************/ 25 | unsigned ZSTD_versionNumber(void) { return ZSTD_VERSION_NUMBER; } 26 | 27 | const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } 28 | 29 | 30 | /*-**************************************** 31 | * ZSTD Error Management 32 | ******************************************/ 33 | /*! ZSTD_isError() : 34 | * tells if a return value is an error code */ 35 | unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } 36 | 37 | /*! ZSTD_getErrorName() : 38 | * provides error code string from function result (useful for debugging) */ 39 | const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } 40 | 41 | /*! ZSTD_getError() : 42 | * convert a `size_t` function result into a proper ZSTD_errorCode enum */ 43 | ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } 44 | 45 | /*! ZSTD_getErrorString() : 46 | * provides error code string from enum */ 47 | const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } 48 | 49 | /*! g_debuglog_enable : 50 | * turn on/off debug traces (global switch) */ 51 | #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG >= 2) 52 | int g_debuglog_enable = 1; 53 | #endif 54 | 55 | 56 | /*=************************************************************** 57 | * Custom allocator 58 | ****************************************************************/ 59 | void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) 60 | { 61 | if (customMem.customAlloc) 62 | return customMem.customAlloc(customMem.opaque, size); 63 | return malloc(size); 64 | } 65 | 66 | void* ZSTD_calloc(size_t size, ZSTD_customMem customMem) 67 | { 68 | if (customMem.customAlloc) { 69 | /* calloc implemented as malloc+memset; 70 | * not as efficient as calloc, but next best guess for custom malloc */ 71 | void* const ptr = customMem.customAlloc(customMem.opaque, size); 72 | memset(ptr, 0, size); 73 | return ptr; 74 | } 75 | return calloc(1, size); 76 | } 77 | 78 | void ZSTD_free(void* ptr, ZSTD_customMem customMem) 79 | { 80 | if (ptr!=NULL) { 81 | if (customMem.customFree) 82 | customMem.customFree(customMem.opaque, ptr); 83 | else 84 | free(ptr); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Zpng 2 | Small experimental lossless photographic image compression library with a C API and command-line interface. 3 | 4 | It's much faster than PNG and compresses better for photographic images. 5 | This compressor often takes less than 6% of the time of a PNG compressor and produces a file that is 66% of the size. 6 | It was written in just 500 lines of C code thanks to Facebook's Zstd library. 7 | 8 | The goal was to see if I could create a better lossless compressor than PNG in just one evening (a few hours) using Zstd and some past experience writing my GCIF library. Zstd is magical. 9 | 10 | I'm not expecting anyone else to use this, but feel free if you need some fast compression in just a few hundred lines of C code. 11 | 12 | 13 | #### Example results 14 | 15 | ``` 16 | $ ./ZpngApp.exe -c IMG_0008.jpg IMG_0008.zpng 17 | Compressing IMG_0008.jpg to IMG_0008.zpng 18 | Loaded IMG_0008.jpg in 338.712 msec 19 | Compressed ZPNG in 248.737 msec 20 | ZPNG compression size: 27731886 bytes 21 | 22 | $ ./ZpngApp.exe -d IMG_0008.zpng IMG_0008.png 23 | Decompressing IMG_0008.zpng to IMG_0008.png (output will be PNG format) 24 | Decompressed ZPNG in 123.028 msec 25 | Compressed PNG in 4339.82 msec 26 | Wrote decompressed PNG file: IMG_0008.png 27 | 28 | $ ll IMG_0008.* 29 | -rw-r--r-- 1 leon 197121 13991058 Jun 25 2017 IMG_0008.jpg 30 | -rw-r--r-- 1 leon 197121 41897485 May 2 22:29 IMG_0008.png 31 | -rw-r--r-- 1 leon 197121 27731886 May 2 22:29 IMG_0008.zpng 32 | ``` 33 | 34 | FLIF and other formats get better compression ratios but you will wait like 20 seconds for them to finish. 35 | 36 | This compressor runs faster than some JPEG decoders! 37 | This compressor takes less than 6% of the time of the PNG compressor and produces a file that is 66% of the size. 38 | 39 | 40 | #### How it works 41 | 42 | This library is similar to PNG in that the image is first filtered, and then submitted to a data compressor. 43 | The filtering step is a bit simpler and faster but somehow more effective than the one used in PNG. 44 | The data compressor used is Zstd, which makes it significantly faster than PNG to compress and decompress. 45 | 46 | Filtering: 47 | 48 | (1) Reversible color channel transformation. 49 | (2) Split each color channel into a separate color plane. 50 | (3) Subtract each color value from the one to its left. 51 | 52 | This kind of filtering works great for large photographic images and is very fast. 53 | 54 | 55 | #### Experimental results 56 | 57 | I ran a few experiments to arrive at this simple codec: 58 | 59 | Interleaving is a 1% compression win, and a 0.3% performance win: Not used. 60 | 61 | Splitting the data into blocks of 4 at a time actually reduces compression. 62 | 63 | ``` 64 | No filtering: 65 | Total size = 2629842851 66 | Total compress time = 12942621 usec 67 | 68 | Subtract only: 69 | Total size = 1570514796 70 | Total compress time = 16961469 usec 71 | 72 | Color filter, followed by subtract: 73 | Total size = 1514724952 74 | Total compress time = 16554638 usec 75 | 76 | Subtract, followed by color filter: 77 | Total size = 1514724952 78 | Total compress time = 16376380 usec 79 | Total de-compress time = 6511436 usec 80 | Notes: Order of applying filter does not matter but this way is faster. 81 | 82 | Subtract, followed by color filter YUVr from JPEG2000: 83 | Total size = 1506802640 84 | Total compress time = 17169743 usec 85 | Total de-compress time = 7107897 usec 86 | Note: Only 0.5% better compression ratio in trade for performance impact. 87 | 88 | Subtract, followed by color filter, splitting into YUV color planes: 89 | Total size = 1486938616 90 | Total compress time = 14514563 usec 91 | Total de-compress time = 6596546 usec 92 | Note: Huge improvement! Let's call this Zpng! 93 | ``` 94 | 95 | #### Credits 96 | 97 | Software by Christopher A. Taylor mrcatid@gmail.com 98 | 99 | Please reach out if you need support or would like to collaborate on a project. 100 | -------------------------------------------------------------------------------- /zstd/zstd_errors.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_ERRORS_H_398273423 12 | #define ZSTD_ERRORS_H_398273423 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | /*===== dependency =====*/ 19 | #include /* size_t */ 20 | 21 | 22 | /* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ 23 | #ifndef ZSTDERRORLIB_VISIBILITY 24 | # if defined(__GNUC__) && (__GNUC__ >= 4) 25 | # define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default"))) 26 | # else 27 | # define ZSTDERRORLIB_VISIBILITY 28 | # endif 29 | #endif 30 | #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) 31 | # define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY 32 | #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) 33 | # define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ 34 | #else 35 | # define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY 36 | #endif 37 | 38 | /*-********************************************* 39 | * Error codes list 40 | *-********************************************* 41 | * Error codes _values_ are pinned down since v1.3.1 only. 42 | * Therefore, don't rely on values if you may link to any version < v1.3.1. 43 | * 44 | * Only values < 100 are considered stable. 45 | * 46 | * note 1 : this API shall be used with static linking only. 47 | * dynamic linking is not yet officially supported. 48 | * note 2 : Prefer relying on the enum than on its value whenever possible 49 | * This is the only supported way to use the error list < v1.3.1 50 | * note 3 : ZSTD_isError() is always correct, whatever the library version. 51 | **********************************************/ 52 | typedef enum { 53 | ZSTD_error_no_error = 0, 54 | ZSTD_error_GENERIC = 1, 55 | ZSTD_error_prefix_unknown = 10, 56 | ZSTD_error_version_unsupported = 12, 57 | ZSTD_error_frameParameter_unsupported = 14, 58 | ZSTD_error_frameParameter_windowTooLarge = 16, 59 | ZSTD_error_corruption_detected = 20, 60 | ZSTD_error_checksum_wrong = 22, 61 | ZSTD_error_dictionary_corrupted = 30, 62 | ZSTD_error_dictionary_wrong = 32, 63 | ZSTD_error_dictionaryCreation_failed = 34, 64 | ZSTD_error_parameter_unsupported = 40, 65 | ZSTD_error_parameter_outOfBound = 42, 66 | ZSTD_error_tableLog_tooLarge = 44, 67 | ZSTD_error_maxSymbolValue_tooLarge = 46, 68 | ZSTD_error_maxSymbolValue_tooSmall = 48, 69 | ZSTD_error_stage_wrong = 60, 70 | ZSTD_error_init_missing = 62, 71 | ZSTD_error_memory_allocation = 64, 72 | ZSTD_error_workSpace_tooSmall= 66, 73 | ZSTD_error_dstSize_tooSmall = 70, 74 | ZSTD_error_srcSize_wrong = 72, 75 | /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ 76 | ZSTD_error_frameIndex_tooLarge = 100, 77 | ZSTD_error_seekableIO = 102, 78 | ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ 79 | } ZSTD_ErrorCode; 80 | 81 | /*! ZSTD_getErrorCode() : 82 | convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, 83 | which can be used to compare with enum list published above */ 84 | ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); 85 | ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ 86 | 87 | 88 | #if defined (__cplusplus) 89 | } 90 | #endif 91 | 92 | #endif /* ZSTD_ERRORS_H_398273423 */ 93 | -------------------------------------------------------------------------------- /zstd/compiler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_COMPILER_H 12 | #define ZSTD_COMPILER_H 13 | 14 | /*-******************************************************* 15 | * Compiler specifics 16 | *********************************************************/ 17 | /* force inlining */ 18 | #if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ 19 | # define INLINE_KEYWORD inline 20 | #else 21 | # define INLINE_KEYWORD 22 | #endif 23 | 24 | #if defined(__GNUC__) 25 | # define FORCE_INLINE_ATTR __attribute__((always_inline)) 26 | #elif defined(_MSC_VER) 27 | # define FORCE_INLINE_ATTR __forceinline 28 | #else 29 | # define FORCE_INLINE_ATTR 30 | #endif 31 | 32 | /** 33 | * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant 34 | * parameters. They must be inlined for the compiler to elimininate the constant 35 | * branches. 36 | */ 37 | #define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR 38 | /** 39 | * HINT_INLINE is used to help the compiler generate better code. It is *not* 40 | * used for "templates", so it can be tweaked based on the compilers 41 | * performance. 42 | * 43 | * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the 44 | * always_inline attribute. 45 | * 46 | * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline 47 | * attribute. 48 | */ 49 | #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 50 | # define HINT_INLINE static INLINE_KEYWORD 51 | #else 52 | # define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR 53 | #endif 54 | 55 | /* force no inlining */ 56 | #ifdef _MSC_VER 57 | # define FORCE_NOINLINE static __declspec(noinline) 58 | #else 59 | # ifdef __GNUC__ 60 | # define FORCE_NOINLINE static __attribute__((__noinline__)) 61 | # else 62 | # define FORCE_NOINLINE static 63 | # endif 64 | #endif 65 | 66 | /* target attribute */ 67 | #ifndef __has_attribute 68 | #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ 69 | #endif 70 | #if defined(__GNUC__) 71 | # define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) 72 | #else 73 | # define TARGET_ATTRIBUTE(target) 74 | #endif 75 | 76 | /* Enable runtime BMI2 dispatch based on the CPU. 77 | * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. 78 | */ 79 | #ifndef DYNAMIC_BMI2 80 | #if ((defined(__clang__) && __has_attribute(__target__)) \ 81 | || (defined(__GNUC__) \ 82 | && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ 83 | && (defined(__x86_64__) || defined(_M_X86)) \ 84 | && !defined(__BMI2__) 85 | # define DYNAMIC_BMI2 1 86 | #else 87 | # define DYNAMIC_BMI2 0 88 | #endif 89 | #endif 90 | 91 | /* prefetch */ 92 | #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ 93 | # include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ 94 | # define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0) 95 | #elif defined(__GNUC__) 96 | # define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0) 97 | #else 98 | # define PREFETCH(ptr) /* disabled */ 99 | #endif 100 | 101 | /* disable warnings */ 102 | #ifdef _MSC_VER /* Visual Studio */ 103 | # include /* For Visual 2005 */ 104 | # pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ 105 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 106 | # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ 107 | # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ 108 | # pragma warning(disable : 4324) /* disable: C4324: padded structure */ 109 | #endif 110 | 111 | #endif /* ZSTD_COMPILER_H */ 112 | -------------------------------------------------------------------------------- /zstd/zstd_ldm.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | 10 | #ifndef ZSTD_LDM_H 11 | #define ZSTD_LDM_H 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | #include "zstd_compress_internal.h" /* ldmParams_t, U32 */ 18 | #include "zstd.h" /* ZSTD_CCtx, size_t */ 19 | 20 | /*-************************************* 21 | * Long distance matching 22 | ***************************************/ 23 | 24 | #define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX 25 | 26 | /** 27 | * ZSTD_ldm_generateSequences(): 28 | * 29 | * Generates the sequences using the long distance match finder. 30 | * Generates long range matching sequences in `sequences`, which parse a prefix 31 | * of the source. `sequences` must be large enough to store every sequence, 32 | * which can be checked with `ZSTD_ldm_getMaxNbSeq()`. 33 | * @returns 0 or an error code. 34 | * 35 | * NOTE: The user must have called ZSTD_window_update() for all of the input 36 | * they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks. 37 | * NOTE: This function returns an error if it runs out of space to store 38 | * sequences. 39 | */ 40 | size_t ZSTD_ldm_generateSequences( 41 | ldmState_t* ldms, rawSeqStore_t* sequences, 42 | ldmParams_t const* params, void const* src, size_t srcSize); 43 | 44 | /** 45 | * ZSTD_ldm_blockCompress(): 46 | * 47 | * Compresses a block using the predefined sequences, along with a secondary 48 | * block compressor. The literals section of every sequence is passed to the 49 | * secondary block compressor, and those sequences are interspersed with the 50 | * predefined sequences. Returns the length of the last literals. 51 | * Updates `rawSeqStore.pos` to indicate how many sequences have been consumed. 52 | * `rawSeqStore.seq` may also be updated to split the last sequence between two 53 | * blocks. 54 | * @return The length of the last literals. 55 | * 56 | * NOTE: The source must be at most the maximum block size, but the predefined 57 | * sequences can be any size, and may be longer than the block. In the case that 58 | * they are longer than the block, the last sequences may need to be split into 59 | * two. We handle that case correctly, and update `rawSeqStore` appropriately. 60 | * NOTE: This function does not return any errors. 61 | */ 62 | size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, 63 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 64 | ZSTD_compressionParameters const* cParams, 65 | void const* src, size_t srcSize, 66 | int const extDict); 67 | 68 | /** 69 | * ZSTD_ldm_skipSequences(): 70 | * 71 | * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`. 72 | * Avoids emitting matches less than `minMatch` bytes. 73 | * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). 74 | */ 75 | void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, 76 | U32 const minMatch); 77 | 78 | 79 | /** ZSTD_ldm_getTableSize() : 80 | * Estimate the space needed for long distance matching tables or 0 if LDM is 81 | * disabled. 82 | */ 83 | size_t ZSTD_ldm_getTableSize(ldmParams_t params); 84 | 85 | /** ZSTD_ldm_getSeqSpace() : 86 | * Return an upper bound on the number of sequences that can be produced by 87 | * the long distance matcher, or 0 if LDM is disabled. 88 | */ 89 | size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize); 90 | 91 | /** ZSTD_ldm_getTableSize() : 92 | * Return prime8bytes^(minMatchLength-1) */ 93 | U64 ZSTD_ldm_getHashPower(U32 minMatchLength); 94 | 95 | /** ZSTD_ldm_adjustParameters() : 96 | * If the params->hashEveryLog is not set, set it to its default value based on 97 | * windowLog and params->hashLog. 98 | * 99 | * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to 100 | * params->hashLog if it is not). 101 | * 102 | * Ensures that the minMatchLength >= targetLength during optimal parsing. 103 | */ 104 | void ZSTD_ldm_adjustParameters(ldmParams_t* params, 105 | ZSTD_compressionParameters const* cParams); 106 | 107 | #if defined (__cplusplus) 108 | } 109 | #endif 110 | 111 | #endif /* ZSTD_FAST_H */ 112 | -------------------------------------------------------------------------------- /zstd/threading.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016 Tino Reichardt 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * 9 | * You can contact the author at: 10 | * - zstdmt source repository: https://github.com/mcmilk/zstdmt 11 | */ 12 | 13 | #ifndef THREADING_H_938743 14 | #define THREADING_H_938743 15 | 16 | #if defined (__cplusplus) 17 | extern "C" { 18 | #endif 19 | 20 | #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) 21 | 22 | /** 23 | * Windows minimalist Pthread Wrapper, based on : 24 | * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html 25 | */ 26 | #ifdef WINVER 27 | # undef WINVER 28 | #endif 29 | #define WINVER 0x0600 30 | 31 | #ifdef _WIN32_WINNT 32 | # undef _WIN32_WINNT 33 | #endif 34 | #define _WIN32_WINNT 0x0600 35 | 36 | #ifndef WIN32_LEAN_AND_MEAN 37 | # define WIN32_LEAN_AND_MEAN 38 | #endif 39 | 40 | #undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ 41 | #include 42 | #undef ERROR 43 | #define ERROR(name) ZSTD_ERROR(name) 44 | 45 | 46 | /* mutex */ 47 | #define ZSTD_pthread_mutex_t CRITICAL_SECTION 48 | #define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0) 49 | #define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a)) 50 | #define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a)) 51 | #define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a)) 52 | 53 | /* condition variable */ 54 | #define ZSTD_pthread_cond_t CONDITION_VARIABLE 55 | #define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0) 56 | #define ZSTD_pthread_cond_destroy(a) ((void)(a)) 57 | #define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE) 58 | #define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a)) 59 | #define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) 60 | 61 | /* ZSTD_pthread_create() and ZSTD_pthread_join() */ 62 | typedef struct { 63 | HANDLE handle; 64 | void* (*start_routine)(void*); 65 | void* arg; 66 | } ZSTD_pthread_t; 67 | 68 | int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, 69 | void* (*start_routine) (void*), void* arg); 70 | 71 | int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); 72 | 73 | /** 74 | * add here more wrappers as required 75 | */ 76 | 77 | 78 | #elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ 79 | /* === POSIX Systems === */ 80 | # include 81 | 82 | #define ZSTD_pthread_mutex_t pthread_mutex_t 83 | #define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) 84 | #define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) 85 | #define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a)) 86 | #define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a)) 87 | 88 | #define ZSTD_pthread_cond_t pthread_cond_t 89 | #define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b)) 90 | #define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a)) 91 | #define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b)) 92 | #define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a)) 93 | #define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a)) 94 | 95 | #define ZSTD_pthread_t pthread_t 96 | #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) 97 | #define ZSTD_pthread_join(a, b) pthread_join((a),(b)) 98 | 99 | #else /* ZSTD_MULTITHREAD not defined */ 100 | /* No multithreading support */ 101 | 102 | typedef int ZSTD_pthread_mutex_t; 103 | #define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0) 104 | #define ZSTD_pthread_mutex_destroy(a) ((void)(a)) 105 | #define ZSTD_pthread_mutex_lock(a) ((void)(a)) 106 | #define ZSTD_pthread_mutex_unlock(a) ((void)(a)) 107 | 108 | typedef int ZSTD_pthread_cond_t; 109 | #define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0) 110 | #define ZSTD_pthread_cond_destroy(a) ((void)(a)) 111 | #define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b)) 112 | #define ZSTD_pthread_cond_signal(a) ((void)(a)) 113 | #define ZSTD_pthread_cond_broadcast(a) ((void)(a)) 114 | 115 | /* do not use ZSTD_pthread_t */ 116 | 117 | #endif /* ZSTD_MULTITHREAD */ 118 | 119 | #if defined (__cplusplus) 120 | } 121 | #endif 122 | 123 | #endif /* THREADING_H_938743 */ 124 | -------------------------------------------------------------------------------- /zpng.h: -------------------------------------------------------------------------------- 1 | /** \file 2 | \brief Zpng - Experimental Lossless Image Compressor 3 | \copyright Copyright (c) 2018 Christopher A. Taylor. All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, 9 | this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | * Neither the name of Zpng nor the names of its contributors may be 14 | used to endorse or promote products derived from this software without 15 | specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 21 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef CAT_ZPNG_H 31 | #define CAT_ZPNG_H 32 | 33 | /** \page Zpng - Experimental Lossless Image Compressor 34 | 35 | It's much faster than PNG and compresses better for photographic images. 36 | This compressor often takes less than 6% of the time of a PNG compressor 37 | and produces a file that is 66% of the size. 38 | It was written in about 600 lines of C code thanks to the Zstd library. 39 | 40 | This library is similar to PNG in that the image is first filtered, 41 | and then submitted to a data compressor. 42 | The filtering step is a bit simpler and faster but somehow more effective 43 | than the one used in PNG. 44 | The data compressor used is Zstd, which makes it significantly faster 45 | than PNG to compress and decompress. 46 | 47 | Filtering: 48 | 49 | (1) Reversible color channel transformation. 50 | (2) Split each color channel into a separate color plane. 51 | (3) Subtract each color value from the one to its left. 52 | */ 53 | 54 | #include 55 | 56 | #ifdef __cplusplus 57 | extern "C" { 58 | #endif 59 | 60 | // Buffer returned by library 61 | struct ZPNG_Buffer 62 | { 63 | // Pointer to data 64 | unsigned char* Data; 65 | 66 | // Size of buffer in bytes 67 | unsigned Bytes; 68 | }; 69 | 70 | // Image data returned by the library 71 | struct ZPNG_ImageData 72 | { 73 | // Pixel data 74 | ZPNG_Buffer Buffer; 75 | 76 | // Number of bytes for each color channel (1-2) 77 | unsigned BytesPerChannel; 78 | 79 | // Number of channels for each pixel (1-4) 80 | unsigned Channels; 81 | 82 | // Width in pixels of image 83 | unsigned WidthPixels; 84 | 85 | // Height in pixels of image 86 | unsigned HeightPixels; 87 | 88 | // Width of pixel row in bytes 89 | unsigned StrideBytes; 90 | }; 91 | 92 | 93 | //------------------------------------------------------------------------------ 94 | // API 95 | 96 | /** 97 | ZPNG_Compress() 98 | 99 | Compress image into a buffer. 100 | 101 | The returned buffer should be passed to ZPNG_Free(). 102 | 103 | On success returns a valid data pointer. 104 | On failure returns a null pointer. 105 | */ 106 | ZPNG_Buffer ZPNG_Compress( 107 | const ZPNG_ImageData* imageData 108 | ); 109 | 110 | /* 111 | ZPNG_Decompress() 112 | 113 | Decompress image from a buffer. 114 | 115 | The returned ZPNG_Buffer should be passed to ZPNG_Free(). 116 | 117 | On success returns a valid data pointer. 118 | On failure returns a null pointer. 119 | */ 120 | ZPNG_ImageData ZPNG_Decompress( 121 | ZPNG_Buffer buffer 122 | ); 123 | 124 | /* 125 | ZPNG_Free() 126 | 127 | Free buffer when done to avoid leaks. 128 | 129 | This will also set the buffer data pointer to null. 130 | */ 131 | void ZPNG_Free( 132 | ZPNG_Buffer* buffer 133 | ); 134 | 135 | 136 | #ifdef __cplusplus 137 | } 138 | #endif 139 | 140 | 141 | #endif // CAT_ZPNG_H 142 | -------------------------------------------------------------------------------- /proj/zpng.vcxproj.filters: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Zstd 7 | 8 | 9 | Zstd 10 | 11 | 12 | Zstd 13 | 14 | 15 | Zstd 16 | 17 | 18 | Zstd 19 | 20 | 21 | Zstd 22 | 23 | 24 | Zstd 25 | 26 | 27 | Zstd 28 | 29 | 30 | Zstd 31 | 32 | 33 | Zstd 34 | 35 | 36 | Zstd 37 | 38 | 39 | Zstd 40 | 41 | 42 | Zstd 43 | 44 | 45 | Zstd 46 | 47 | 48 | Zstd 49 | 50 | 51 | Zstd 52 | 53 | 54 | Zstd 55 | 56 | 57 | Zstd 58 | 59 | 60 | Zstd 61 | 62 | 63 | Zstd 64 | 65 | 66 | Zstd 67 | 68 | 69 | 70 | 71 | 72 | Zstd 73 | 74 | 75 | Zstd 76 | 77 | 78 | Zstd 79 | 80 | 81 | Zstd 82 | 83 | 84 | Zstd 85 | 86 | 87 | Zstd 88 | 89 | 90 | Zstd 91 | 92 | 93 | Zstd 94 | 95 | 96 | Zstd 97 | 98 | 99 | Zstd 100 | 101 | 102 | Zstd 103 | 104 | 105 | Zstd 106 | 107 | 108 | Zstd 109 | 110 | 111 | Zstd 112 | 113 | 114 | Zstd 115 | 116 | 117 | Zstd 118 | 119 | 120 | Zstd 121 | 122 | 123 | Zstd 124 | 125 | 126 | Zstd 127 | 128 | 129 | Zstd 130 | 131 | 132 | Zstd 133 | 134 | 135 | Zstd 136 | 137 | 138 | 139 | 140 | {084ebdbd-c9d6-4712-b2ef-77849057123b} 141 | 142 | 143 | -------------------------------------------------------------------------------- /zstd/cpu.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018-present, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_COMMON_CPU_H 12 | #define ZSTD_COMMON_CPU_H 13 | 14 | /** 15 | * Implementation taken from folly/CpuId.h 16 | * https://github.com/facebook/folly/blob/master/folly/CpuId.h 17 | */ 18 | 19 | #include 20 | 21 | #include "mem.h" 22 | 23 | #ifdef _MSC_VER 24 | #include 25 | #endif 26 | 27 | typedef struct { 28 | U32 f1c; 29 | U32 f1d; 30 | U32 f7b; 31 | U32 f7c; 32 | } ZSTD_cpuid_t; 33 | 34 | MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { 35 | U32 f1c = 0; 36 | U32 f1d = 0; 37 | U32 f7b = 0; 38 | U32 f7c = 0; 39 | #ifdef _MSC_VER 40 | int reg[4]; 41 | __cpuid((int*)reg, 0); 42 | { 43 | int const n = reg[0]; 44 | if (n >= 1) { 45 | __cpuid((int*)reg, 1); 46 | f1c = (U32)reg[2]; 47 | f1d = (U32)reg[3]; 48 | } 49 | if (n >= 7) { 50 | __cpuidex((int*)reg, 7, 0); 51 | f7b = (U32)reg[1]; 52 | f7c = (U32)reg[2]; 53 | } 54 | } 55 | #elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) 56 | /* The following block like the normal cpuid branch below, but gcc 57 | * reserves ebx for use of its pic register so we must specially 58 | * handle the save and restore to avoid clobbering the register 59 | */ 60 | U32 n; 61 | __asm__( 62 | "pushl %%ebx\n\t" 63 | "cpuid\n\t" 64 | "popl %%ebx\n\t" 65 | : "=a"(n) 66 | : "a"(0) 67 | : "ecx", "edx"); 68 | if (n >= 1) { 69 | U32 f1a; 70 | __asm__( 71 | "pushl %%ebx\n\t" 72 | "cpuid\n\t" 73 | "popl %%ebx\n\t" 74 | : "=a"(f1a), "=c"(f1c), "=d"(f1d) 75 | : "a"(1) 76 | :); 77 | } 78 | if (n >= 7) { 79 | __asm__( 80 | "pushl %%ebx\n\t" 81 | "cpuid\n\t" 82 | "movl %%ebx, %%eax\n\r" 83 | "popl %%ebx" 84 | : "=a"(f7b), "=c"(f7c) 85 | : "a"(7), "c"(0) 86 | : "edx"); 87 | } 88 | #elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) 89 | U32 n; 90 | __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); 91 | if (n >= 1) { 92 | U32 f1a; 93 | __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); 94 | } 95 | if (n >= 7) { 96 | U32 f7a; 97 | __asm__("cpuid" 98 | : "=a"(f7a), "=b"(f7b), "=c"(f7c) 99 | : "a"(7), "c"(0) 100 | : "edx"); 101 | } 102 | #endif 103 | { 104 | ZSTD_cpuid_t cpuid; 105 | cpuid.f1c = f1c; 106 | cpuid.f1d = f1d; 107 | cpuid.f7b = f7b; 108 | cpuid.f7c = f7c; 109 | return cpuid; 110 | } 111 | } 112 | 113 | #define X(name, r, bit) \ 114 | MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ 115 | return ((cpuid.r) & (1U << bit)) != 0; \ 116 | } 117 | 118 | /* cpuid(1): Processor Info and Feature Bits. */ 119 | #define C(name, bit) X(name, f1c, bit) 120 | C(sse3, 0) 121 | C(pclmuldq, 1) 122 | C(dtes64, 2) 123 | C(monitor, 3) 124 | C(dscpl, 4) 125 | C(vmx, 5) 126 | C(smx, 6) 127 | C(eist, 7) 128 | C(tm2, 8) 129 | C(ssse3, 9) 130 | C(cnxtid, 10) 131 | C(fma, 12) 132 | C(cx16, 13) 133 | C(xtpr, 14) 134 | C(pdcm, 15) 135 | C(pcid, 17) 136 | C(dca, 18) 137 | C(sse41, 19) 138 | C(sse42, 20) 139 | C(x2apic, 21) 140 | C(movbe, 22) 141 | C(popcnt, 23) 142 | C(tscdeadline, 24) 143 | C(aes, 25) 144 | C(xsave, 26) 145 | C(osxsave, 27) 146 | C(avx, 28) 147 | C(f16c, 29) 148 | C(rdrand, 30) 149 | #undef C 150 | #define D(name, bit) X(name, f1d, bit) 151 | D(fpu, 0) 152 | D(vme, 1) 153 | D(de, 2) 154 | D(pse, 3) 155 | D(tsc, 4) 156 | D(msr, 5) 157 | D(pae, 6) 158 | D(mce, 7) 159 | D(cx8, 8) 160 | D(apic, 9) 161 | D(sep, 11) 162 | D(mtrr, 12) 163 | D(pge, 13) 164 | D(mca, 14) 165 | D(cmov, 15) 166 | D(pat, 16) 167 | D(pse36, 17) 168 | D(psn, 18) 169 | D(clfsh, 19) 170 | D(ds, 21) 171 | D(acpi, 22) 172 | D(mmx, 23) 173 | D(fxsr, 24) 174 | D(sse, 25) 175 | D(sse2, 26) 176 | D(ss, 27) 177 | D(htt, 28) 178 | D(tm, 29) 179 | D(pbe, 31) 180 | #undef D 181 | 182 | /* cpuid(7): Extended Features. */ 183 | #define B(name, bit) X(name, f7b, bit) 184 | B(bmi1, 3) 185 | B(hle, 4) 186 | B(avx2, 5) 187 | B(smep, 7) 188 | B(bmi2, 8) 189 | B(erms, 9) 190 | B(invpcid, 10) 191 | B(rtm, 11) 192 | B(mpx, 14) 193 | B(avx512f, 16) 194 | B(avx512dq, 17) 195 | B(rdseed, 18) 196 | B(adx, 19) 197 | B(smap, 20) 198 | B(avx512ifma, 21) 199 | B(pcommit, 22) 200 | B(clflushopt, 23) 201 | B(clwb, 24) 202 | B(avx512pf, 26) 203 | B(avx512er, 27) 204 | B(avx512cd, 28) 205 | B(sha, 29) 206 | B(avx512bw, 30) 207 | B(avx512vl, 31) 208 | #undef B 209 | #define C(name, bit) X(name, f7c, bit) 210 | C(prefetchwt1, 0) 211 | C(avx512vbmi, 1) 212 | #undef C 213 | 214 | #undef X 215 | 216 | #endif /* ZSTD_COMMON_CPU_H */ 217 | -------------------------------------------------------------------------------- /apps/zpng_app.cpp: -------------------------------------------------------------------------------- 1 | #include "../zpng.h" 2 | 3 | #include 4 | #include 5 | #include 6 | using namespace std; 7 | 8 | #define STB_IMAGE_IMPLEMENTATION /* compile it here */ 9 | #include "thirdparty/stb_image.h" 10 | 11 | #define STB_IMAGE_WRITE_IMPLEMENTATION 12 | #include "thirdparty/stb_image_write.h" 13 | 14 | 15 | #ifdef _WIN32 16 | #ifndef NOMINMAX 17 | #define NOMINMAX 18 | #endif 19 | #include 20 | #elif __MACH__ 21 | #include 22 | #include 23 | #include 24 | 25 | extern mach_port_t clock_port; 26 | #else 27 | #include 28 | #include 29 | #endif 30 | 31 | 32 | //------------------------------------------------------------------------------ 33 | // Timing 34 | 35 | #ifdef _WIN32 36 | static double PerfFrequencyInverse = 0.; 37 | 38 | static void InitPerfFrequencyInverse() 39 | { 40 | LARGE_INTEGER freq = {}; 41 | if (!::QueryPerformanceFrequency(&freq) || freq.QuadPart == 0) 42 | return; 43 | PerfFrequencyInverse = 1000000. / (double)freq.QuadPart; 44 | } 45 | #elif __MACH__ 46 | static bool m_clock_serv_init = false; 47 | static clock_serv_t m_clock_serv = 0; 48 | 49 | static void InitClockServ() 50 | { 51 | m_clock_serv_init = true; 52 | host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &m_clock_serv); 53 | } 54 | #endif // _WIN32 55 | 56 | uint64_t GetTimeUsec() 57 | { 58 | #ifdef _WIN32 59 | LARGE_INTEGER timeStamp = {}; 60 | if (!::QueryPerformanceCounter(&timeStamp)) 61 | return 0; 62 | if (PerfFrequencyInverse == 0.) 63 | InitPerfFrequencyInverse(); 64 | return (uint64_t)(PerfFrequencyInverse * timeStamp.QuadPart); 65 | #elif __MACH__ 66 | if (!m_clock_serv_init) 67 | InitClockServ(); 68 | 69 | mach_timespec_t tv; 70 | clock_get_time(m_clock_serv, &tv); 71 | 72 | return 1000000 * tv.tv_sec + tv.tv_nsec / 1000; 73 | #else 74 | struct timeval tv; 75 | gettimeofday(&tv, nullptr); 76 | return 1000000 * tv.tv_sec + tv.tv_usec; 77 | #endif 78 | } 79 | 80 | uint64_t GetTimeMsec() 81 | { 82 | return GetTimeUsec() / 1000; 83 | } 84 | 85 | 86 | #include 87 | 88 | std::ifstream::pos_type filesize(const char* filename) 89 | { 90 | std::ifstream in(filename, std::ifstream::ate | std::ifstream::binary); 91 | return in.tellg(); 92 | } 93 | 94 | 95 | 96 | int main(int argc, char** argv) 97 | { 98 | bool compress = false; 99 | bool decompress = false; 100 | const char* sourceFile = nullptr; 101 | const char* destFile = nullptr; 102 | 103 | if (argc >= 4) 104 | { 105 | if (0 == strcmp(argv[1], "-c")) { 106 | compress = true; 107 | } 108 | else if (0 == strcmp(argv[1], "-d")) { 109 | decompress = true; 110 | } 111 | sourceFile = argv[2]; 112 | destFile = argv[3]; 113 | } 114 | 115 | if (compress) 116 | { 117 | cout << "Compressing " << sourceFile << " to " << destFile << endl; 118 | 119 | uint64_t t0 = GetTimeUsec(); 120 | 121 | int x, y, comp; 122 | stbi_uc* data = stbi_load(sourceFile, &x, &y, &comp, 0); 123 | if (!data) 124 | { 125 | cout << "Unable to load file: " << sourceFile << endl; 126 | return -1; 127 | } 128 | 129 | uint64_t t1 = GetTimeUsec(); 130 | 131 | cout << "Loaded " << sourceFile << " in " << (t1 - t0) / 1000.f << " msec" << endl; 132 | 133 | ZPNG_ImageData image; 134 | image.Buffer.Data = data; 135 | image.Buffer.Bytes = x * y * comp; 136 | image.BytesPerChannel = 1; 137 | image.Channels = comp; 138 | image.HeightPixels = y; 139 | image.WidthPixels = x; 140 | image.StrideBytes = x * image.Channels; 141 | 142 | t0 = GetTimeUsec(); 143 | 144 | ZPNG_Buffer buffer = ZPNG_Compress(&image); 145 | 146 | if (!buffer.Data) 147 | { 148 | cout << "ZPNG compression failed" << endl; 149 | return -2; 150 | } 151 | 152 | t1 = GetTimeUsec(); 153 | 154 | cout << "Compressed ZPNG in " << (t1 - t0) / 1000.f << " msec" << endl; 155 | 156 | cout << "ZPNG compression size: " << buffer.Bytes << " bytes" << endl; 157 | 158 | std::ofstream output(destFile, std::ios::binary); 159 | if (!output) 160 | { 161 | cout << "Could not open output file" << endl; 162 | return -3; 163 | } 164 | output.write((char*)buffer.Data, buffer.Bytes); 165 | output.close(); 166 | 167 | stbi_image_free(data); 168 | 169 | ZPNG_Free(&buffer); 170 | } 171 | else if (decompress) 172 | { 173 | cout << "Decompressing " << sourceFile << " to " << destFile << " (output will be PNG format)" << endl; 174 | 175 | std::ifstream input(sourceFile, std::ios::binary); 176 | if (!input) 177 | { 178 | cout << "Could not open input file" << endl; 179 | return -4; 180 | } 181 | 182 | // copies all data into buffer 183 | std::vector fileData(( 184 | std::istreambuf_iterator(input)), 185 | (std::istreambuf_iterator())); 186 | 187 | ZPNG_Buffer buffer; 188 | buffer.Data = (uint8_t*)&fileData[0]; 189 | buffer.Bytes = (unsigned)fileData.size(); 190 | 191 | uint64_t t0 = GetTimeUsec(); 192 | 193 | ZPNG_ImageData decompressResult = ZPNG_Decompress(buffer); 194 | 195 | uint64_t t1 = GetTimeUsec(); 196 | 197 | if (!decompressResult.Buffer.Data) 198 | { 199 | cout << "Decompression failed" << endl; 200 | return -5; 201 | } 202 | 203 | cout << "Decompressed ZPNG in " << (t1 - t0) / 1000.f << " msec" << endl; 204 | 205 | t0 = GetTimeUsec(); 206 | 207 | int writeResult = stbi_write_png( 208 | destFile, 209 | decompressResult.WidthPixels, 210 | decompressResult.HeightPixels, 211 | decompressResult.Channels, 212 | decompressResult.Buffer.Data, 213 | decompressResult.StrideBytes); 214 | 215 | t1 = GetTimeUsec(); 216 | 217 | if (!writeResult) 218 | { 219 | cout << "Failed to compress PNG" << endl; 220 | return -6; 221 | } 222 | 223 | cout << "Compressed PNG in " << (t1 - t0) / 1000.f << " msec" << endl; 224 | 225 | cout << "Wrote decompressed PNG file: " << destFile << endl; 226 | 227 | ZPNG_Free(&decompressResult.Buffer); 228 | } 229 | else 230 | { 231 | cout << "Usage: zpng -c Input.PNG Test.ZPNG" << endl; 232 | cout << "Usage: zpng -d Test.ZPNG Output.PNG" << endl; 233 | } 234 | 235 | return 0; 236 | } 237 | -------------------------------------------------------------------------------- /zstd/zstdmt_compress.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTDMT_COMPRESS_H 12 | #define ZSTDMT_COMPRESS_H 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | 19 | /* Note : This is an internal API. 20 | * Some methods are still exposed (ZSTDLIB_API), 21 | * because it used to be the only way to invoke MT compression. 22 | * Now, it's recommended to use ZSTD_compress_generic() instead. 23 | * These methods will stop being exposed in a future version */ 24 | 25 | /* === Dependencies === */ 26 | #include /* size_t */ 27 | #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ 28 | #include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ 29 | 30 | 31 | /* === Memory management === */ 32 | typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; 33 | ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers); 34 | ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, 35 | ZSTD_customMem cMem); 36 | ZSTDLIB_API size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); 37 | 38 | ZSTDLIB_API size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); 39 | 40 | 41 | /* === Simple one-pass compression function === */ 42 | 43 | ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, 44 | void* dst, size_t dstCapacity, 45 | const void* src, size_t srcSize, 46 | int compressionLevel); 47 | 48 | 49 | 50 | /* === Streaming functions === */ 51 | 52 | ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel); 53 | ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */ 54 | 55 | ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input); 56 | 57 | ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */ 58 | ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */ 59 | 60 | 61 | /* === Advanced functions and parameters === */ 62 | 63 | #ifndef ZSTDMT_JOBSIZE_MIN 64 | # define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */ 65 | #endif 66 | 67 | ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, 68 | void* dst, size_t dstCapacity, 69 | const void* src, size_t srcSize, 70 | const ZSTD_CDict* cdict, 71 | ZSTD_parameters params, 72 | unsigned overlapLog); 73 | 74 | ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, 75 | const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */ 76 | ZSTD_parameters params, 77 | unsigned long long pledgedSrcSize); /* pledgedSrcSize is optional and can be zero == unknown */ 78 | 79 | ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, 80 | const ZSTD_CDict* cdict, 81 | ZSTD_frameParameters fparams, 82 | unsigned long long pledgedSrcSize); /* note : zero means empty */ 83 | 84 | /* ZSTDMT_parameter : 85 | * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */ 86 | typedef enum { 87 | ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */ 88 | ZSTDMT_p_overlapSectionLog /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */ 89 | } ZSTDMT_parameter; 90 | 91 | /* ZSTDMT_setMTCtxParameter() : 92 | * allow setting individual parameters, one at a time, among a list of enums defined in ZSTDMT_parameter. 93 | * The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__ 94 | * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions. 95 | * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ 96 | ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value); 97 | 98 | 99 | /*! ZSTDMT_compressStream_generic() : 100 | * Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream() 101 | * depending on flush directive. 102 | * @return : minimum amount of data still to be flushed 103 | * 0 if fully flushed 104 | * or an error code 105 | * note : needs to be init using any ZSTD_initCStream*() variant */ 106 | ZSTDLIB_API size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, 107 | ZSTD_outBuffer* output, 108 | ZSTD_inBuffer* input, 109 | ZSTD_EndDirective endOp); 110 | 111 | 112 | /* ======================================================== 113 | * === Private interface, for use by ZSTD_compress.c === 114 | * === Not exposed in libzstd. Never invoke directly === 115 | * ======================================================== */ 116 | 117 | size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value); 118 | 119 | /* ZSTDMT_CCtxParam_setNbWorkers() 120 | * Set nbWorkers, and clamp it. 121 | * Also reset jobSize and overlapLog */ 122 | size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers); 123 | 124 | /*! ZSTDMT_updateCParams_whileCompressing() : 125 | * Updates only a selected set of compression parameters, to remain compatible with current frame. 126 | * New parameters will be applied to next compression job. */ 127 | void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams); 128 | 129 | /* ZSTDMT_getNbWorkers(): 130 | * @return nb threads currently active in mtctx. 131 | * mtctx must be valid */ 132 | unsigned ZSTDMT_getNbWorkers(const ZSTDMT_CCtx* mtctx); 133 | 134 | /* ZSTDMT_getFrameProgression(): 135 | * tells how much data has been consumed (input) and produced (output) for current frame. 136 | * able to count progression inside worker threads. 137 | */ 138 | ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx); 139 | 140 | 141 | /*! ZSTDMT_initCStream_internal() : 142 | * Private use only. Init streaming operation. 143 | * expects params to be valid. 144 | * must receive dict, or cdict, or none, but not both. 145 | * @return : 0, or an error code */ 146 | size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, 147 | const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, 148 | const ZSTD_CDict* cdict, 149 | ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); 150 | 151 | 152 | #if defined (__cplusplus) 153 | } 154 | #endif 155 | 156 | #endif /* ZSTDMT_COMPRESS_H */ 157 | -------------------------------------------------------------------------------- /proj/ZpngTester.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | Debug 14 | x64 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | {3D09F90D-738F-44F6-A846-859DF5C1F35D} 23 | Win32Proj 24 | ZpngTester 25 | 8.1 26 | 27 | 28 | 29 | Application 30 | true 31 | v140 32 | Unicode 33 | 34 | 35 | Application 36 | false 37 | v140 38 | true 39 | Unicode 40 | 41 | 42 | Application 43 | true 44 | v140 45 | Unicode 46 | 47 | 48 | Application 49 | false 50 | v140 51 | true 52 | Unicode 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | true 74 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 75 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 76 | 77 | 78 | true 79 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 80 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 81 | 82 | 83 | false 84 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 85 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 86 | 87 | 88 | false 89 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 90 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 91 | 92 | 93 | 94 | 95 | 96 | Level3 97 | Disabled 98 | WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) 99 | 100 | 101 | Console 102 | true 103 | 104 | 105 | 106 | 107 | 108 | 109 | Level3 110 | Disabled 111 | _DEBUG;_CONSOLE;%(PreprocessorDefinitions) 112 | 113 | 114 | Console 115 | true 116 | 117 | 118 | 119 | 120 | Level3 121 | 122 | 123 | Full 124 | true 125 | true 126 | WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 127 | AnySuitable 128 | Speed 129 | true 130 | MultiThreaded 131 | false 132 | false 133 | 134 | 135 | Console 136 | true 137 | true 138 | true 139 | 140 | 141 | 142 | 143 | Level3 144 | 145 | 146 | Full 147 | true 148 | true 149 | NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 150 | AnySuitable 151 | Speed 152 | true 153 | MultiThreaded 154 | false 155 | false 156 | 157 | 158 | Console 159 | true 160 | true 161 | true 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | {6218a984-db56-41d8-a2b9-3ffefed36a69} 174 | 175 | 176 | 177 | 178 | 179 | -------------------------------------------------------------------------------- /proj/ZpngApp.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | Debug 14 | x64 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | {36C2D12E-C734-49B0-8528-E57B3CF37909} 23 | Win32Proj 24 | ZpngTester 25 | 8.1 26 | 27 | 28 | 29 | Application 30 | true 31 | v140 32 | Unicode 33 | 34 | 35 | Application 36 | false 37 | v140 38 | true 39 | Unicode 40 | 41 | 42 | Application 43 | true 44 | v140 45 | Unicode 46 | 47 | 48 | Application 49 | false 50 | v140 51 | true 52 | Unicode 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | true 74 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 75 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 76 | 77 | 78 | true 79 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 80 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 81 | 82 | 83 | false 84 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 85 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 86 | 87 | 88 | false 89 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 90 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 91 | 92 | 93 | 94 | 95 | 96 | Level3 97 | Disabled 98 | WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) 99 | 100 | 101 | Console 102 | true 103 | $(SolutionDir) 104 | 105 | 106 | 107 | 108 | 109 | 110 | Level3 111 | Disabled 112 | _DEBUG;_CONSOLE;%(PreprocessorDefinitions) 113 | 114 | 115 | Console 116 | true 117 | $(SolutionDir) 118 | 119 | 120 | 121 | 122 | Level3 123 | 124 | 125 | Full 126 | true 127 | true 128 | WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 129 | AnySuitable 130 | Speed 131 | true 132 | false 133 | MultiThreaded 134 | false 135 | 136 | 137 | Console 138 | true 139 | true 140 | true 141 | $(SolutionDir) 142 | 143 | 144 | 145 | 146 | Level3 147 | 148 | 149 | Full 150 | true 151 | true 152 | NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 153 | AnySuitable 154 | Speed 155 | true 156 | false 157 | MultiThreaded 158 | false 159 | 160 | 161 | Console 162 | true 163 | true 164 | true 165 | $(SolutionDir) 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | {6218a984-db56-41d8-a2b9-3ffefed36a69} 178 | 179 | 180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /zstd/entropy_common.c: -------------------------------------------------------------------------------- 1 | /* 2 | Common functions of New Generation Entropy library 3 | Copyright (C) 2016, Yann Collet. 4 | 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy 32 | - Public forum : https://groups.google.com/forum/#!forum/lz4c 33 | *************************************************************************** */ 34 | 35 | /* ************************************* 36 | * Dependencies 37 | ***************************************/ 38 | #include "mem.h" 39 | #include "error_private.h" /* ERR_*, ERROR */ 40 | #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ 41 | #include "fse.h" 42 | #define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ 43 | #include "huf.h" 44 | 45 | 46 | /*=== Version ===*/ 47 | unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } 48 | 49 | 50 | /*=== Error Management ===*/ 51 | unsigned FSE_isError(size_t code) { return ERR_isError(code); } 52 | const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } 53 | 54 | unsigned HUF_isError(size_t code) { return ERR_isError(code); } 55 | const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } 56 | 57 | 58 | /*-************************************************************** 59 | * FSE NCount encoding-decoding 60 | ****************************************************************/ 61 | size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, 62 | const void* headerBuffer, size_t hbSize) 63 | { 64 | const BYTE* const istart = (const BYTE*) headerBuffer; 65 | const BYTE* const iend = istart + hbSize; 66 | const BYTE* ip = istart; 67 | int nbBits; 68 | int remaining; 69 | int threshold; 70 | U32 bitStream; 71 | int bitCount; 72 | unsigned charnum = 0; 73 | int previous0 = 0; 74 | 75 | if (hbSize < 4) return ERROR(srcSize_wrong); 76 | bitStream = MEM_readLE32(ip); 77 | nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ 78 | if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); 79 | bitStream >>= 4; 80 | bitCount = 4; 81 | *tableLogPtr = nbBits; 82 | remaining = (1<1) & (charnum<=*maxSVPtr)) { 87 | if (previous0) { 88 | unsigned n0 = charnum; 89 | while ((bitStream & 0xFFFF) == 0xFFFF) { 90 | n0 += 24; 91 | if (ip < iend-5) { 92 | ip += 2; 93 | bitStream = MEM_readLE32(ip) >> bitCount; 94 | } else { 95 | bitStream >>= 16; 96 | bitCount += 16; 97 | } } 98 | while ((bitStream & 3) == 3) { 99 | n0 += 3; 100 | bitStream >>= 2; 101 | bitCount += 2; 102 | } 103 | n0 += bitStream & 3; 104 | bitCount += 2; 105 | if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); 106 | while (charnum < n0) normalizedCounter[charnum++] = 0; 107 | if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { 108 | ip += bitCount>>3; 109 | bitCount &= 7; 110 | bitStream = MEM_readLE32(ip) >> bitCount; 111 | } else { 112 | bitStream >>= 2; 113 | } } 114 | { int const max = (2*threshold-1) - remaining; 115 | int count; 116 | 117 | if ((bitStream & (threshold-1)) < (U32)max) { 118 | count = bitStream & (threshold-1); 119 | bitCount += nbBits-1; 120 | } else { 121 | count = bitStream & (2*threshold-1); 122 | if (count >= threshold) count -= max; 123 | bitCount += nbBits; 124 | } 125 | 126 | count--; /* extra accuracy */ 127 | remaining -= count < 0 ? -count : count; /* -1 means +1 */ 128 | normalizedCounter[charnum++] = (short)count; 129 | previous0 = !count; 130 | while (remaining < threshold) { 131 | nbBits--; 132 | threshold >>= 1; 133 | } 134 | 135 | if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { 136 | ip += bitCount>>3; 137 | bitCount &= 7; 138 | } else { 139 | bitCount -= (int)(8 * (iend - 4 - ip)); 140 | ip = iend - 4; 141 | } 142 | bitStream = MEM_readLE32(ip) >> (bitCount & 31); 143 | } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ 144 | if (remaining != 1) return ERROR(corruption_detected); 145 | if (bitCount > 32) return ERROR(corruption_detected); 146 | *maxSVPtr = charnum-1; 147 | 148 | ip += (bitCount+7)>>3; 149 | return ip-istart; 150 | } 151 | 152 | 153 | /*! HUF_readStats() : 154 | Read compact Huffman tree, saved by HUF_writeCTable(). 155 | `huffWeight` is destination buffer. 156 | `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. 157 | @return : size read from `src` , or an error Code . 158 | Note : Needed by HUF_readCTable() and HUF_readDTableX?() . 159 | */ 160 | size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, 161 | U32* nbSymbolsPtr, U32* tableLogPtr, 162 | const void* src, size_t srcSize) 163 | { 164 | U32 weightTotal; 165 | const BYTE* ip = (const BYTE*) src; 166 | size_t iSize; 167 | size_t oSize; 168 | 169 | if (!srcSize) return ERROR(srcSize_wrong); 170 | iSize = ip[0]; 171 | /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ 172 | 173 | if (iSize >= 128) { /* special header */ 174 | oSize = iSize - 127; 175 | iSize = ((oSize+1)/2); 176 | if (iSize+1 > srcSize) return ERROR(srcSize_wrong); 177 | if (oSize >= hwSize) return ERROR(corruption_detected); 178 | ip += 1; 179 | { U32 n; 180 | for (n=0; n> 4; 182 | huffWeight[n+1] = ip[n/2] & 15; 183 | } } } 184 | else { /* header compressed with FSE (normal case) */ 185 | FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */ 186 | if (iSize+1 > srcSize) return ERROR(srcSize_wrong); 187 | oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */ 188 | if (FSE_isError(oSize)) return oSize; 189 | } 190 | 191 | /* collect weight stats */ 192 | memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); 193 | weightTotal = 0; 194 | { U32 n; for (n=0; n= HUF_TABLELOG_MAX) return ERROR(corruption_detected); 196 | rankStats[huffWeight[n]]++; 197 | weightTotal += (1 << huffWeight[n]) >> 1; 198 | } } 199 | if (weightTotal == 0) return ERROR(corruption_detected); 200 | 201 | /* get last non-null symbol weight (implied, total must be 2^n) */ 202 | { U32 const tableLog = BIT_highbit32(weightTotal) + 1; 203 | if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); 204 | *tableLogPtr = tableLog; 205 | /* determine last weight */ 206 | { U32 const total = 1 << tableLog; 207 | U32 const rest = total - weightTotal; 208 | U32 const verif = 1 << BIT_highbit32(rest); 209 | U32 const lastWeight = BIT_highbit32(rest) + 1; 210 | if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ 211 | huffWeight[oSize] = (BYTE)lastWeight; 212 | rankStats[lastWeight]++; 213 | } } 214 | 215 | /* check tree construction validity */ 216 | if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ 217 | 218 | /* results */ 219 | *nbSymbolsPtr = (U32)(oSize+1); 220 | return iSize+1; 221 | } 222 | -------------------------------------------------------------------------------- /zstd/pool.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | 12 | /* ====== Dependencies ======= */ 13 | #include /* size_t */ 14 | #include "pool.h" 15 | #include "zstd_internal.h" /* ZSTD_malloc, ZSTD_free */ 16 | 17 | /* ====== Compiler specifics ====== */ 18 | #if defined(_MSC_VER) 19 | # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ 20 | #endif 21 | 22 | 23 | #ifdef ZSTD_MULTITHREAD 24 | 25 | #include "threading.h" /* pthread adaptation */ 26 | 27 | /* A job is a function and an opaque argument */ 28 | typedef struct POOL_job_s { 29 | POOL_function function; 30 | void *opaque; 31 | } POOL_job; 32 | 33 | struct POOL_ctx_s { 34 | ZSTD_customMem customMem; 35 | /* Keep track of the threads */ 36 | ZSTD_pthread_t *threads; 37 | size_t numThreads; 38 | 39 | /* The queue is a circular buffer */ 40 | POOL_job *queue; 41 | size_t queueHead; 42 | size_t queueTail; 43 | size_t queueSize; 44 | 45 | /* The number of threads working on jobs */ 46 | size_t numThreadsBusy; 47 | /* Indicates if the queue is empty */ 48 | int queueEmpty; 49 | 50 | /* The mutex protects the queue */ 51 | ZSTD_pthread_mutex_t queueMutex; 52 | /* Condition variable for pushers to wait on when the queue is full */ 53 | ZSTD_pthread_cond_t queuePushCond; 54 | /* Condition variables for poppers to wait on when the queue is empty */ 55 | ZSTD_pthread_cond_t queuePopCond; 56 | /* Indicates if the queue is shutting down */ 57 | int shutdown; 58 | }; 59 | 60 | /* POOL_thread() : 61 | Work thread for the thread pool. 62 | Waits for jobs and executes them. 63 | @returns : NULL on failure else non-null. 64 | */ 65 | static void* POOL_thread(void* opaque) { 66 | POOL_ctx* const ctx = (POOL_ctx*)opaque; 67 | if (!ctx) { return NULL; } 68 | for (;;) { 69 | /* Lock the mutex and wait for a non-empty queue or until shutdown */ 70 | ZSTD_pthread_mutex_lock(&ctx->queueMutex); 71 | 72 | while (ctx->queueEmpty && !ctx->shutdown) { 73 | ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex); 74 | } 75 | /* empty => shutting down: so stop */ 76 | if (ctx->queueEmpty) { 77 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex); 78 | return opaque; 79 | } 80 | /* Pop a job off the queue */ 81 | { POOL_job const job = ctx->queue[ctx->queueHead]; 82 | ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; 83 | ctx->numThreadsBusy++; 84 | ctx->queueEmpty = ctx->queueHead == ctx->queueTail; 85 | /* Unlock the mutex, signal a pusher, and run the job */ 86 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex); 87 | ZSTD_pthread_cond_signal(&ctx->queuePushCond); 88 | 89 | job.function(job.opaque); 90 | 91 | /* If the intended queue size was 0, signal after finishing job */ 92 | if (ctx->queueSize == 1) { 93 | ZSTD_pthread_mutex_lock(&ctx->queueMutex); 94 | ctx->numThreadsBusy--; 95 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex); 96 | ZSTD_pthread_cond_signal(&ctx->queuePushCond); 97 | } } 98 | } /* for (;;) */ 99 | /* Unreachable */ 100 | } 101 | 102 | POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { 103 | return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); 104 | } 105 | 106 | POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) { 107 | POOL_ctx* ctx; 108 | /* Check the parameters */ 109 | if (!numThreads) { return NULL; } 110 | /* Allocate the context and zero initialize */ 111 | ctx = (POOL_ctx*)ZSTD_calloc(sizeof(POOL_ctx), customMem); 112 | if (!ctx) { return NULL; } 113 | /* Initialize the job queue. 114 | * It needs one extra space since one space is wasted to differentiate empty 115 | * and full queues. 116 | */ 117 | ctx->queueSize = queueSize + 1; 118 | ctx->queue = (POOL_job*)ZSTD_malloc(ctx->queueSize * sizeof(POOL_job), customMem); 119 | ctx->queueHead = 0; 120 | ctx->queueTail = 0; 121 | ctx->numThreadsBusy = 0; 122 | ctx->queueEmpty = 1; 123 | (void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); 124 | (void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); 125 | (void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); 126 | ctx->shutdown = 0; 127 | /* Allocate space for the thread handles */ 128 | ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem); 129 | ctx->numThreads = 0; 130 | ctx->customMem = customMem; 131 | /* Check for errors */ 132 | if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; } 133 | /* Initialize the threads */ 134 | { size_t i; 135 | for (i = 0; i < numThreads; ++i) { 136 | if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) { 137 | ctx->numThreads = i; 138 | POOL_free(ctx); 139 | return NULL; 140 | } } 141 | ctx->numThreads = numThreads; 142 | } 143 | return ctx; 144 | } 145 | 146 | /*! POOL_join() : 147 | Shutdown the queue, wake any sleeping threads, and join all of the threads. 148 | */ 149 | static void POOL_join(POOL_ctx* ctx) { 150 | /* Shut down the queue */ 151 | ZSTD_pthread_mutex_lock(&ctx->queueMutex); 152 | ctx->shutdown = 1; 153 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex); 154 | /* Wake up sleeping threads */ 155 | ZSTD_pthread_cond_broadcast(&ctx->queuePushCond); 156 | ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); 157 | /* Join all of the threads */ 158 | { size_t i; 159 | for (i = 0; i < ctx->numThreads; ++i) { 160 | ZSTD_pthread_join(ctx->threads[i], NULL); 161 | } } 162 | } 163 | 164 | void POOL_free(POOL_ctx *ctx) { 165 | if (!ctx) { return; } 166 | POOL_join(ctx); 167 | ZSTD_pthread_mutex_destroy(&ctx->queueMutex); 168 | ZSTD_pthread_cond_destroy(&ctx->queuePushCond); 169 | ZSTD_pthread_cond_destroy(&ctx->queuePopCond); 170 | ZSTD_free(ctx->queue, ctx->customMem); 171 | ZSTD_free(ctx->threads, ctx->customMem); 172 | ZSTD_free(ctx, ctx->customMem); 173 | } 174 | 175 | size_t POOL_sizeof(POOL_ctx *ctx) { 176 | if (ctx==NULL) return 0; /* supports sizeof NULL */ 177 | return sizeof(*ctx) 178 | + ctx->queueSize * sizeof(POOL_job) 179 | + ctx->numThreads * sizeof(ZSTD_pthread_t); 180 | } 181 | 182 | /** 183 | * Returns 1 if the queue is full and 0 otherwise. 184 | * 185 | * If the queueSize is 1 (the pool was created with an intended queueSize of 0), 186 | * then a queue is empty if there is a thread free and no job is waiting. 187 | */ 188 | static int isQueueFull(POOL_ctx const* ctx) { 189 | if (ctx->queueSize > 1) { 190 | return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize); 191 | } else { 192 | return ctx->numThreadsBusy == ctx->numThreads || 193 | !ctx->queueEmpty; 194 | } 195 | } 196 | 197 | 198 | static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) 199 | { 200 | POOL_job const job = {function, opaque}; 201 | assert(ctx != NULL); 202 | if (ctx->shutdown) return; 203 | 204 | ctx->queueEmpty = 0; 205 | ctx->queue[ctx->queueTail] = job; 206 | ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize; 207 | ZSTD_pthread_cond_signal(&ctx->queuePopCond); 208 | } 209 | 210 | void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) 211 | { 212 | assert(ctx != NULL); 213 | ZSTD_pthread_mutex_lock(&ctx->queueMutex); 214 | /* Wait until there is space in the queue for the new job */ 215 | while (isQueueFull(ctx) && (!ctx->shutdown)) { 216 | ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); 217 | } 218 | POOL_add_internal(ctx, function, opaque); 219 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex); 220 | } 221 | 222 | 223 | int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) 224 | { 225 | assert(ctx != NULL); 226 | ZSTD_pthread_mutex_lock(&ctx->queueMutex); 227 | if (isQueueFull(ctx)) { 228 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex); 229 | return 0; 230 | } 231 | POOL_add_internal(ctx, function, opaque); 232 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex); 233 | return 1; 234 | } 235 | 236 | 237 | #else /* ZSTD_MULTITHREAD not defined */ 238 | 239 | /* ========================== */ 240 | /* No multi-threading support */ 241 | /* ========================== */ 242 | 243 | 244 | /* We don't need any data, but if it is empty, malloc() might return NULL. */ 245 | struct POOL_ctx_s { 246 | int dummy; 247 | }; 248 | static POOL_ctx g_ctx; 249 | 250 | POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { 251 | return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); 252 | } 253 | 254 | POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) { 255 | (void)numThreads; 256 | (void)queueSize; 257 | (void)customMem; 258 | return &g_ctx; 259 | } 260 | 261 | void POOL_free(POOL_ctx* ctx) { 262 | assert(!ctx || ctx == &g_ctx); 263 | (void)ctx; 264 | } 265 | 266 | void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) { 267 | (void)ctx; 268 | function(opaque); 269 | } 270 | 271 | int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) { 272 | (void)ctx; 273 | function(opaque); 274 | return 1; 275 | } 276 | 277 | size_t POOL_sizeof(POOL_ctx* ctx) { 278 | if (ctx==NULL) return 0; /* supports sizeof NULL */ 279 | assert(ctx == &g_ctx); 280 | return sizeof(*ctx); 281 | } 282 | 283 | #endif /* ZSTD_MULTITHREAD */ 284 | -------------------------------------------------------------------------------- /apps/zpng_test.cpp: -------------------------------------------------------------------------------- 1 | #include "../zpng.h" 2 | 3 | #include 4 | using namespace std; 5 | 6 | #define STB_IMAGE_IMPLEMENTATION /* compile it here */ 7 | #include "thirdparty/stb_image.h" 8 | 9 | #define STB_IMAGE_WRITE_IMPLEMENTATION 10 | #include "thirdparty/stb_image_write.h" 11 | 12 | 13 | 14 | #ifdef _WIN32 15 | #ifndef NOMINMAX 16 | #define NOMINMAX 17 | #endif 18 | #include 19 | #elif __MACH__ 20 | #include 21 | #include 22 | #include 23 | 24 | extern mach_port_t clock_port; 25 | #else 26 | #include 27 | #include 28 | #endif 29 | 30 | 31 | //------------------------------------------------------------------------------ 32 | // Timing 33 | 34 | #ifdef _WIN32 35 | static double PerfFrequencyInverse = 0.; 36 | 37 | static void InitPerfFrequencyInverse() 38 | { 39 | LARGE_INTEGER freq = {}; 40 | if (!::QueryPerformanceFrequency(&freq) || freq.QuadPart == 0) 41 | return; 42 | PerfFrequencyInverse = 1000000. / (double)freq.QuadPart; 43 | } 44 | #elif __MACH__ 45 | static bool m_clock_serv_init = false; 46 | static clock_serv_t m_clock_serv = 0; 47 | 48 | static void InitClockServ() 49 | { 50 | m_clock_serv_init = true; 51 | host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &m_clock_serv); 52 | } 53 | #endif // _WIN32 54 | 55 | uint64_t GetTimeUsec() 56 | { 57 | #ifdef _WIN32 58 | LARGE_INTEGER timeStamp = {}; 59 | if (!::QueryPerformanceCounter(&timeStamp)) 60 | return 0; 61 | if (PerfFrequencyInverse == 0.) 62 | InitPerfFrequencyInverse(); 63 | return (uint64_t)(PerfFrequencyInverse * timeStamp.QuadPart); 64 | #elif __MACH__ 65 | if (!m_clock_serv_init) 66 | InitClockServ(); 67 | 68 | mach_timespec_t tv; 69 | clock_get_time(m_clock_serv, &tv); 70 | 71 | return 1000000 * tv.tv_sec + tv.tv_nsec / 1000; 72 | #else 73 | struct timeval tv; 74 | gettimeofday(&tv, nullptr); 75 | return 1000000 * tv.tv_sec + tv.tv_usec; 76 | #endif 77 | } 78 | 79 | uint64_t GetTimeMsec() 80 | { 81 | return GetTimeUsec() / 1000; 82 | } 83 | 84 | 85 | #include 86 | 87 | std::ifstream::pos_type filesize(const char* filename) 88 | { 89 | std::ifstream in(filename, std::ifstream::ate | std::ifstream::binary); 90 | return in.tellg(); 91 | } 92 | 93 | 94 | static const char* kTestImages[] = { 95 | "images/braid/aleph.png", 96 | "images/braid/ape.png", 97 | "images/braid/cannon1.png", 98 | "images/braid/cannon2.png", 99 | "images/braid/claw.png", 100 | "images/braid/clawpipe.png", 101 | "images/braid/door0.png", 102 | "images/braid/door1.png", 103 | "images/braid/door2.png", 104 | "images/braid/door3.png", 105 | "images/braid/door4.png", 106 | "images/braid/door5.png", 107 | "images/braid/door6.png", 108 | "images/braid/flora1.png", 109 | "images/braid/flora2.png", 110 | "images/braid/flora3.png", 111 | "images/braid/flower.png", 112 | "images/braid/gate.png", 113 | "images/braid/greeter1.png", 114 | "images/braid/greeter2.png", 115 | "images/braid/greeter3.png", 116 | "images/braid/greeter4.png", 117 | "images/braid/key1.png", 118 | "images/braid/key2.png", 119 | "images/braid/killsign.png", 120 | "images/braid/mimic1.png", 121 | "images/braid/mimic2.png", 122 | "images/braid/mimic3.png", 123 | "images/braid/mimic4.png", 124 | "images/braid/mimic5.png", 125 | "images/braid/monster1.png", 126 | "images/braid/monster2.png", 127 | "images/braid/stevinus.png", 128 | "images/braid/switch1.png", 129 | "images/braid/switch2.png", 130 | "images/braid/tim0.png", 131 | "images/braid/tim1.png", 132 | "images/braid/tim10.png", 133 | "images/braid/tim11.png", 134 | "images/braid/tim12.png", 135 | "images/braid/tim2.png", 136 | "images/braid/tim3.png", 137 | "images/braid/tim4.png", 138 | "images/braid/tim5.png", 139 | "images/braid/tim6.png", 140 | "images/braid/tim7.png", 141 | "images/braid/tim8.png", 142 | "images/braid/tim9.png", 143 | "images/rgb8bit/bridge.ppm", 144 | "images/rgb8bit/artificial.ppm", 145 | "images/rgb8bit/big_building.ppm", 146 | "images/rgb8bit/big_tree.ppm", 147 | "images/rgb8bit/cathedral.ppm", 148 | "images/rgb8bit/deer.ppm", 149 | "images/rgb8bit/fireworks.ppm", 150 | "images/rgb8bit/flower_foveon.ppm", 151 | "images/rgb8bit/hdr.ppm", 152 | "images/rgb8bit/leaves_iso_1600.ppm", 153 | "images/rgb8bit/leaves_iso_200.ppm", 154 | "images/rgb8bit/nightshot_iso_100.ppm", 155 | "images/rgb8bit/nightshot_iso_1600.ppm", 156 | "images/rgb8bit/spider_web.ppm", 157 | "images/cake/IMG_0008.jpg", 158 | "images/cake/IMG_0017.jpg", 159 | "images/cake/IMG_0025.jpg", 160 | "images/cake/IMG_0033.jpg", 161 | "images/cake/IMG_0041.jpg", 162 | "images/cake/IMG_0009.jpg", 163 | "images/cake/IMG_0018.jpg", 164 | "images/cake/IMG_0026.jpg", 165 | "images/cake/IMG_0034.jpg", 166 | "images/cake/IMG_0042.jpg", 167 | "images/cake/IMG_0010.jpg", 168 | "images/cake/IMG_0019.jpg", 169 | "images/cake/IMG_0027.jpg", 170 | "images/cake/IMG_0035.jpg", 171 | "images/cake/IMG_0043.jpg", 172 | "images/cake/IMG_0011.jpg", 173 | "images/cake/IMG_0020.jpg", 174 | "images/cake/IMG_0028.jpg", 175 | "images/cake/IMG_0036.jpg", 176 | "images/cake/IMG_0044.jpg", 177 | "images/cake/IMG_0013.jpg", 178 | "images/cake/IMG_0021.jpg", 179 | "images/cake/IMG_0029.jpg", 180 | "images/cake/IMG_0037.jpg", 181 | "images/cake/IMG_0045.jpg", 182 | "images/cake/IMG_0014.jpg", 183 | "images/cake/IMG_0022.jpg", 184 | "images/cake/IMG_0030.jpg", 185 | "images/cake/IMG_0038.jpg", 186 | "images/cake/IMG_0046.jpg", 187 | "images/cake/IMG_0015.jpg", 188 | "images/cake/IMG_0023.jpg", 189 | "images/cake/IMG_0031.jpg", 190 | "images/cake/IMG_0039.jpg", 191 | "images/cake/IMG_0047.jpg", 192 | "images/cake/IMG_0016.jpg", 193 | "images/cake/IMG_0024.jpg", 194 | "images/cake/IMG_0032.jpg", 195 | "images/cake/IMG_0040.jpg", 196 | "images/hiking/IMG_0096.jpg", 197 | "images/hiking/IMG_0103.jpg", 198 | "images/hiking/IMG_0109.jpg", 199 | "images/hiking/IMG_0112.jpg", 200 | "images/hiking/IMG_0130.jpg", 201 | "images/hiking/IMG_0102.jpg", 202 | "images/hiking/IMG_0105.jpg", 203 | "images/hiking/IMG_0110.jpg", 204 | "images/hiking/IMG_0125.jpg", 205 | }; 206 | 207 | static const int kNumTestImages = (int)(sizeof(kTestImages) / sizeof(kTestImages[0])); 208 | 209 | 210 | static const bool WritePngOutput = false; 211 | static const bool TestDecompress = true; 212 | 213 | int main(/*int argc, char** argv*/) 214 | { 215 | cout << "Testing ZPNG" << endl; 216 | 217 | uint64_t TotalCompressTime = 0; 218 | uint64_t TotalDecompressTime = 0; 219 | uint64_t TotalCompressedSize = 0; 220 | 221 | for (int i = 0; i < kNumTestImages; ++i) 222 | { 223 | const char* sourceFile = kTestImages[i]; 224 | 225 | uint64_t t0 = GetTimeUsec(); 226 | 227 | int x, y, comp; 228 | stbi_uc* data = stbi_load(sourceFile, &x, &y, &comp, 0); 229 | if (!data) 230 | { 231 | cout << "Unable to load file: " << sourceFile << endl; 232 | continue; 233 | } 234 | 235 | uint64_t t1 = GetTimeUsec(); 236 | 237 | cout << "Loaded " << sourceFile << " in " << (t1 - t0) / 1000.f << " msec" << endl; 238 | 239 | ZPNG_ImageData image; 240 | image.Buffer.Data = data; 241 | image.Buffer.Bytes = x * y * comp; 242 | image.BytesPerChannel = 1; 243 | image.Channels = comp; 244 | image.HeightPixels = y; 245 | image.WidthPixels = x; 246 | image.StrideBytes = x * image.Channels; 247 | 248 | t0 = GetTimeUsec(); 249 | 250 | ZPNG_Buffer buffer = ZPNG_Compress(&image); 251 | 252 | if (!buffer.Data) 253 | { 254 | cout << "ZPNG compression failed" << endl; 255 | return -2; 256 | } 257 | 258 | t1 = GetTimeUsec(); 259 | 260 | TotalCompressTime += t1 - t0; 261 | TotalCompressedSize += buffer.Bytes; 262 | 263 | cout << "Compressed ZPNG in " << (t1 - t0) / 1000.f << " msec" << endl; 264 | 265 | cout << "ZPNG compression size: " << buffer.Bytes << " bytes" << endl; 266 | 267 | if (TestDecompress) 268 | { 269 | t0 = GetTimeUsec(); 270 | 271 | ZPNG_ImageData decompressResult = ZPNG_Decompress(buffer); 272 | 273 | t1 = GetTimeUsec(); 274 | 275 | TotalDecompressTime += t1 - t0; 276 | 277 | cout << "Decompressed ZPNG in " << (t1 - t0) / 1000.f << " msec" << endl; 278 | 279 | if (!decompressResult.Buffer.Data || 280 | decompressResult.BytesPerChannel != 1 || 281 | decompressResult.Channels != comp || 282 | decompressResult.HeightPixels != y || 283 | decompressResult.WidthPixels != x || 284 | decompressResult.StrideBytes != x * comp || 285 | decompressResult.Buffer.Bytes != image.Buffer.Bytes) 286 | { 287 | cout << "Bad output" << endl; 288 | return -3; 289 | } 290 | 291 | if (0 != memcmp(image.Buffer.Data, decompressResult.Buffer.Data, image.Buffer.Bytes)) 292 | { 293 | cout << "Data mismatch" << endl; 294 | return -4; 295 | } 296 | 297 | if (WritePngOutput) 298 | { 299 | t0 = GetTimeUsec(); 300 | 301 | int writeResult = stbi_write_png( 302 | "output.png", 303 | decompressResult.WidthPixels, 304 | decompressResult.HeightPixels, 305 | decompressResult.Channels, 306 | decompressResult.Buffer.Data, 307 | decompressResult.StrideBytes); 308 | 309 | t1 = GetTimeUsec(); 310 | 311 | cout << "Compressed PNG in " << (t1 - t0) / 1000.f << " msec" << endl; 312 | 313 | unsigned pngSize = filesize("output.png"); 314 | 315 | cout << "Compressed PNG size: " << pngSize << " bytes" << endl; 316 | 317 | cout << "ZPNG / PNG ratio = " << buffer.Bytes / (float)pngSize << endl; 318 | 319 | cout << "Wrote decompressed result to output.png. Write result = " << writeResult << endl; 320 | } 321 | 322 | ZPNG_Free(&decompressResult.Buffer); 323 | } 324 | 325 | stbi_image_free(data); 326 | 327 | ZPNG_Free(&buffer); 328 | } 329 | 330 | cout << "Total size = " << TotalCompressedSize << endl; 331 | cout << "Total compress time = " << TotalCompressTime << " usec" << endl; 332 | cout << "Total de-compress time = " << TotalDecompressTime << " usec" << endl; 333 | 334 | return 0; 335 | } 336 | -------------------------------------------------------------------------------- /proj/zpng.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | Debug 14 | x64 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | {6218A984-DB56-41D8-A2B9-3FFEFED36A69} 72 | Win32Proj 73 | zpng 74 | 8.1 75 | 76 | 77 | 78 | StaticLibrary 79 | true 80 | v140 81 | Unicode 82 | 83 | 84 | StaticLibrary 85 | false 86 | v140 87 | true 88 | Unicode 89 | 90 | 91 | StaticLibrary 92 | true 93 | v140 94 | Unicode 95 | 96 | 97 | StaticLibrary 98 | false 99 | v140 100 | true 101 | Unicode 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | true 123 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 124 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 125 | 126 | 127 | true 128 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 129 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 130 | 131 | 132 | false 133 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 134 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 135 | 136 | 137 | false 138 | Output/$(ProjectName)/$(Configuration)/$(Platform)/ 139 | Obj/$(ProjectName)/$(Configuration)/$(Platform)/ 140 | 141 | 142 | 143 | 144 | 145 | Level4 146 | Disabled 147 | WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) 148 | 149 | 150 | Console 151 | true 152 | 153 | 154 | 155 | 156 | 157 | 158 | Level4 159 | Disabled 160 | _DEBUG;_CONSOLE;%(PreprocessorDefinitions) 161 | 162 | 163 | Console 164 | true 165 | 166 | 167 | 168 | 169 | Level4 170 | 171 | 172 | Full 173 | true 174 | true 175 | WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 176 | AnySuitable 177 | Speed 178 | true 179 | MultiThreaded 180 | false 181 | false 182 | 183 | 184 | Console 185 | true 186 | true 187 | true 188 | 189 | 190 | 191 | 192 | Level4 193 | 194 | 195 | Full 196 | true 197 | true 198 | NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 199 | AnySuitable 200 | Speed 201 | true 202 | MultiThreaded 203 | false 204 | false 205 | 206 | 207 | Console 208 | true 209 | true 210 | true 211 | 212 | 213 | 214 | 215 | 216 | -------------------------------------------------------------------------------- /zstd/zstd_internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_CCOMMON_H_MODULE 12 | #define ZSTD_CCOMMON_H_MODULE 13 | 14 | /* this module contains definitions which must be identical 15 | * across compression, decompression and dictBuilder. 16 | * It also contains a few functions useful to at least 2 of them 17 | * and which benefit from being inlined */ 18 | 19 | /*-************************************* 20 | * Dependencies 21 | ***************************************/ 22 | #include "compiler.h" 23 | #include "mem.h" 24 | #include "error_private.h" 25 | #define ZSTD_STATIC_LINKING_ONLY 26 | #include "zstd.h" 27 | #define FSE_STATIC_LINKING_ONLY 28 | #include "fse.h" 29 | #define HUF_STATIC_LINKING_ONLY 30 | #include "huf.h" 31 | #ifndef XXH_STATIC_LINKING_ONLY 32 | # define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ 33 | #endif 34 | #include "xxhash.h" /* XXH_reset, update, digest */ 35 | 36 | 37 | #if defined (__cplusplus) 38 | extern "C" { 39 | #endif 40 | 41 | 42 | /*-************************************* 43 | * Debug 44 | ***************************************/ 45 | #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1) 46 | # include 47 | #else 48 | # ifndef assert 49 | # define assert(condition) ((void)0) 50 | # endif 51 | #endif 52 | 53 | #define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; } 54 | 55 | #if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=2) 56 | # include 57 | extern int g_debuglog_enable; 58 | /* recommended values for ZSTD_DEBUG display levels : 59 | * 1 : no display, enables assert() only 60 | * 2 : reserved for currently active debug path 61 | * 3 : events once per object lifetime (CCtx, CDict, etc.) 62 | * 4 : events once per frame 63 | * 5 : events once per block 64 | * 6 : events once per sequence (*very* verbose) */ 65 | # define RAWLOG(l, ...) { \ 66 | if ((g_debuglog_enable) & (l<=ZSTD_DEBUG)) { \ 67 | fprintf(stderr, __VA_ARGS__); \ 68 | } } 69 | # define DEBUGLOG(l, ...) { \ 70 | if ((g_debuglog_enable) & (l<=ZSTD_DEBUG)) { \ 71 | fprintf(stderr, __FILE__ ": " __VA_ARGS__); \ 72 | fprintf(stderr, " \n"); \ 73 | } } 74 | #else 75 | # define RAWLOG(l, ...) {} /* disabled */ 76 | # define DEBUGLOG(l, ...) {} /* disabled */ 77 | #endif 78 | 79 | 80 | /*-************************************* 81 | * shared macros 82 | ***************************************/ 83 | #undef MIN 84 | #undef MAX 85 | #define MIN(a,b) ((a)<(b) ? (a) : (b)) 86 | #define MAX(a,b) ((a)>(b) ? (a) : (b)) 87 | #define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */ 88 | #define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); } /* check and send Error code */ 89 | 90 | 91 | /*-************************************* 92 | * Common constants 93 | ***************************************/ 94 | #define ZSTD_OPT_NUM (1<<12) 95 | 96 | #define ZSTD_REP_NUM 3 /* number of repcodes */ 97 | #define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) 98 | static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; 99 | 100 | #define KB *(1 <<10) 101 | #define MB *(1 <<20) 102 | #define GB *(1U<<30) 103 | 104 | #define BIT7 128 105 | #define BIT6 64 106 | #define BIT5 32 107 | #define BIT4 16 108 | #define BIT1 2 109 | #define BIT0 1 110 | 111 | #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 112 | #define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */ 113 | static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; 114 | static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; 115 | 116 | #define ZSTD_FRAMEIDSIZE 4 117 | static const size_t ZSTD_frameIdSize = ZSTD_FRAMEIDSIZE; /* magic number size */ 118 | 119 | #define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ 120 | static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; 121 | typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; 122 | 123 | #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ 124 | #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ 125 | 126 | #define HufLog 12 127 | typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; 128 | 129 | #define LONGNBSEQ 0x7F00 130 | 131 | #define MINMATCH 3 132 | 133 | #define Litbits 8 134 | #define MaxLit ((1<= 3) /* GCC Intrinsic */ 252 | return 31 - __builtin_clz(val); 253 | # else /* Software version */ 254 | static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; 255 | U32 v = val; 256 | v |= v >> 1; 257 | v |= v >> 2; 258 | v |= v >> 4; 259 | v |= v >> 8; 260 | v |= v >> 16; 261 | return DeBruijnClz[(v * 0x07C4ACDDU) >> 27]; 262 | # endif 263 | } 264 | } 265 | 266 | 267 | /* ZSTD_invalidateRepCodes() : 268 | * ensures next compression will not use repcodes from previous block. 269 | * Note : only works with regular variant; 270 | * do not use with extDict variant ! */ 271 | void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ 272 | 273 | 274 | typedef struct { 275 | blockType_e blockType; 276 | U32 lastBlock; 277 | U32 origSize; 278 | } blockProperties_t; 279 | 280 | /*! ZSTD_getcBlockSize() : 281 | * Provides the size of compressed block from block header `src` */ 282 | /* Used by: decompress, fullbench (does not get its definition from here) */ 283 | size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, 284 | blockProperties_t* bpPtr); 285 | 286 | #if defined (__cplusplus) 287 | } 288 | #endif 289 | 290 | #endif /* ZSTD_CCOMMON_H_MODULE */ 291 | -------------------------------------------------------------------------------- /zstd/zdict.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef DICTBUILDER_H_001 12 | #define DICTBUILDER_H_001 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | 19 | /*====== Dependencies ======*/ 20 | #include /* size_t */ 21 | 22 | 23 | /* ===== ZDICTLIB_API : control library symbols visibility ===== */ 24 | #ifndef ZDICTLIB_VISIBILITY 25 | # if defined(__GNUC__) && (__GNUC__ >= 4) 26 | # define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default"))) 27 | # else 28 | # define ZDICTLIB_VISIBILITY 29 | # endif 30 | #endif 31 | #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) 32 | # define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY 33 | #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) 34 | # define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ 35 | #else 36 | # define ZDICTLIB_API ZDICTLIB_VISIBILITY 37 | #endif 38 | 39 | 40 | /*! ZDICT_trainFromBuffer(): 41 | * Train a dictionary from an array of samples. 42 | * Redirect towards ZDICT_optimizeTrainFromBuffer_cover() single-threaded, with d=8 and steps=4. 43 | * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, 44 | * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. 45 | * The resulting dictionary will be saved into `dictBuffer`. 46 | * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) 47 | * or an error code, which can be tested with ZDICT_isError(). 48 | * Note: ZDICT_trainFromBuffer() requires about 9 bytes of memory for each input byte. 49 | * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. 50 | * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. 51 | * In general, it's recommended to provide a few thousands samples, though this can vary a lot. 52 | * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. 53 | */ 54 | ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, 55 | const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); 56 | 57 | 58 | /*====== Helper functions ======*/ 59 | ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ 60 | ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); 61 | ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); 62 | 63 | 64 | 65 | #ifdef ZDICT_STATIC_LINKING_ONLY 66 | 67 | /* ==================================================================================== 68 | * The definitions in this section are considered experimental. 69 | * They should never be used with a dynamic library, as they may change in the future. 70 | * They are provided for advanced usages. 71 | * Use them only in association with static linking. 72 | * ==================================================================================== */ 73 | 74 | typedef struct { 75 | int compressionLevel; /* optimize for a specific zstd compression level; 0 means default */ 76 | unsigned notificationLevel; /* Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ 77 | unsigned dictID; /* force dictID value; 0 means auto mode (32-bits random value) */ 78 | } ZDICT_params_t; 79 | 80 | /*! ZDICT_cover_params_t: 81 | * k and d are the only required parameters. 82 | * For others, value 0 means default. 83 | */ 84 | typedef struct { 85 | unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ 86 | unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ 87 | unsigned steps; /* Number of steps : Only used for optimization : 0 means default (32) : Higher means more parameters checked */ 88 | unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ 89 | ZDICT_params_t zParams; 90 | } ZDICT_cover_params_t; 91 | 92 | 93 | /*! ZDICT_trainFromBuffer_cover(): 94 | * Train a dictionary from an array of samples using the COVER algorithm. 95 | * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, 96 | * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. 97 | * The resulting dictionary will be saved into `dictBuffer`. 98 | * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) 99 | * or an error code, which can be tested with ZDICT_isError(). 100 | * Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte. 101 | * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. 102 | * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. 103 | * In general, it's recommended to provide a few thousands samples, though this can vary a lot. 104 | * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. 105 | */ 106 | ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( 107 | void *dictBuffer, size_t dictBufferCapacity, 108 | const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, 109 | ZDICT_cover_params_t parameters); 110 | 111 | /*! ZDICT_optimizeTrainFromBuffer_cover(): 112 | * The same requirements as above hold for all the parameters except `parameters`. 113 | * This function tries many parameter combinations and picks the best parameters. 114 | * `*parameters` is filled with the best parameters found, 115 | * dictionary constructed with those parameters is stored in `dictBuffer`. 116 | * 117 | * All of the parameters d, k, steps are optional. 118 | * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8, 10, 12, 14, 16}. 119 | * if steps is zero it defaults to its default value. 120 | * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [16, 2048]. 121 | * 122 | * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) 123 | * or an error code, which can be tested with ZDICT_isError(). 124 | * On success `*parameters` contains the parameters selected. 125 | * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread. 126 | */ 127 | ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( 128 | void* dictBuffer, size_t dictBufferCapacity, 129 | const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, 130 | ZDICT_cover_params_t* parameters); 131 | 132 | /*! ZDICT_finalizeDictionary(): 133 | * Given a custom content as a basis for dictionary, and a set of samples, 134 | * finalize dictionary by adding headers and statistics. 135 | * 136 | * Samples must be stored concatenated in a flat buffer `samplesBuffer`, 137 | * supplied with an array of sizes `samplesSizes`, providing the size of each sample in order. 138 | * 139 | * dictContentSize must be >= ZDICT_CONTENTSIZE_MIN bytes. 140 | * maxDictSize must be >= dictContentSize, and must be >= ZDICT_DICTSIZE_MIN bytes. 141 | * 142 | * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`), 143 | * or an error code, which can be tested by ZDICT_isError(). 144 | * Note: ZDICT_finalizeDictionary() will push notifications into stderr if instructed to, using notificationLevel>0. 145 | * Note 2: dictBuffer and dictContent can overlap 146 | */ 147 | #define ZDICT_CONTENTSIZE_MIN 128 148 | #define ZDICT_DICTSIZE_MIN 256 149 | ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dictBuffer, size_t dictBufferCapacity, 150 | const void* dictContent, size_t dictContentSize, 151 | const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, 152 | ZDICT_params_t parameters); 153 | 154 | typedef struct { 155 | unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */ 156 | ZDICT_params_t zParams; 157 | } ZDICT_legacy_params_t; 158 | 159 | /*! ZDICT_trainFromBuffer_legacy(): 160 | * Train a dictionary from an array of samples. 161 | * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, 162 | * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. 163 | * The resulting dictionary will be saved into `dictBuffer`. 164 | * `parameters` is optional and can be provided with values set to 0 to mean "default". 165 | * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) 166 | * or an error code, which can be tested with ZDICT_isError(). 167 | * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. 168 | * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. 169 | * In general, it's recommended to provide a few thousands samples, though this can vary a lot. 170 | * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. 171 | * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0. 172 | */ 173 | ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( 174 | void *dictBuffer, size_t dictBufferCapacity, 175 | const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, 176 | ZDICT_legacy_params_t parameters); 177 | 178 | /* Deprecation warnings */ 179 | /* It is generally possible to disable deprecation warnings from compiler, 180 | for example with -Wno-deprecated-declarations for gcc 181 | or _CRT_SECURE_NO_WARNINGS in Visual. 182 | Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */ 183 | #ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS 184 | # define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */ 185 | #else 186 | # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 187 | # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ 188 | # define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API 189 | # elif (ZDICT_GCC_VERSION >= 405) || defined(__clang__) 190 | # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message))) 191 | # elif (ZDICT_GCC_VERSION >= 301) 192 | # define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated)) 193 | # elif defined(_MSC_VER) 194 | # define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message)) 195 | # else 196 | # pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler") 197 | # define ZDICT_DEPRECATED(message) ZDICTLIB_API 198 | # endif 199 | #endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */ 200 | 201 | ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead") 202 | size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, 203 | const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); 204 | 205 | 206 | #endif /* ZDICT_STATIC_LINKING_ONLY */ 207 | 208 | #if defined (__cplusplus) 209 | } 210 | #endif 211 | 212 | #endif /* DICTBUILDER_H_001 */ 213 | -------------------------------------------------------------------------------- /zstd/zstd_fast.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #include "zstd_compress_internal.h" 12 | #include "zstd_fast.h" 13 | 14 | 15 | void ZSTD_fillHashTable(ZSTD_matchState_t* ms, 16 | ZSTD_compressionParameters const* cParams, 17 | void const* end, ZSTD_dictTableLoadMethod_e dtlm) 18 | { 19 | U32* const hashTable = ms->hashTable; 20 | U32 const hBits = cParams->hashLog; 21 | U32 const mls = cParams->searchLength; 22 | const BYTE* const base = ms->window.base; 23 | const BYTE* ip = base + ms->nextToUpdate; 24 | const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; 25 | const U32 fastHashFillStep = 3; 26 | 27 | /* Always insert every fastHashFillStep position into the hash table. 28 | * Insert the other positions if their hash entry is empty. 29 | */ 30 | for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { 31 | U32 const current = (U32)(ip - base); 32 | U32 i; 33 | for (i = 0; i < fastHashFillStep; ++i) { 34 | size_t const hash = ZSTD_hashPtr(ip + i, hBits, mls); 35 | if (i == 0 || hashTable[hash] == 0) 36 | hashTable[hash] = current + i; 37 | /* Only load extra positions for ZSTD_dtlm_full */ 38 | if (dtlm == ZSTD_dtlm_fast) 39 | break; 40 | } 41 | } 42 | } 43 | 44 | FORCE_INLINE_TEMPLATE 45 | size_t ZSTD_compressBlock_fast_generic( 46 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 47 | void const* src, size_t srcSize, 48 | U32 const hlog, U32 const stepSize, U32 const mls) 49 | { 50 | U32* const hashTable = ms->hashTable; 51 | const BYTE* const base = ms->window.base; 52 | const BYTE* const istart = (const BYTE*)src; 53 | const BYTE* ip = istart; 54 | const BYTE* anchor = istart; 55 | const U32 lowestIndex = ms->window.dictLimit; 56 | const BYTE* const lowest = base + lowestIndex; 57 | const BYTE* const iend = istart + srcSize; 58 | const BYTE* const ilimit = iend - HASH_READ_SIZE; 59 | U32 offset_1=rep[0], offset_2=rep[1]; 60 | U32 offsetSaved = 0; 61 | 62 | /* init */ 63 | ip += (ip==lowest); 64 | { U32 const maxRep = (U32)(ip-lowest); 65 | if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; 66 | if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; 67 | } 68 | 69 | /* Main Search Loop */ 70 | while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ 71 | size_t mLength; 72 | size_t const h = ZSTD_hashPtr(ip, hlog, mls); 73 | U32 const current = (U32)(ip-base); 74 | U32 const matchIndex = hashTable[h]; 75 | const BYTE* match = base + matchIndex; 76 | hashTable[h] = current; /* update hash table */ 77 | 78 | if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { 79 | mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; 80 | ip++; 81 | ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); 82 | } else { 83 | if ( (matchIndex <= lowestIndex) 84 | || (MEM_read32(match) != MEM_read32(ip)) ) { 85 | assert(stepSize >= 1); 86 | ip += ((ip-anchor) >> kSearchStrength) + stepSize; 87 | continue; 88 | } 89 | mLength = ZSTD_count(ip+4, match+4, iend) + 4; 90 | { U32 const offset = (U32)(ip-match); 91 | while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ 92 | offset_2 = offset_1; 93 | offset_1 = offset; 94 | ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); 95 | } } 96 | 97 | /* match found */ 98 | ip += mLength; 99 | anchor = ip; 100 | 101 | if (ip <= ilimit) { 102 | /* Fill Table */ 103 | hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; /* here because current+2 could be > iend-8 */ 104 | hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); 105 | /* check immediate repcode */ 106 | while ( (ip <= ilimit) 107 | && ( (offset_2>0) 108 | & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { 109 | /* store sequence */ 110 | size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; 111 | { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ 112 | hashTable[ZSTD_hashPtr(ip, hlog, mls)] = (U32)(ip-base); 113 | ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); 114 | ip += rLength; 115 | anchor = ip; 116 | continue; /* faster when present ... (?) */ 117 | } } } 118 | 119 | /* save reps for next block */ 120 | rep[0] = offset_1 ? offset_1 : offsetSaved; 121 | rep[1] = offset_2 ? offset_2 : offsetSaved; 122 | 123 | /* Return the last literals size */ 124 | return iend - anchor; 125 | } 126 | 127 | 128 | size_t ZSTD_compressBlock_fast( 129 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 130 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) 131 | { 132 | U32 const hlog = cParams->hashLog; 133 | U32 const mls = cParams->searchLength; 134 | U32 const stepSize = cParams->targetLength; 135 | switch(mls) 136 | { 137 | default: /* includes case 3 */ 138 | case 4 : 139 | return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 4); 140 | case 5 : 141 | return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 5); 142 | case 6 : 143 | return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 6); 144 | case 7 : 145 | return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 7); 146 | } 147 | } 148 | 149 | 150 | static size_t ZSTD_compressBlock_fast_extDict_generic( 151 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 152 | void const* src, size_t srcSize, 153 | U32 const hlog, U32 const stepSize, U32 const mls) 154 | { 155 | U32* hashTable = ms->hashTable; 156 | const BYTE* const base = ms->window.base; 157 | const BYTE* const dictBase = ms->window.dictBase; 158 | const BYTE* const istart = (const BYTE*)src; 159 | const BYTE* ip = istart; 160 | const BYTE* anchor = istart; 161 | const U32 lowestIndex = ms->window.lowLimit; 162 | const BYTE* const dictStart = dictBase + lowestIndex; 163 | const U32 dictLimit = ms->window.dictLimit; 164 | const BYTE* const lowPrefixPtr = base + dictLimit; 165 | const BYTE* const dictEnd = dictBase + dictLimit; 166 | const BYTE* const iend = istart + srcSize; 167 | const BYTE* const ilimit = iend - 8; 168 | U32 offset_1=rep[0], offset_2=rep[1]; 169 | 170 | /* Search Loop */ 171 | while (ip < ilimit) { /* < instead of <=, because (ip+1) */ 172 | const size_t h = ZSTD_hashPtr(ip, hlog, mls); 173 | const U32 matchIndex = hashTable[h]; 174 | const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base; 175 | const BYTE* match = matchBase + matchIndex; 176 | const U32 current = (U32)(ip-base); 177 | const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */ 178 | const BYTE* repBase = repIndex < dictLimit ? dictBase : base; 179 | const BYTE* repMatch = repBase + repIndex; 180 | size_t mLength; 181 | hashTable[h] = current; /* update hash table */ 182 | 183 | if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) 184 | && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { 185 | const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; 186 | mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4; 187 | ip++; 188 | ZSTD_storeSeq(seqStore, ip-anchor, anchor, 0, mLength-MINMATCH); 189 | } else { 190 | if ( (matchIndex < lowestIndex) || 191 | (MEM_read32(match) != MEM_read32(ip)) ) { 192 | assert(stepSize >= 1); 193 | ip += ((ip-anchor) >> kSearchStrength) + stepSize; 194 | continue; 195 | } 196 | { const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; 197 | const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; 198 | U32 offset; 199 | mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; 200 | while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ 201 | offset = current - matchIndex; 202 | offset_2 = offset_1; 203 | offset_1 = offset; 204 | ZSTD_storeSeq(seqStore, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); 205 | } } 206 | 207 | /* found a match : store it */ 208 | ip += mLength; 209 | anchor = ip; 210 | 211 | if (ip <= ilimit) { 212 | /* Fill Table */ 213 | hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; 214 | hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); 215 | /* check immediate repcode */ 216 | while (ip <= ilimit) { 217 | U32 const current2 = (U32)(ip-base); 218 | U32 const repIndex2 = current2 - offset_2; 219 | const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; 220 | if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ 221 | && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { 222 | const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; 223 | size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, lowPrefixPtr) + 4; 224 | U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ 225 | ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); 226 | hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; 227 | ip += repLength2; 228 | anchor = ip; 229 | continue; 230 | } 231 | break; 232 | } } } 233 | 234 | /* save reps for next block */ 235 | rep[0] = offset_1; 236 | rep[1] = offset_2; 237 | 238 | /* Return the last literals size */ 239 | return iend - anchor; 240 | } 241 | 242 | 243 | size_t ZSTD_compressBlock_fast_extDict( 244 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 245 | ZSTD_compressionParameters const* cParams, void const* src, size_t srcSize) 246 | { 247 | U32 const hlog = cParams->hashLog; 248 | U32 const mls = cParams->searchLength; 249 | U32 const stepSize = cParams->targetLength; 250 | switch(mls) 251 | { 252 | default: /* includes case 3 */ 253 | case 4 : 254 | return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 4); 255 | case 5 : 256 | return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 5); 257 | case 6 : 258 | return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 6); 259 | case 7 : 260 | return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, hlog, stepSize, 7); 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /zstd/fse_decompress.c: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | FSE : Finite State Entropy decoder 3 | Copyright (C) 2013-2015, Yann Collet. 4 | 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy 32 | - Public forum : https://groups.google.com/forum/#!forum/lz4c 33 | ****************************************************************** */ 34 | 35 | 36 | /* ************************************************************** 37 | * Includes 38 | ****************************************************************/ 39 | #include /* malloc, free, qsort */ 40 | #include /* memcpy, memset */ 41 | #include "bitstream.h" 42 | #include "compiler.h" 43 | #define FSE_STATIC_LINKING_ONLY 44 | #include "fse.h" 45 | #include "error_private.h" 46 | 47 | 48 | /* ************************************************************** 49 | * Error Management 50 | ****************************************************************/ 51 | #define FSE_isError ERR_isError 52 | #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ 53 | 54 | /* check and forward error code */ 55 | #define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } 56 | 57 | 58 | /* ************************************************************** 59 | * Templates 60 | ****************************************************************/ 61 | /* 62 | designed to be included 63 | for type-specific functions (template emulation in C) 64 | Objective is to write these functions only once, for improved maintenance 65 | */ 66 | 67 | /* safety checks */ 68 | #ifndef FSE_FUNCTION_EXTENSION 69 | # error "FSE_FUNCTION_EXTENSION must be defined" 70 | #endif 71 | #ifndef FSE_FUNCTION_TYPE 72 | # error "FSE_FUNCTION_TYPE must be defined" 73 | #endif 74 | 75 | /* Function names */ 76 | #define FSE_CAT(X,Y) X##Y 77 | #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) 78 | #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) 79 | 80 | 81 | /* Function templates */ 82 | FSE_DTable* FSE_createDTable (unsigned tableLog) 83 | { 84 | if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; 85 | return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); 86 | } 87 | 88 | void FSE_freeDTable (FSE_DTable* dt) 89 | { 90 | free(dt); 91 | } 92 | 93 | size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) 94 | { 95 | void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ 96 | FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); 97 | U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; 98 | 99 | U32 const maxSV1 = maxSymbolValue + 1; 100 | U32 const tableSize = 1 << tableLog; 101 | U32 highThreshold = tableSize-1; 102 | 103 | /* Sanity Checks */ 104 | if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); 105 | if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); 106 | 107 | /* Init, lay down lowprob symbols */ 108 | { FSE_DTableHeader DTableH; 109 | DTableH.tableLog = (U16)tableLog; 110 | DTableH.fastMode = 1; 111 | { S16 const largeLimit= (S16)(1 << (tableLog-1)); 112 | U32 s; 113 | for (s=0; s= largeLimit) DTableH.fastMode=0; 119 | symbolNext[s] = normalizedCounter[s]; 120 | } } } 121 | memcpy(dt, &DTableH, sizeof(DTableH)); 122 | } 123 | 124 | /* Spread symbols */ 125 | { U32 const tableMask = tableSize-1; 126 | U32 const step = FSE_TABLESTEP(tableSize); 127 | U32 s, position = 0; 128 | for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ 134 | } } 135 | if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ 136 | } 137 | 138 | /* Build Decoding table */ 139 | { U32 u; 140 | for (u=0; utableLog = 0; 164 | DTableH->fastMode = 0; 165 | 166 | cell->newState = 0; 167 | cell->symbol = symbolValue; 168 | cell->nbBits = 0; 169 | 170 | return 0; 171 | } 172 | 173 | 174 | size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) 175 | { 176 | void* ptr = dt; 177 | FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; 178 | void* dPtr = dt + 1; 179 | FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; 180 | const unsigned tableSize = 1 << nbBits; 181 | const unsigned tableMask = tableSize - 1; 182 | const unsigned maxSV1 = tableMask+1; 183 | unsigned s; 184 | 185 | /* Sanity checks */ 186 | if (nbBits < 1) return ERROR(GENERIC); /* min size */ 187 | 188 | /* Build Decoding Table */ 189 | DTableH->tableLog = (U16)nbBits; 190 | DTableH->fastMode = 1; 191 | for (s=0; s sizeof(bitD.bitContainer)*8) /* This test must be static */ 227 | BIT_reloadDStream(&bitD); 228 | 229 | op[1] = FSE_GETSYMBOL(&state2); 230 | 231 | if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ 232 | { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } 233 | 234 | op[2] = FSE_GETSYMBOL(&state1); 235 | 236 | if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ 237 | BIT_reloadDStream(&bitD); 238 | 239 | op[3] = FSE_GETSYMBOL(&state2); 240 | } 241 | 242 | /* tail */ 243 | /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ 244 | while (1) { 245 | if (op>(omax-2)) return ERROR(dstSize_tooSmall); 246 | *op++ = FSE_GETSYMBOL(&state1); 247 | if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { 248 | *op++ = FSE_GETSYMBOL(&state2); 249 | break; 250 | } 251 | 252 | if (op>(omax-2)) return ERROR(dstSize_tooSmall); 253 | *op++ = FSE_GETSYMBOL(&state2); 254 | if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { 255 | *op++ = FSE_GETSYMBOL(&state1); 256 | break; 257 | } } 258 | 259 | return op-ostart; 260 | } 261 | 262 | 263 | size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, 264 | const void* cSrc, size_t cSrcSize, 265 | const FSE_DTable* dt) 266 | { 267 | const void* ptr = dt; 268 | const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; 269 | const U32 fastMode = DTableH->fastMode; 270 | 271 | /* select fast mode (static) */ 272 | if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); 273 | return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); 274 | } 275 | 276 | 277 | size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog) 278 | { 279 | const BYTE* const istart = (const BYTE*)cSrc; 280 | const BYTE* ip = istart; 281 | short counting[FSE_MAX_SYMBOL_VALUE+1]; 282 | unsigned tableLog; 283 | unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; 284 | 285 | /* normal FSE decoding mode */ 286 | size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); 287 | if (FSE_isError(NCountLength)) return NCountLength; 288 | //if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */ 289 | if (tableLog > maxLog) return ERROR(tableLog_tooLarge); 290 | ip += NCountLength; 291 | cSrcSize -= NCountLength; 292 | 293 | CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) ); 294 | 295 | return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ 296 | } 297 | 298 | 299 | typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; 300 | 301 | size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) 302 | { 303 | DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ 304 | return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG); 305 | } 306 | 307 | 308 | 309 | #endif /* FSE_COMMONDEFS_ONLY */ 310 | -------------------------------------------------------------------------------- /zstd/mem.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef MEM_H_MODULE 12 | #define MEM_H_MODULE 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | /*-**************************************** 19 | * Dependencies 20 | ******************************************/ 21 | #include /* size_t, ptrdiff_t */ 22 | #include /* memcpy */ 23 | 24 | 25 | /*-**************************************** 26 | * Compiler specifics 27 | ******************************************/ 28 | #if defined(_MSC_VER) /* Visual Studio */ 29 | # include /* _byteswap_ulong */ 30 | # include /* _byteswap_* */ 31 | #endif 32 | #if defined(__GNUC__) 33 | # define MEM_STATIC static __inline __attribute__((unused)) 34 | #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 35 | # define MEM_STATIC static inline 36 | #elif defined(_MSC_VER) 37 | # define MEM_STATIC static __inline 38 | #else 39 | # define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ 40 | #endif 41 | 42 | /* code only tested on 32 and 64 bits systems */ 43 | #define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; } 44 | MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } 45 | 46 | 47 | /*-************************************************************** 48 | * Basic Types 49 | *****************************************************************/ 50 | #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) 51 | # include 52 | typedef uint8_t BYTE; 53 | typedef uint16_t U16; 54 | typedef int16_t S16; 55 | typedef uint32_t U32; 56 | typedef int32_t S32; 57 | typedef uint64_t U64; 58 | typedef int64_t S64; 59 | #else 60 | typedef unsigned char BYTE; 61 | typedef unsigned short U16; 62 | typedef signed short S16; 63 | typedef unsigned int U32; 64 | typedef signed int S32; 65 | typedef unsigned long long U64; 66 | typedef signed long long S64; 67 | #endif 68 | 69 | 70 | /*-************************************************************** 71 | * Memory I/O 72 | *****************************************************************/ 73 | /* MEM_FORCE_MEMORY_ACCESS : 74 | * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. 75 | * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. 76 | * The below switch allow to select different access method for improved performance. 77 | * Method 0 (default) : use `memcpy()`. Safe and portable. 78 | * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). 79 | * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. 80 | * Method 2 : direct access. This method is portable but violate C standard. 81 | * It can generate buggy code on targets depending on alignment. 82 | * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) 83 | * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. 84 | * Prefer these methods in priority order (0 > 1 > 2) 85 | */ 86 | #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ 87 | 88 | // This is otherwise extremely slow on desktop PCs -catid 89 | # if defined(__i386__) || defined(i386) || defined(intel) || defined(_M_IX86) || \ 90 | defined(__ia64) || defined(__ia64__) || defined(__x86_64) || defined(_M_IA64) || \ 91 | defined(_M_X64) || defined(_M_I86) || defined(sun386) || defined(__OS2__) 92 | # define MEM_FORCE_MEMORY_ACCESS 2 93 | # elif defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) 94 | # define MEM_FORCE_MEMORY_ACCESS 2 95 | # elif defined(__INTEL_COMPILER) || defined(__GNUC__) 96 | # define MEM_FORCE_MEMORY_ACCESS 1 97 | # endif 98 | #endif 99 | 100 | MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } 101 | MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } 102 | 103 | MEM_STATIC unsigned MEM_isLittleEndian(void) 104 | { 105 | const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ 106 | return one.c[0]; 107 | } 108 | 109 | #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) 110 | 111 | /* violates C standard, by lying on structure alignment. 112 | Only use if no other choice to achieve best performance on target platform */ 113 | MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } 114 | MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } 115 | MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } 116 | MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } 117 | 118 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } 119 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } 120 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } 121 | 122 | #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) 123 | 124 | /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ 125 | /* currently only defined for gcc and icc */ 126 | #if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) 127 | __pragma( pack(push, 1) ) 128 | typedef struct { U16 v; } unalign16; 129 | typedef struct { U32 v; } unalign32; 130 | typedef struct { U64 v; } unalign64; 131 | typedef struct { size_t v; } unalignArch; 132 | __pragma( pack(pop) ) 133 | #else 134 | typedef struct { U16 v; } __attribute__((packed)) unalign16; 135 | typedef struct { U32 v; } __attribute__((packed)) unalign32; 136 | typedef struct { U64 v; } __attribute__((packed)) unalign64; 137 | typedef struct { size_t v; } __attribute__((packed)) unalignArch; 138 | #endif 139 | 140 | MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; } 141 | MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; } 142 | MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; } 143 | MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; } 144 | 145 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; } 146 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; } 147 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; } 148 | 149 | #else 150 | 151 | /* default method, safe and standard. 152 | can sometimes prove slower */ 153 | 154 | MEM_STATIC U16 MEM_read16(const void* memPtr) 155 | { 156 | U16 val; memcpy(&val, memPtr, sizeof(val)); return val; 157 | } 158 | 159 | MEM_STATIC U32 MEM_read32(const void* memPtr) 160 | { 161 | U32 val; memcpy(&val, memPtr, sizeof(val)); return val; 162 | } 163 | 164 | MEM_STATIC U64 MEM_read64(const void* memPtr) 165 | { 166 | U64 val; memcpy(&val, memPtr, sizeof(val)); return val; 167 | } 168 | 169 | MEM_STATIC size_t MEM_readST(const void* memPtr) 170 | { 171 | size_t val; memcpy(&val, memPtr, sizeof(val)); return val; 172 | } 173 | 174 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) 175 | { 176 | memcpy(memPtr, &value, sizeof(value)); 177 | } 178 | 179 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) 180 | { 181 | memcpy(memPtr, &value, sizeof(value)); 182 | } 183 | 184 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) 185 | { 186 | memcpy(memPtr, &value, sizeof(value)); 187 | } 188 | 189 | #endif /* MEM_FORCE_MEMORY_ACCESS */ 190 | 191 | MEM_STATIC U32 MEM_swap32(U32 in) 192 | { 193 | #if defined(_MSC_VER) /* Visual Studio */ 194 | return _byteswap_ulong(in); 195 | #elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) 196 | return __builtin_bswap32(in); 197 | #else 198 | return ((in << 24) & 0xff000000 ) | 199 | ((in << 8) & 0x00ff0000 ) | 200 | ((in >> 8) & 0x0000ff00 ) | 201 | ((in >> 24) & 0x000000ff ); 202 | #endif 203 | } 204 | 205 | MEM_STATIC U64 MEM_swap64(U64 in) 206 | { 207 | #if defined(_MSC_VER) /* Visual Studio */ 208 | return _byteswap_uint64(in); 209 | #elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403) 210 | return __builtin_bswap64(in); 211 | #else 212 | return ((in << 56) & 0xff00000000000000ULL) | 213 | ((in << 40) & 0x00ff000000000000ULL) | 214 | ((in << 24) & 0x0000ff0000000000ULL) | 215 | ((in << 8) & 0x000000ff00000000ULL) | 216 | ((in >> 8) & 0x00000000ff000000ULL) | 217 | ((in >> 24) & 0x0000000000ff0000ULL) | 218 | ((in >> 40) & 0x000000000000ff00ULL) | 219 | ((in >> 56) & 0x00000000000000ffULL); 220 | #endif 221 | } 222 | 223 | MEM_STATIC size_t MEM_swapST(size_t in) 224 | { 225 | if (MEM_32bits()) 226 | return (size_t)MEM_swap32((U32)in); 227 | else 228 | return (size_t)MEM_swap64((U64)in); 229 | } 230 | 231 | /*=== Little endian r/w ===*/ 232 | 233 | MEM_STATIC U16 MEM_readLE16(const void* memPtr) 234 | { 235 | if (MEM_isLittleEndian()) 236 | return MEM_read16(memPtr); 237 | else { 238 | const BYTE* p = (const BYTE*)memPtr; 239 | return (U16)(p[0] + (p[1]<<8)); 240 | } 241 | } 242 | 243 | MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) 244 | { 245 | if (MEM_isLittleEndian()) { 246 | MEM_write16(memPtr, val); 247 | } else { 248 | BYTE* p = (BYTE*)memPtr; 249 | p[0] = (BYTE)val; 250 | p[1] = (BYTE)(val>>8); 251 | } 252 | } 253 | 254 | MEM_STATIC U32 MEM_readLE24(const void* memPtr) 255 | { 256 | return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); 257 | } 258 | 259 | MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) 260 | { 261 | MEM_writeLE16(memPtr, (U16)val); 262 | ((BYTE*)memPtr)[2] = (BYTE)(val>>16); 263 | } 264 | 265 | MEM_STATIC U32 MEM_readLE32(const void* memPtr) 266 | { 267 | if (MEM_isLittleEndian()) 268 | return MEM_read32(memPtr); 269 | else 270 | return MEM_swap32(MEM_read32(memPtr)); 271 | } 272 | 273 | MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) 274 | { 275 | if (MEM_isLittleEndian()) 276 | MEM_write32(memPtr, val32); 277 | else 278 | MEM_write32(memPtr, MEM_swap32(val32)); 279 | } 280 | 281 | MEM_STATIC U64 MEM_readLE64(const void* memPtr) 282 | { 283 | if (MEM_isLittleEndian()) 284 | return MEM_read64(memPtr); 285 | else 286 | return MEM_swap64(MEM_read64(memPtr)); 287 | } 288 | 289 | MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) 290 | { 291 | if (MEM_isLittleEndian()) 292 | MEM_write64(memPtr, val64); 293 | else 294 | MEM_write64(memPtr, MEM_swap64(val64)); 295 | } 296 | 297 | MEM_STATIC size_t MEM_readLEST(const void* memPtr) 298 | { 299 | if (MEM_32bits()) 300 | return (size_t)MEM_readLE32(memPtr); 301 | else 302 | return (size_t)MEM_readLE64(memPtr); 303 | } 304 | 305 | MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) 306 | { 307 | if (MEM_32bits()) 308 | MEM_writeLE32(memPtr, (U32)val); 309 | else 310 | MEM_writeLE64(memPtr, (U64)val); 311 | } 312 | 313 | /*=== Big endian r/w ===*/ 314 | 315 | MEM_STATIC U32 MEM_readBE32(const void* memPtr) 316 | { 317 | if (MEM_isLittleEndian()) 318 | return MEM_swap32(MEM_read32(memPtr)); 319 | else 320 | return MEM_read32(memPtr); 321 | } 322 | 323 | MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) 324 | { 325 | if (MEM_isLittleEndian()) 326 | MEM_write32(memPtr, MEM_swap32(val32)); 327 | else 328 | MEM_write32(memPtr, val32); 329 | } 330 | 331 | MEM_STATIC U64 MEM_readBE64(const void* memPtr) 332 | { 333 | if (MEM_isLittleEndian()) 334 | return MEM_swap64(MEM_read64(memPtr)); 335 | else 336 | return MEM_read64(memPtr); 337 | } 338 | 339 | MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) 340 | { 341 | if (MEM_isLittleEndian()) 342 | MEM_write64(memPtr, MEM_swap64(val64)); 343 | else 344 | MEM_write64(memPtr, val64); 345 | } 346 | 347 | MEM_STATIC size_t MEM_readBEST(const void* memPtr) 348 | { 349 | if (MEM_32bits()) 350 | return (size_t)MEM_readBE32(memPtr); 351 | else 352 | return (size_t)MEM_readBE64(memPtr); 353 | } 354 | 355 | MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) 356 | { 357 | if (MEM_32bits()) 358 | MEM_writeBE32(memPtr, (U32)val); 359 | else 360 | MEM_writeBE64(memPtr, (U64)val); 361 | } 362 | 363 | 364 | #if defined (__cplusplus) 365 | } 366 | #endif 367 | 368 | #endif /* MEM_H_MODULE */ 369 | -------------------------------------------------------------------------------- /zstd/xxhash.h: -------------------------------------------------------------------------------- 1 | /* 2 | xxHash - Extremely Fast Hash algorithm 3 | Header File 4 | Copyright (C) 2012-2016, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - xxHash source repository : https://github.com/Cyan4973/xxHash 33 | */ 34 | 35 | /* Notice extracted from xxHash homepage : 36 | 37 | xxHash is an extremely fast Hash algorithm, running at RAM speed limits. 38 | It also successfully passes all tests from the SMHasher suite. 39 | 40 | Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) 41 | 42 | Name Speed Q.Score Author 43 | xxHash 5.4 GB/s 10 44 | CrapWow 3.2 GB/s 2 Andrew 45 | MumurHash 3a 2.7 GB/s 10 Austin Appleby 46 | SpookyHash 2.0 GB/s 10 Bob Jenkins 47 | SBox 1.4 GB/s 9 Bret Mulvey 48 | Lookup3 1.2 GB/s 9 Bob Jenkins 49 | SuperFastHash 1.2 GB/s 1 Paul Hsieh 50 | CityHash64 1.05 GB/s 10 Pike & Alakuijala 51 | FNV 0.55 GB/s 5 Fowler, Noll, Vo 52 | CRC32 0.43 GB/s 9 53 | MD5-32 0.33 GB/s 10 Ronald L. Rivest 54 | SHA1-32 0.28 GB/s 10 55 | 56 | Q.Score is a measure of quality of the hash function. 57 | It depends on successfully passing SMHasher test set. 58 | 10 is a perfect score. 59 | 60 | A 64-bits version, named XXH64, is available since r35. 61 | It offers much better speed, but for 64-bits applications only. 62 | Name Speed on 64 bits Speed on 32 bits 63 | XXH64 13.8 GB/s 1.9 GB/s 64 | XXH32 6.8 GB/s 6.0 GB/s 65 | */ 66 | 67 | #if defined (__cplusplus) 68 | extern "C" { 69 | #endif 70 | 71 | #ifndef XXHASH_H_5627135585666179 72 | #define XXHASH_H_5627135585666179 1 73 | 74 | 75 | /* **************************** 76 | * Definitions 77 | ******************************/ 78 | #include /* size_t */ 79 | typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; 80 | 81 | 82 | /* **************************** 83 | * API modifier 84 | ******************************/ 85 | /** XXH_PRIVATE_API 86 | * This is useful if you want to include xxhash functions in `static` mode 87 | * in order to inline them, and remove their symbol from the public list. 88 | * Methodology : 89 | * #define XXH_PRIVATE_API 90 | * #include "xxhash.h" 91 | * `xxhash.c` is automatically included. 92 | * It's not useful to compile and link it as a separate module anymore. 93 | */ 94 | #ifdef XXH_PRIVATE_API 95 | # ifndef XXH_STATIC_LINKING_ONLY 96 | # define XXH_STATIC_LINKING_ONLY 97 | # endif 98 | # if defined(__GNUC__) 99 | # define XXH_PUBLIC_API static __inline __attribute__((unused)) 100 | # elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 101 | # define XXH_PUBLIC_API static inline 102 | # elif defined(_MSC_VER) 103 | # define XXH_PUBLIC_API static __inline 104 | # else 105 | # define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ 106 | # endif 107 | #else 108 | # define XXH_PUBLIC_API /* do nothing */ 109 | #endif /* XXH_PRIVATE_API */ 110 | 111 | /*!XXH_NAMESPACE, aka Namespace Emulation : 112 | 113 | If you want to include _and expose_ xxHash functions from within your own library, 114 | but also want to avoid symbol collisions with another library which also includes xxHash, 115 | 116 | you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library 117 | with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). 118 | 119 | Note that no change is required within the calling program as long as it includes `xxhash.h` : 120 | regular symbol name will be automatically translated by this header. 121 | */ 122 | #ifdef XXH_NAMESPACE 123 | # define XXH_CAT(A,B) A##B 124 | # define XXH_NAME2(A,B) XXH_CAT(A,B) 125 | # define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) 126 | # define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) 127 | # define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) 128 | # define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) 129 | # define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) 130 | # define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) 131 | # define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) 132 | # define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) 133 | # define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) 134 | # define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) 135 | # define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) 136 | # define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) 137 | # define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) 138 | # define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) 139 | # define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) 140 | # define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) 141 | # define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) 142 | # define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) 143 | # define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) 144 | #endif 145 | 146 | 147 | /* ************************************* 148 | * Version 149 | ***************************************/ 150 | #define XXH_VERSION_MAJOR 0 151 | #define XXH_VERSION_MINOR 6 152 | #define XXH_VERSION_RELEASE 2 153 | #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) 154 | XXH_PUBLIC_API unsigned XXH_versionNumber (void); 155 | 156 | 157 | /* **************************** 158 | * Simple Hash Functions 159 | ******************************/ 160 | typedef unsigned int XXH32_hash_t; 161 | typedef unsigned long long XXH64_hash_t; 162 | 163 | XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); 164 | XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); 165 | 166 | /*! 167 | XXH32() : 168 | Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". 169 | The memory between input & input+length must be valid (allocated and read-accessible). 170 | "seed" can be used to alter the result predictably. 171 | Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s 172 | XXH64() : 173 | Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". 174 | "seed" can be used to alter the result predictably. 175 | This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). 176 | */ 177 | 178 | 179 | /* **************************** 180 | * Streaming Hash Functions 181 | ******************************/ 182 | typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ 183 | typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ 184 | 185 | /*! State allocation, compatible with dynamic libraries */ 186 | 187 | XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); 188 | XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); 189 | 190 | XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); 191 | XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); 192 | 193 | 194 | /* hash streaming */ 195 | 196 | XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); 197 | XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); 198 | XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); 199 | 200 | XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); 201 | XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); 202 | XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); 203 | 204 | /* 205 | These functions generate the xxHash of an input provided in multiple segments. 206 | Note that, for small input, they are slower than single-call functions, due to state management. 207 | For small input, prefer `XXH32()` and `XXH64()` . 208 | 209 | XXH state must first be allocated, using XXH*_createState() . 210 | 211 | Start a new hash by initializing state with a seed, using XXH*_reset(). 212 | 213 | Then, feed the hash state by calling XXH*_update() as many times as necessary. 214 | Obviously, input must be allocated and read accessible. 215 | The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. 216 | 217 | Finally, a hash value can be produced anytime, by using XXH*_digest(). 218 | This function returns the nn-bits hash as an int or long long. 219 | 220 | It's still possible to continue inserting input into the hash state after a digest, 221 | and generate some new hashes later on, by calling again XXH*_digest(). 222 | 223 | When done, free XXH state space if it was allocated dynamically. 224 | */ 225 | 226 | 227 | /* ************************** 228 | * Utils 229 | ****************************/ 230 | #if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ 231 | # define restrict /* disable restrict */ 232 | #endif 233 | 234 | XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); 235 | XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); 236 | 237 | 238 | /* ************************** 239 | * Canonical representation 240 | ****************************/ 241 | /* Default result type for XXH functions are primitive unsigned 32 and 64 bits. 242 | * The canonical representation uses human-readable write convention, aka big-endian (large digits first). 243 | * These functions allow transformation of hash result into and from its canonical format. 244 | * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. 245 | */ 246 | typedef struct { unsigned char digest[4]; } XXH32_canonical_t; 247 | typedef struct { unsigned char digest[8]; } XXH64_canonical_t; 248 | 249 | XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); 250 | XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); 251 | 252 | XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); 253 | XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); 254 | 255 | #endif /* XXHASH_H_5627135585666179 */ 256 | 257 | 258 | 259 | /* ================================================================================================ 260 | This section contains definitions which are not guaranteed to remain stable. 261 | They may change in future versions, becoming incompatible with a different version of the library. 262 | They shall only be used with static linking. 263 | Never use these definitions in association with dynamic linking ! 264 | =================================================================================================== */ 265 | #if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345) 266 | #define XXH_STATIC_H_3543687687345 267 | 268 | /* These definitions are only meant to allow allocation of XXH state 269 | statically, on stack, or in a struct for example. 270 | Do not use members directly. */ 271 | 272 | struct XXH32_state_s { 273 | unsigned total_len_32; 274 | unsigned large_len; 275 | unsigned v1; 276 | unsigned v2; 277 | unsigned v3; 278 | unsigned v4; 279 | unsigned mem32[4]; /* buffer defined as U32 for alignment */ 280 | unsigned memsize; 281 | unsigned reserved; /* never read nor write, will be removed in a future version */ 282 | }; /* typedef'd to XXH32_state_t */ 283 | 284 | struct XXH64_state_s { 285 | unsigned long long total_len; 286 | unsigned long long v1; 287 | unsigned long long v2; 288 | unsigned long long v3; 289 | unsigned long long v4; 290 | unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ 291 | unsigned memsize; 292 | unsigned reserved[2]; /* never read nor write, will be removed in a future version */ 293 | }; /* typedef'd to XXH64_state_t */ 294 | 295 | 296 | # ifdef XXH_PRIVATE_API 297 | # include "xxhash.c" /* include xxhash functions as `static`, for inlining */ 298 | # endif 299 | 300 | #endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ 301 | 302 | 303 | #if defined (__cplusplus) 304 | } 305 | #endif 306 | --------------------------------------------------------------------------------