├── .gitignore ├── .rspec ├── Gemfile ├── ext └── zstd │ ├── libzstd │ ├── .gitignore │ ├── libzstd.pc.in │ ├── LICENSE │ ├── common │ │ ├── error_public.h │ │ ├── zstd_common.c │ │ ├── error_private.h │ │ ├── zstd_internal.h │ │ ├── entropy_common.c │ │ ├── zbuff.h │ │ ├── huf.h │ │ ├── mem.h │ │ ├── fse_decompress.c │ │ ├── xxhash.h │ │ └── bitstream.h │ ├── PATENTS │ ├── dictBuilder │ │ ├── divsufsort.h │ │ └── zdict.h │ ├── README.md │ ├── legacy │ │ ├── zstd_v02.h │ │ ├── zstd_v03.h │ │ ├── zstd_v01.h │ │ ├── zstd_v04.h │ │ ├── zstd_v05.h │ │ ├── zstd_v06.h │ │ ├── zstd_v07.h │ │ └── zstd_legacy.h │ ├── Makefile │ ├── decompress │ │ └── zbuff_decompress.c │ └── compress │ │ └── zbuff_compress.c │ ├── extconf.rb │ └── rubyzstd.c ├── lib ├── zstd.rb └── zstd │ └── version.rb ├── Rakefile ├── zstd.gemspec ├── Gemfile.lock └── spec ├── simple_api_spec.rb └── spec_helper.rb /.gitignore: -------------------------------------------------------------------------------- 1 | tmp/ 2 | *.so -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --require spec_helper 3 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/.gitignore: -------------------------------------------------------------------------------- 1 | # make install artefact 2 | libzstd.pc 3 | -------------------------------------------------------------------------------- /lib/zstd.rb: -------------------------------------------------------------------------------- 1 | require_relative 'zstd/version' 2 | require_relative 'zstd/zstd' 3 | 4 | module Zstd 5 | 6 | end 7 | -------------------------------------------------------------------------------- /lib/zstd/version.rb: -------------------------------------------------------------------------------- 1 | module Zstd 2 | LIBZSTD_VERSION = '1.0.0' 3 | GEM_REVISION = 1 4 | 5 | VERSION = "#{LIBZSTD_VERSION}.#{GEM_REVISION}" 6 | end 7 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "rake/extensiontask" 2 | 3 | if Gem.win_platform? 4 | task :devkit do 5 | begin 6 | require 'devkit' 7 | rescue LoadError 8 | warn 'Failed to load devkit, installation might fail' 9 | end 10 | end 11 | 12 | task :compile => [:devkit] 13 | end 14 | 15 | GEMSPEC = Gem::Specification.load('zstd.gemspec') 16 | 17 | Rake::ExtensionTask.new("zstd", GEMSPEC) do |task| 18 | task.lib_dir = "lib/zstd" 19 | end 20 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/libzstd.pc.in: -------------------------------------------------------------------------------- 1 | # ZSTD - standard compression algorithm 2 | # Copyright (C) 2014-2015, Yann Collet. 3 | # BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 4 | 5 | prefix=@PREFIX@ 6 | libdir=@LIBDIR@ 7 | includedir=@INCLUDEDIR@ 8 | 9 | Name: zstd 10 | Description: lossless compression algorithm library 11 | URL: https://github.com/Cyan4973/zstd 12 | Version: @VERSION@ 13 | Libs: -L@LIBDIR@ -lzstd 14 | Cflags: -I@INCLUDEDIR@ 15 | -------------------------------------------------------------------------------- /zstd.gemspec: -------------------------------------------------------------------------------- 1 | require File.expand_path('../lib/zstd/version', __FILE__) 2 | 3 | Gem::Specification.new do |s| 4 | s.name = 'zstd' 5 | s.version = Zstd::VERSION 6 | s.summary = '' 7 | s.author = 'Jarred Holman' 8 | s.licenses = ['MIT'] 9 | 10 | s.files = Dir['{lib,ext}/**/*.rb'] 11 | 12 | s.platform = Gem::Platform::RUBY 13 | s.extensions << 'ext/zstd/extconf.rb' 14 | 15 | s.require_paths = ['lib', 'ext'] 16 | 17 | s.add_development_dependency 'rspec' 18 | end -------------------------------------------------------------------------------- /ext/zstd/extconf.rb: -------------------------------------------------------------------------------- 1 | require 'mkmf' 2 | 3 | $CFLAGS = '-I. -O3 -std=c99 -pedantic' 4 | 5 | Dir.chdir File.expand_path('..', __FILE__) do 6 | $srcs = Dir['**/*.c'] 7 | 8 | Dir.glob('libzstd/*') do |path| 9 | if Dir.exist?(path) 10 | $VPATH << "$(srcdir)/#{path}" 11 | $INCFLAGS << " -I$(srcdir)/#{path}" 12 | end 13 | end 14 | 15 | end 16 | 17 | # add include path to the internal folder 18 | # $(srcdir) is a root folder, where "extconf.rb" is stored 19 | $INCFLAGS << " -I$(srcdir) -I$(srcdir)/libzstd" 20 | 21 | # add folder, where compiler can search source files 22 | $VPATH << "$(srcdir)" 23 | 24 | create_makefile 'zstd/zstd' 25 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | zstd (1.0.0.1) 5 | 6 | GEM 7 | remote: https://rubygems.org/ 8 | specs: 9 | diff-lcs (1.2.5) 10 | rspec (3.5.0) 11 | rspec-core (~> 3.5.0) 12 | rspec-expectations (~> 3.5.0) 13 | rspec-mocks (~> 3.5.0) 14 | rspec-core (3.5.2) 15 | rspec-support (~> 3.5.0) 16 | rspec-expectations (3.5.0) 17 | diff-lcs (>= 1.2.0, < 2.0) 18 | rspec-support (~> 3.5.0) 19 | rspec-mocks (3.5.0) 20 | diff-lcs (>= 1.2.0, < 2.0) 21 | rspec-support (~> 3.5.0) 22 | rspec-support (3.5.0) 23 | 24 | PLATFORMS 25 | x64-mingw32 26 | 27 | DEPENDENCIES 28 | rspec 29 | zstd! 30 | 31 | BUNDLED WITH 32 | 1.11.2 33 | -------------------------------------------------------------------------------- /spec/simple_api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'zstd' 3 | 4 | RSpec.describe Zstd do 5 | describe 'compress' do 6 | it 'should work' do 7 | compressed = Zstd.compress('abc') 8 | expect(compressed).to be_a(String) 9 | expect(compressed).to_not eq('abc') 10 | end 11 | 12 | it 'should support compression levels' do 13 | compressed = Zstd.compress('abc', 1) 14 | expect(compressed).to be_a(String) 15 | expect(compressed).to_not eq('abc') 16 | end 17 | end 18 | 19 | describe 'decompress' do 20 | it 'should work' do 21 | compressed = Zstd.compress('abc') 22 | decompressed = Zstd.decompress(compressed) 23 | expect(decompressed).to eq('abc') 24 | end 25 | 26 | it 'should work for empty strings' do 27 | compressed = Zstd.compress('') 28 | expect(compressed.bytesize).to eq(9) 29 | decompressed = Zstd.decompress(compressed) 30 | expect(decompressed).to eq('') 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/LICENSE: -------------------------------------------------------------------------------- 1 | BSD License 2 | 3 | For Zstandard software 4 | 5 | Copyright (c) 2016-present, Facebook, Inc. All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, 8 | are permitted provided that the following conditions are met: 9 | 10 | * Redistributions of source code must retain the above copyright notice, this 11 | list of conditions and the following disclaimer. 12 | 13 | * Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | 17 | * Neither the name Facebook nor the names of its contributors may be used to 18 | endorse or promote products derived from this software without specific 19 | prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 25 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 26 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 28 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/common/error_public.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef ERROR_PUBLIC_H_MODULE 11 | #define ERROR_PUBLIC_H_MODULE 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | /*===== dependency =====*/ 18 | #include /* size_t */ 19 | 20 | 21 | /*-**************************************** 22 | * error codes list 23 | ******************************************/ 24 | typedef enum { 25 | ZSTD_error_no_error, 26 | ZSTD_error_GENERIC, 27 | ZSTD_error_prefix_unknown, 28 | ZSTD_error_version_unsupported, 29 | ZSTD_error_parameter_unknown, 30 | ZSTD_error_frameParameter_unsupported, 31 | ZSTD_error_frameParameter_unsupportedBy32bits, 32 | ZSTD_error_compressionParameter_unsupported, 33 | ZSTD_error_init_missing, 34 | ZSTD_error_memory_allocation, 35 | ZSTD_error_stage_wrong, 36 | ZSTD_error_dstSize_tooSmall, 37 | ZSTD_error_srcSize_wrong, 38 | ZSTD_error_corruption_detected, 39 | ZSTD_error_checksum_wrong, 40 | ZSTD_error_tableLog_tooLarge, 41 | ZSTD_error_maxSymbolValue_tooLarge, 42 | ZSTD_error_maxSymbolValue_tooSmall, 43 | ZSTD_error_dictionary_corrupted, 44 | ZSTD_error_dictionary_wrong, 45 | ZSTD_error_maxCode 46 | } ZSTD_ErrorCode; 47 | 48 | /*! ZSTD_getErrorCode() : 49 | convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, 50 | which can be used to compare directly with enum list published into "error_public.h" */ 51 | ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); 52 | const char* ZSTD_getErrorString(ZSTD_ErrorCode code); 53 | 54 | 55 | #if defined (__cplusplus) 56 | } 57 | #endif 58 | 59 | #endif /* ERROR_PUBLIC_H_MODULE */ 60 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/PATENTS: -------------------------------------------------------------------------------- 1 | Additional Grant of Patent Rights Version 2 2 | 3 | "Software" means the Zstandard software distributed by Facebook, Inc. 4 | 5 | Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software 6 | ("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable 7 | (subject to the termination provision below) license under any Necessary 8 | Claims, to make, have made, use, sell, offer to sell, import, and otherwise 9 | transfer the Software. For avoidance of doubt, no license is granted under 10 | Facebook’s rights in any patent claims that are infringed by (i) modifications 11 | to the Software made by you or any third party or (ii) the Software in 12 | combination with any software or other technology. 13 | 14 | The license granted hereunder will terminate, automatically and without notice, 15 | if you (or any of your subsidiaries, corporate affiliates or agents) initiate 16 | directly or indirectly, or take a direct financial interest in, any Patent 17 | Assertion: (i) against Facebook or any of its subsidiaries or corporate 18 | affiliates, (ii) against any party if such Patent Assertion arises in whole or 19 | in part from any software, technology, product or service of Facebook or any of 20 | its subsidiaries or corporate affiliates, or (iii) against any party relating 21 | to the Software. Notwithstanding the foregoing, if Facebook or any of its 22 | subsidiaries or corporate affiliates files a lawsuit alleging patent 23 | infringement against you in the first instance, and you respond by filing a 24 | patent infringement counterclaim in that lawsuit against that party that is 25 | unrelated to the Software, the license granted hereunder will not terminate 26 | under section (i) of this paragraph due to such counterclaim. 27 | 28 | A "Necessary Claim" is a claim of a patent owned by Facebook that is 29 | necessarily infringed by the Software standing alone. 30 | 31 | A "Patent Assertion" is any lawsuit or other action alleging direct, indirect, 32 | or contributory infringement or inducement to infringe any patent, including a 33 | cross-claim or counterclaim. 34 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/dictBuilder/divsufsort.h: -------------------------------------------------------------------------------- 1 | /* 2 | * divsufsort.h for libdivsufsort-lite 3 | * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. 4 | * 5 | * Permission is hereby granted, free of charge, to any person 6 | * obtaining a copy of this software and associated documentation 7 | * files (the "Software"), to deal in the Software without 8 | * restriction, including without limitation the rights to use, 9 | * copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the 11 | * Software is furnished to do so, subject to the following 12 | * conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be 15 | * included in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 19 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 21 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | * OTHER DEALINGS IN THE SOFTWARE. 25 | */ 26 | 27 | #ifndef _DIVSUFSORT_H 28 | #define _DIVSUFSORT_H 1 29 | 30 | #ifdef __cplusplus 31 | extern "C" { 32 | #endif /* __cplusplus */ 33 | 34 | 35 | /*- Prototypes -*/ 36 | 37 | /** 38 | * Constructs the suffix array of a given string. 39 | * @param T [0..n-1] The input string. 40 | * @param SA [0..n-1] The output array of suffixes. 41 | * @param n The length of the given string. 42 | * @param openMP enables OpenMP optimization. 43 | * @return 0 if no error occurred, -1 or -2 otherwise. 44 | */ 45 | int 46 | divsufsort(const unsigned char *T, int *SA, int n, int openMP); 47 | 48 | /** 49 | * Constructs the burrows-wheeler transformed string of a given string. 50 | * @param T [0..n-1] The input string. 51 | * @param U [0..n-1] The output string. (can be T) 52 | * @param A [0..n-1] The temporary array. (can be NULL) 53 | * @param n The length of the given string. 54 | * @param num_indexes The length of secondary indexes array. (can be NULL) 55 | * @param indexes The secondary indexes array. (can be NULL) 56 | * @param openMP enables OpenMP optimization. 57 | * @return The primary index if no error occurred, -1 or -2 otherwise. 58 | */ 59 | int 60 | divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP); 61 | 62 | 63 | #ifdef __cplusplus 64 | } /* extern "C" */ 65 | #endif /* __cplusplus */ 66 | 67 | #endif /* _DIVSUFSORT_H */ 68 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/README.md: -------------------------------------------------------------------------------- 1 | Zstandard library files 2 | ================================ 3 | 4 | The __lib__ directory contains several directories. 5 | Depending on target use case, it's enough to include only files from relevant directories. 6 | 7 | 8 | #### API 9 | 10 | Zstandard's stable API is exposed within [zstd.h](zstd.h), 11 | at the root of `lib` directory. 12 | 13 | 14 | #### Advanced API 15 | 16 | Some additional API may be useful if you're looking into advanced features : 17 | - common/error_public.h : transforms `size_t` function results into an `enum`, 18 | for precise error handling. 19 | - ZSTD_STATIC_LINKING_ONLY : if you define this macro _before_ including `zstd.h`, 20 | it will give access to advanced and experimental API. 21 | These APIs shall ___never be used with dynamic library___ ! 22 | They are not "stable", their definition may change in the future. 23 | Only static linking is allowed. 24 | 25 | 26 | #### Modular build 27 | 28 | Directory `common/` is required in all circumstances. 29 | You can select to support compression only, by just adding files from the `compress/` directory, 30 | In a similar way, you can build a decompressor-only library with the `decompress/` directory. 31 | 32 | Other optional functionalities provided are : 33 | 34 | - `dictBuilder/` : source files to create dictionaries. 35 | The API can be consulted in `dictBuilder/zdict.h`. 36 | This module also depends on `common/` and `compress/` . 37 | 38 | - `legacy/` : source code to decompress previous versions of zstd, starting from `v0.1`. 39 | This module also depends on `common/` and `decompress/` . 40 | Library compilation must include directive `ZSTD_LEGACY_SUPPORT = 1` . 41 | The main API can be consulted in `legacy/zstd_legacy.h`. 42 | Advanced API from each version can be found in their relevant header file. 43 | For example, advanced API for version `v0.4` is in `legacy/zstd_v04.h` . 44 | 45 | 46 | #### Obsolete streaming API 47 | 48 | Streaming is now provided within `zstd.h`. 49 | Older streaming API is still provided within `common/zbuff.h`. 50 | It is considered obsolete, and will be removed in a future version. 51 | Consider migrating towards newer streaming API. 52 | 53 | 54 | #### Miscellaneous 55 | 56 | The other files are not source code. There are : 57 | 58 | - LICENSE : contains the BSD license text 59 | - Makefile : script to compile or install zstd library (static and dynamic) 60 | - libzstd.pc.in : for pkg-config (`make install`) 61 | - README.md : this file 62 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/common/zstd_common.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | 11 | 12 | /*-************************************* 13 | * Dependencies 14 | ***************************************/ 15 | #include /* malloc */ 16 | #include "error_private.h" 17 | #define ZSTD_STATIC_LINKING_ONLY 18 | #include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */ 19 | #include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */ 20 | 21 | 22 | /*-**************************************** 23 | * Version 24 | ******************************************/ 25 | unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } 26 | 27 | 28 | /*-**************************************** 29 | * ZSTD Error Management 30 | ******************************************/ 31 | /*! ZSTD_isError() : 32 | * tells if a return value is an error code */ 33 | unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } 34 | 35 | /*! ZSTD_getErrorName() : 36 | * provides error code string from function result (useful for debugging) */ 37 | const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } 38 | 39 | /*! ZSTD_getError() : 40 | * convert a `size_t` function result into a proper ZSTD_errorCode enum */ 41 | ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } 42 | 43 | /*! ZSTD_getErrorString() : 44 | * provides error code string from enum */ 45 | const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); } 46 | 47 | 48 | /* ************************************************************** 49 | * ZBUFF Error Management 50 | ****************************************************************/ 51 | unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } 52 | 53 | const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } 54 | 55 | 56 | 57 | /*=************************************************************** 58 | * Custom allocator 59 | ****************************************************************/ 60 | /* default uses stdlib */ 61 | void* ZSTD_defaultAllocFunction(void* opaque, size_t size) 62 | { 63 | void* address = malloc(size); 64 | (void)opaque; 65 | return address; 66 | } 67 | 68 | void ZSTD_defaultFreeFunction(void* opaque, void* address) 69 | { 70 | (void)opaque; 71 | free(address); 72 | } 73 | 74 | void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) 75 | { 76 | return customMem.customAlloc(customMem.opaque, size); 77 | } 78 | 79 | void ZSTD_free(void* ptr, ZSTD_customMem customMem) 80 | { 81 | if (ptr!=NULL) 82 | customMem.customFree(customMem.opaque, ptr); 83 | } 84 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/legacy/zstd_v02.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef ZSTD_V02_H_4174539423 11 | #define ZSTD_V02_H_4174539423 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | /* ************************************* 18 | * Includes 19 | ***************************************/ 20 | #include /* size_t */ 21 | 22 | 23 | /* ************************************* 24 | * Simple one-step function 25 | ***************************************/ 26 | /** 27 | ZSTDv02_decompress() : decompress ZSTD frames compliant with v0.2.x format 28 | compressedSize : is the exact source size 29 | maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. 30 | It must be equal or larger than originalSize, otherwise decompression will fail. 31 | return : the number of bytes decompressed into destination buffer (originalSize) 32 | or an errorCode if it fails (which can be tested using ZSTDv01_isError()) 33 | */ 34 | size_t ZSTDv02_decompress( void* dst, size_t maxOriginalSize, 35 | const void* src, size_t compressedSize); 36 | 37 | /** 38 | ZSTDv02_isError() : tells if the result of ZSTDv02_decompress() is an error 39 | */ 40 | unsigned ZSTDv02_isError(size_t code); 41 | 42 | 43 | /* ************************************* 44 | * Advanced functions 45 | ***************************************/ 46 | typedef struct ZSTDv02_Dctx_s ZSTDv02_Dctx; 47 | ZSTDv02_Dctx* ZSTDv02_createDCtx(void); 48 | size_t ZSTDv02_freeDCtx(ZSTDv02_Dctx* dctx); 49 | 50 | size_t ZSTDv02_decompressDCtx(void* ctx, 51 | void* dst, size_t maxOriginalSize, 52 | const void* src, size_t compressedSize); 53 | 54 | /* ************************************* 55 | * Streaming functions 56 | ***************************************/ 57 | size_t ZSTDv02_resetDCtx(ZSTDv02_Dctx* dctx); 58 | 59 | size_t ZSTDv02_nextSrcSizeToDecompress(ZSTDv02_Dctx* dctx); 60 | size_t ZSTDv02_decompressContinue(ZSTDv02_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); 61 | /** 62 | Use above functions alternatively. 63 | ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). 64 | ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. 65 | Result is the number of bytes regenerated within 'dst'. 66 | It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. 67 | */ 68 | 69 | /* ************************************* 70 | * Prefix - version detection 71 | ***************************************/ 72 | #define ZSTDv02_magicNumber 0xFD2FB522 /* v0.2 */ 73 | 74 | 75 | #if defined (__cplusplus) 76 | } 77 | #endif 78 | 79 | #endif /* ZSTD_V02_H_4174539423 */ 80 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/legacy/zstd_v03.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef ZSTD_V03_H_298734209782 11 | #define ZSTD_V03_H_298734209782 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | /* ************************************* 18 | * Includes 19 | ***************************************/ 20 | #include /* size_t */ 21 | 22 | 23 | /* ************************************* 24 | * Simple one-step function 25 | ***************************************/ 26 | /** 27 | ZSTDv03_decompress() : decompress ZSTD frames compliant with v0.3.x format 28 | compressedSize : is the exact source size 29 | maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. 30 | It must be equal or larger than originalSize, otherwise decompression will fail. 31 | return : the number of bytes decompressed into destination buffer (originalSize) 32 | or an errorCode if it fails (which can be tested using ZSTDv01_isError()) 33 | */ 34 | size_t ZSTDv03_decompress( void* dst, size_t maxOriginalSize, 35 | const void* src, size_t compressedSize); 36 | 37 | /** 38 | ZSTDv03_isError() : tells if the result of ZSTDv03_decompress() is an error 39 | */ 40 | unsigned ZSTDv03_isError(size_t code); 41 | 42 | 43 | /* ************************************* 44 | * Advanced functions 45 | ***************************************/ 46 | typedef struct ZSTDv03_Dctx_s ZSTDv03_Dctx; 47 | ZSTDv03_Dctx* ZSTDv03_createDCtx(void); 48 | size_t ZSTDv03_freeDCtx(ZSTDv03_Dctx* dctx); 49 | 50 | size_t ZSTDv03_decompressDCtx(void* ctx, 51 | void* dst, size_t maxOriginalSize, 52 | const void* src, size_t compressedSize); 53 | 54 | /* ************************************* 55 | * Streaming functions 56 | ***************************************/ 57 | size_t ZSTDv03_resetDCtx(ZSTDv03_Dctx* dctx); 58 | 59 | size_t ZSTDv03_nextSrcSizeToDecompress(ZSTDv03_Dctx* dctx); 60 | size_t ZSTDv03_decompressContinue(ZSTDv03_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); 61 | /** 62 | Use above functions alternatively. 63 | ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). 64 | ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. 65 | Result is the number of bytes regenerated within 'dst'. 66 | It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. 67 | */ 68 | 69 | /* ************************************* 70 | * Prefix - version detection 71 | ***************************************/ 72 | #define ZSTDv03_magicNumber 0xFD2FB523 /* v0.3 */ 73 | 74 | 75 | #if defined (__cplusplus) 76 | } 77 | #endif 78 | 79 | #endif /* ZSTD_V03_H_298734209782 */ 80 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/legacy/zstd_v01.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef ZSTD_V01_H_28739879432 11 | #define ZSTD_V01_H_28739879432 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | /* ************************************* 18 | * Includes 19 | ***************************************/ 20 | #include /* size_t */ 21 | 22 | 23 | /* ************************************* 24 | * Simple one-step function 25 | ***************************************/ 26 | /** 27 | ZSTDv01_decompress() : decompress ZSTD frames compliant with v0.1.x format 28 | compressedSize : is the exact source size 29 | maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. 30 | It must be equal or larger than originalSize, otherwise decompression will fail. 31 | return : the number of bytes decompressed into destination buffer (originalSize) 32 | or an errorCode if it fails (which can be tested using ZSTDv01_isError()) 33 | */ 34 | size_t ZSTDv01_decompress( void* dst, size_t maxOriginalSize, 35 | const void* src, size_t compressedSize); 36 | 37 | /** 38 | ZSTDv01_isError() : tells if the result of ZSTDv01_decompress() is an error 39 | */ 40 | unsigned ZSTDv01_isError(size_t code); 41 | 42 | 43 | /* ************************************* 44 | * Advanced functions 45 | ***************************************/ 46 | typedef struct ZSTDv01_Dctx_s ZSTDv01_Dctx; 47 | ZSTDv01_Dctx* ZSTDv01_createDCtx(void); 48 | size_t ZSTDv01_freeDCtx(ZSTDv01_Dctx* dctx); 49 | 50 | size_t ZSTDv01_decompressDCtx(void* ctx, 51 | void* dst, size_t maxOriginalSize, 52 | const void* src, size_t compressedSize); 53 | 54 | /* ************************************* 55 | * Streaming functions 56 | ***************************************/ 57 | size_t ZSTDv01_resetDCtx(ZSTDv01_Dctx* dctx); 58 | 59 | size_t ZSTDv01_nextSrcSizeToDecompress(ZSTDv01_Dctx* dctx); 60 | size_t ZSTDv01_decompressContinue(ZSTDv01_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); 61 | /** 62 | Use above functions alternatively. 63 | ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). 64 | ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. 65 | Result is the number of bytes regenerated within 'dst'. 66 | It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. 67 | */ 68 | 69 | /* ************************************* 70 | * Prefix - version detection 71 | ***************************************/ 72 | #define ZSTDv01_magicNumber 0xFD2FB51E /* Big Endian version */ 73 | #define ZSTDv01_magicNumberLE 0x1EB52FFD /* Little Endian version */ 74 | 75 | 76 | #if defined (__cplusplus) 77 | } 78 | #endif 79 | 80 | #endif /* ZSTD_V01_H_28739879432 */ 81 | -------------------------------------------------------------------------------- /ext/zstd/rubyzstd.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | 5 | #include 6 | 7 | 8 | static VALUE compress(int argc, VALUE *argv, VALUE self) 9 | { 10 | VALUE input_value; 11 | VALUE compression_level_value; 12 | rb_scan_args(argc, argv, "11", &input_value, &compression_level_value); 13 | 14 | Check_Type(input_value, RUBY_T_STRING); 15 | const char* input_data = RSTRING_PTR(input_value); 16 | size_t input_size = RSTRING_LEN(input_value); 17 | 18 | int compression_level; 19 | if (NIL_P(compression_level_value)) { 20 | compression_level = 1; 21 | } else { 22 | Check_Type(compression_level_value, RUBY_T_FIXNUM); 23 | compression_level = FIX2INT(compression_level_value); 24 | } 25 | 26 | // do compress 27 | size_t max_compressed_size = ZSTD_compressBound(input_size); 28 | 29 | VALUE output = rb_str_new(NULL, max_compressed_size); 30 | char* output_data = RSTRING_PTR(output); 31 | 32 | size_t compressed_size = ZSTD_compress((void*)output_data, max_compressed_size, 33 | (const void*)input_data, input_size, 1); 34 | 35 | if (ZSTD_isError(compressed_size)) { 36 | rb_raise(rb_eRuntimeError, "%s: %s", "compress failed", ZSTD_getErrorName(compressed_size)); 37 | } else { 38 | rb_str_resize(output, compressed_size); 39 | } 40 | 41 | return output; 42 | } 43 | 44 | static VALUE decompress_buffered(const char* input_data, size_t input_size) 45 | { 46 | const size_t outputBufferSize = 4096; 47 | 48 | ZSTD_DStream* const dstream = ZSTD_createDStream(); 49 | if (dstream == NULL) { 50 | rb_raise(rb_eRuntimeError, "%s", "ZSTD_createDStream failed"); 51 | } 52 | 53 | size_t initResult = ZSTD_initDStream(dstream); 54 | if (ZSTD_isError(initResult)) { 55 | ZSTD_freeDStream(dstream); 56 | rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_initDStream failed", ZSTD_getErrorName(initResult)); 57 | } 58 | 59 | 60 | VALUE output_string = rb_str_new(NULL, 0); 61 | ZSTD_outBuffer output = { NULL, 0, 0 }; 62 | 63 | ZSTD_inBuffer input = { input_data, input_size, 0 }; 64 | while (input.pos < input.size) { 65 | output.size += outputBufferSize; 66 | rb_str_resize(output_string, output.size); 67 | output.dst = RSTRING_PTR(output_string); 68 | 69 | size_t readHint = ZSTD_decompressStream(dstream, &output, &input); 70 | if (ZSTD_isError(readHint)) { 71 | ZSTD_freeDStream(dstream); 72 | rb_raise(rb_eRuntimeError, "%s: %s", "ZSTD_decompressStream failed", ZSTD_getErrorName(readHint)); 73 | } 74 | } 75 | 76 | ZSTD_freeDStream(dstream); 77 | rb_str_resize(output_string, output.pos); 78 | return output_string; 79 | } 80 | 81 | static VALUE decompress(VALUE self, VALUE input) 82 | { 83 | Check_Type(input, T_STRING); 84 | const char* input_data = RSTRING_PTR(input); 85 | size_t input_size = RSTRING_LEN(input); 86 | 87 | uint64_t uncompressed_size = ZSTD_getDecompressedSize(input_data, input_size); 88 | 89 | if (uncompressed_size == 0) { 90 | return decompress_buffered(input_data, input_size); 91 | } 92 | 93 | VALUE output = rb_str_new(NULL, uncompressed_size); 94 | char* output_data = RSTRING_PTR(output); 95 | 96 | size_t decompress_size = ZSTD_decompress((void*)output_data, uncompressed_size, 97 | (const void*)input_data, input_size); 98 | 99 | if (ZSTD_isError(decompress_size)) { 100 | rb_raise(rb_eRuntimeError, "%s: %s", "decompress error", ZSTD_getErrorName(decompress_size)); 101 | } 102 | 103 | return output; 104 | } 105 | 106 | void Init_zstd() 107 | { 108 | VALUE mZstd = rb_const_get(rb_cObject, rb_intern("Zstd")); 109 | 110 | rb_define_module_function(mZstd, "compress", compress, -1); 111 | rb_define_module_function(mZstd, "decompress", decompress, 1); 112 | } 113 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/common/error_private.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | /* Note : this module is expected to remain private, do not expose it */ 11 | 12 | #ifndef ERROR_H_MODULE 13 | #define ERROR_H_MODULE 14 | 15 | #if defined (__cplusplus) 16 | extern "C" { 17 | #endif 18 | 19 | 20 | /* **************************************** 21 | * Dependencies 22 | ******************************************/ 23 | #include /* size_t */ 24 | #include "error_public.h" /* enum list */ 25 | 26 | 27 | /* **************************************** 28 | * Compiler-specific 29 | ******************************************/ 30 | #if defined(__GNUC__) 31 | # define ERR_STATIC static __attribute__((unused)) 32 | #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 33 | # define ERR_STATIC static inline 34 | #elif defined(_MSC_VER) 35 | # define ERR_STATIC static __inline 36 | #else 37 | # define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ 38 | #endif 39 | 40 | 41 | /*-**************************************** 42 | * Customization (error_public.h) 43 | ******************************************/ 44 | typedef ZSTD_ErrorCode ERR_enum; 45 | #define PREFIX(name) ZSTD_error_##name 46 | 47 | 48 | /*-**************************************** 49 | * Error codes handling 50 | ******************************************/ 51 | #ifdef ERROR 52 | # undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ 53 | #endif 54 | #define ERROR(name) ((size_t)-PREFIX(name)) 55 | 56 | ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } 57 | 58 | ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } 59 | 60 | 61 | /*-**************************************** 62 | * Error Strings 63 | ******************************************/ 64 | 65 | ERR_STATIC const char* ERR_getErrorString(ERR_enum code) 66 | { 67 | static const char* notErrorCode = "Unspecified error code"; 68 | switch( code ) 69 | { 70 | case PREFIX(no_error): return "No error detected"; 71 | case PREFIX(GENERIC): return "Error (generic)"; 72 | case PREFIX(prefix_unknown): return "Unknown frame descriptor"; 73 | case PREFIX(version_unsupported): return "Version not supported"; 74 | case PREFIX(parameter_unknown): return "Unknown parameter type"; 75 | case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; 76 | case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; 77 | case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; 78 | case PREFIX(init_missing): return "Context should be init first"; 79 | case PREFIX(memory_allocation): return "Allocation error : not enough memory"; 80 | case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; 81 | case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; 82 | case PREFIX(srcSize_wrong): return "Src size incorrect"; 83 | case PREFIX(corruption_detected): return "Corrupted block detected"; 84 | case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; 85 | case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; 86 | case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; 87 | case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; 88 | case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; 89 | case PREFIX(dictionary_wrong): return "Dictionary mismatch"; 90 | case PREFIX(maxCode): 91 | default: return notErrorCode; 92 | } 93 | } 94 | 95 | ERR_STATIC const char* ERR_getErrorName(size_t code) 96 | { 97 | return ERR_getErrorString(ERR_getErrorCode(code)); 98 | } 99 | 100 | #if defined (__cplusplus) 101 | } 102 | #endif 103 | 104 | #endif /* ERROR_H_MODULE */ 105 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # This file was generated by the `rspec --init` command. Conventionally, all 2 | # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. 3 | # The generated `.rspec` file contains `--require spec_helper` which will cause 4 | # this file to always be loaded, without a need to explicitly require it in any 5 | # files. 6 | # 7 | # Given that it is always loaded, you are encouraged to keep this file as 8 | # light-weight as possible. Requiring heavyweight dependencies from this file 9 | # will add to the boot time of your test suite on EVERY test run, even for an 10 | # individual file that may not need all of that loaded. Instead, consider making 11 | # a separate helper file that requires the additional dependencies and performs 12 | # the additional setup, and require it from the spec files that actually need 13 | # it. 14 | # 15 | # The `.rspec` file also contains a few flags that are not defaults but that 16 | # users commonly want. 17 | # 18 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 19 | RSpec.configure do |config| 20 | # rspec-expectations config goes here. You can use an alternate 21 | # assertion/expectation library such as wrong or the stdlib/minitest 22 | # assertions if you prefer. 23 | config.expect_with :rspec do |expectations| 24 | # This option will default to `true` in RSpec 4. It makes the `description` 25 | # and `failure_message` of custom matchers include text for helper methods 26 | # defined using `chain`, e.g.: 27 | # be_bigger_than(2).and_smaller_than(4).description 28 | # # => "be bigger than 2 and smaller than 4" 29 | # ...rather than: 30 | # # => "be bigger than 2" 31 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 32 | end 33 | 34 | # rspec-mocks config goes here. You can use an alternate test double 35 | # library (such as bogus or mocha) by changing the `mock_with` option here. 36 | config.mock_with :rspec do |mocks| 37 | # Prevents you from mocking or stubbing a method that does not exist on 38 | # a real object. This is generally recommended, and will default to 39 | # `true` in RSpec 4. 40 | mocks.verify_partial_doubles = true 41 | end 42 | 43 | # The settings below are suggested to provide a good initial experience 44 | # with RSpec, but feel free to customize to your heart's content. 45 | =begin 46 | # These two settings work together to allow you to limit a spec run 47 | # to individual examples or groups you care about by tagging them with 48 | # `:focus` metadata. When nothing is tagged with `:focus`, all examples 49 | # get run. 50 | config.filter_run :focus 51 | config.run_all_when_everything_filtered = true 52 | 53 | # Allows RSpec to persist some state between runs in order to support 54 | # the `--only-failures` and `--next-failure` CLI options. We recommend 55 | # you configure your source control system to ignore this file. 56 | config.example_status_persistence_file_path = "spec/examples.txt" 57 | 58 | # Limits the available syntax to the non-monkey patched syntax that is 59 | # recommended. For more details, see: 60 | # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ 61 | # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ 62 | # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode 63 | config.disable_monkey_patching! 64 | 65 | # This setting enables warnings. It's recommended, but in some cases may 66 | # be too noisy due to issues in dependencies. 67 | config.warnings = true 68 | 69 | # Many RSpec users commonly either run the entire suite or an individual 70 | # file, and it's useful to allow more verbose output when running an 71 | # individual spec file. 72 | if config.files_to_run.one? 73 | # Use the documentation formatter for detailed output, 74 | # unless a formatter has already been configured 75 | # (e.g. via a command-line flag). 76 | config.default_formatter = 'doc' 77 | end 78 | 79 | # Print the 10 slowest examples and example groups at the 80 | # end of the spec run, to help surface which specs are running 81 | # particularly slow. 82 | config.profile_examples = 10 83 | 84 | # Run specs in random order to surface order dependencies. If you find an 85 | # order dependency and want to debug it, you can fix the order by providing 86 | # the seed, which is printed after each run. 87 | # --seed 1234 88 | config.order = :random 89 | 90 | # Seed global randomization in this process using the `--seed` CLI option. 91 | # Setting this allows you to use `--seed` to deterministically reproduce 92 | # test failures related to randomization by passing the same `--seed` value 93 | # as the one that triggered the failure. 94 | Kernel.srand config.seed 95 | =end 96 | end 97 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under the BSD-style license found in the 6 | # LICENSE file in the root directory of this source tree. An additional grant 7 | # of patent rights can be found in the PATENTS file in the same directory. 8 | # ################################################################ 9 | 10 | # Version numbers 11 | LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h` 12 | LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h` 13 | LIBVER_PATCH_SCRIPT:=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h` 14 | LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) 15 | LIBVER_MAJOR := $(shell echo $(LIBVER_MAJOR_SCRIPT)) 16 | LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) 17 | LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) 18 | LIBVER := $(shell echo $(LIBVER_SCRIPT)) 19 | VERSION?= $(LIBVER) 20 | 21 | DESTDIR?= 22 | PREFIX ?= /usr/local 23 | LIBDIR ?= $(PREFIX)/lib 24 | INCLUDEDIR=$(PREFIX)/include 25 | 26 | CPPFLAGS= -I. -I./common 27 | CFLAGS ?= -O3 28 | CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \ 29 | -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef 30 | FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) 31 | 32 | 33 | ZSTD_FILES := common/*.c compress/*.c decompress/*.c dictBuilder/*.c 34 | 35 | ifeq ($(ZSTD_LEGACY_SUPPORT), 0) 36 | CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0 37 | else 38 | ZSTD_FILES+= legacy/*.c 39 | CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1 40 | endif 41 | 42 | 43 | # OS X linker doesn't support -soname, and use different extension 44 | # see : https://developer.apple.com/library/mac/documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/DynamicLibraryDesignGuidelines.html 45 | ifeq ($(shell uname), Darwin) 46 | SHARED_EXT = dylib 47 | SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT) 48 | SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT) 49 | SONAME_FLAGS = -install_name $(PREFIX)/lib/$@.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER) 50 | else 51 | SONAME_FLAGS = -Wl,-soname=$@.$(SHARED_EXT).$(LIBVER_MAJOR) 52 | SHARED_EXT = so 53 | SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR) 54 | SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER) 55 | endif 56 | 57 | 58 | .PHONY: default all clean install uninstall 59 | 60 | default: clean libzstd 61 | 62 | all: clean libzstd 63 | 64 | libzstd: $(ZSTD_FILES) 65 | @echo compiling static library 66 | @$(CC) $(FLAGS) -c $^ 67 | @$(AR) rcs $@.a *.o 68 | @echo compiling dynamic library $(LIBVER) 69 | @$(CC) $(FLAGS) -shared $^ -fPIC $(SONAME_FLAGS) -o $@.$(SHARED_EXT_VER) 70 | @echo creating versioned links 71 | @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT_MAJOR) 72 | @ln -sf $@.$(SHARED_EXT_VER) $@.$(SHARED_EXT) 73 | 74 | clean: 75 | @rm -f core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc 76 | @rm -f decompress/*.o 77 | @echo Cleaning library completed 78 | 79 | #------------------------------------------------------------------------ 80 | #make install is validated only for Linux, OSX, kFreeBSD, Hurd and some BSD targets 81 | ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly)) 82 | 83 | libzstd.pc: 84 | libzstd.pc: libzstd.pc.in 85 | @echo creating pkgconfig 86 | @sed -e 's|@PREFIX@|$(PREFIX)|' \ 87 | -e 's|@LIBDIR@|$(LIBDIR)|' \ 88 | -e 's|@INCLUDEDIR@|$(INCLUDEDIR)|' \ 89 | -e 's|@VERSION@|$(VERSION)|' \ 90 | $< >$@ 91 | 92 | install: libzstd libzstd.pc 93 | @install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/ 94 | @install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) 95 | @cp -a libzstd.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR) 96 | @cp -a libzstd.$(SHARED_EXT) $(DESTDIR)$(LIBDIR) 97 | @cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/ 98 | @install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a 99 | @install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h 100 | @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h 101 | @install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h 102 | @echo zstd static and shared library installed 103 | 104 | uninstall: 105 | $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT) 106 | $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) 107 | $(RM) $(DESTDIR)$(LIBDIR)/pkgconfig/libzstd.pc 108 | [ -x $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) ] && $(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER) 109 | @[ -f $(DESTDIR)$(LIBDIR)/libzstd.a ] && $(RM) $(DESTDIR)$(LIBDIR)/libzstd.a 110 | @[ -f $(DESTDIR)$(INCLUDEDIR)/zstd.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h 111 | @[ -f $(DESTDIR)$(INCLUDEDIR)/zbuff.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h 112 | @[ -f $(DESTDIR)$(INCLUDEDIR)/zdict.h ] && $(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h 113 | @echo zstd libraries successfully uninstalled 114 | 115 | endif 116 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/dictBuilder/zdict.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef DICTBUILDER_H_001 11 | #define DICTBUILDER_H_001 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | 18 | /*====== Dependencies ======*/ 19 | #include /* size_t */ 20 | 21 | 22 | /*====== Export for Windows ======*/ 23 | /*! 24 | * ZSTD_DLL_EXPORT : 25 | * Enable exporting of functions when building a Windows DLL 26 | */ 27 | #if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) 28 | # define ZDICTLIB_API __declspec(dllexport) 29 | #else 30 | # define ZDICTLIB_API 31 | #endif 32 | 33 | 34 | /*! ZDICT_trainFromBuffer() : 35 | Train a dictionary from an array of samples. 36 | Samples must be stored concatenated in a single flat buffer `samplesBuffer`, 37 | supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. 38 | The resulting dictionary will be saved into `dictBuffer`. 39 | @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) 40 | or an error code, which can be tested with ZDICT_isError(). 41 | Tips : In general, a reasonable dictionary has a size of ~ 100 KB. 42 | It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`. 43 | In general, it's recommended to provide a few thousands samples, but this can vary a lot. 44 | It's recommended that total size of all samples be about ~x100 times the target size of dictionary. 45 | */ 46 | ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, 47 | const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); 48 | 49 | 50 | /*====== Helper functions ======*/ 51 | ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ 52 | ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); 53 | ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); 54 | 55 | 56 | 57 | #ifdef ZDICT_STATIC_LINKING_ONLY 58 | 59 | /* ==================================================================================== 60 | * The definitions in this section are considered experimental. 61 | * They should never be used with a dynamic library, as they may change in the future. 62 | * They are provided for advanced usages. 63 | * Use them only in association with static linking. 64 | * ==================================================================================== */ 65 | 66 | typedef struct { 67 | unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */ 68 | int compressionLevel; /* 0 means default; target a specific zstd compression level */ 69 | unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ 70 | unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */ 71 | unsigned reserved[2]; /* space for future parameters */ 72 | } ZDICT_params_t; 73 | 74 | 75 | /*! ZDICT_trainFromBuffer_advanced() : 76 | Same as ZDICT_trainFromBuffer() with control over more parameters. 77 | `parameters` is optional and can be provided with values set to 0 to mean "default". 78 | @return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`), 79 | or an error code, which can be tested by ZDICT_isError(). 80 | note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0. 81 | */ 82 | size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity, 83 | const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, 84 | ZDICT_params_t parameters); 85 | 86 | 87 | /*! ZDICT_addEntropyTablesFromBuffer() : 88 | 89 | Given a content-only dictionary (built using any 3rd party algorithm), 90 | add entropy tables computed from an array of samples. 91 | Samples must be stored concatenated in a flat buffer `samplesBuffer`, 92 | supplied with an array of sizes `samplesSizes`, providing the size of each sample in order. 93 | 94 | The input dictionary content must be stored *at the end* of `dictBuffer`. 95 | Its size is `dictContentSize`. 96 | The resulting dictionary with added entropy tables will be *written back to `dictBuffer`*, 97 | starting from its beginning. 98 | @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`). 99 | */ 100 | size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, 101 | const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); 102 | 103 | 104 | 105 | #endif /* ZDICT_STATIC_LINKING_ONLY */ 106 | 107 | #if defined (__cplusplus) 108 | } 109 | #endif 110 | 111 | #endif /* DICTBUILDER_H_001 */ 112 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/legacy/zstd_v04.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef ZSTD_V04_H_91868324769238 11 | #define ZSTD_V04_H_91868324769238 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | /* ************************************* 18 | * Includes 19 | ***************************************/ 20 | #include /* size_t */ 21 | 22 | 23 | /* ************************************* 24 | * Simple one-step function 25 | ***************************************/ 26 | /** 27 | ZSTDv04_decompress() : decompress ZSTD frames compliant with v0.4.x format 28 | compressedSize : is the exact source size 29 | maxOriginalSize : is the size of the 'dst' buffer, which must be already allocated. 30 | It must be equal or larger than originalSize, otherwise decompression will fail. 31 | return : the number of bytes decompressed into destination buffer (originalSize) 32 | or an errorCode if it fails (which can be tested using ZSTDv01_isError()) 33 | */ 34 | size_t ZSTDv04_decompress( void* dst, size_t maxOriginalSize, 35 | const void* src, size_t compressedSize); 36 | 37 | /** 38 | ZSTDv04_isError() : tells if the result of ZSTDv04_decompress() is an error 39 | */ 40 | unsigned ZSTDv04_isError(size_t code); 41 | 42 | 43 | /* ************************************* 44 | * Advanced functions 45 | ***************************************/ 46 | typedef struct ZSTDv04_Dctx_s ZSTDv04_Dctx; 47 | ZSTDv04_Dctx* ZSTDv04_createDCtx(void); 48 | size_t ZSTDv04_freeDCtx(ZSTDv04_Dctx* dctx); 49 | 50 | size_t ZSTDv04_decompressDCtx(ZSTDv04_Dctx* dctx, 51 | void* dst, size_t maxOriginalSize, 52 | const void* src, size_t compressedSize); 53 | 54 | 55 | /* ************************************* 56 | * Direct Streaming 57 | ***************************************/ 58 | size_t ZSTDv04_resetDCtx(ZSTDv04_Dctx* dctx); 59 | 60 | size_t ZSTDv04_nextSrcSizeToDecompress(ZSTDv04_Dctx* dctx); 61 | size_t ZSTDv04_decompressContinue(ZSTDv04_Dctx* dctx, void* dst, size_t maxDstSize, const void* src, size_t srcSize); 62 | /** 63 | Use above functions alternatively. 64 | ZSTD_nextSrcSizeToDecompress() tells how much bytes to provide as 'srcSize' to ZSTD_decompressContinue(). 65 | ZSTD_decompressContinue() will use previous data blocks to improve compression if they are located prior to current block. 66 | Result is the number of bytes regenerated within 'dst'. 67 | It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header. 68 | */ 69 | 70 | 71 | /* ************************************* 72 | * Buffered Streaming 73 | ***************************************/ 74 | typedef struct ZBUFFv04_DCtx_s ZBUFFv04_DCtx; 75 | ZBUFFv04_DCtx* ZBUFFv04_createDCtx(void); 76 | size_t ZBUFFv04_freeDCtx(ZBUFFv04_DCtx* dctx); 77 | 78 | size_t ZBUFFv04_decompressInit(ZBUFFv04_DCtx* dctx); 79 | size_t ZBUFFv04_decompressWithDictionary(ZBUFFv04_DCtx* dctx, const void* dict, size_t dictSize); 80 | 81 | size_t ZBUFFv04_decompressContinue(ZBUFFv04_DCtx* dctx, void* dst, size_t* maxDstSizePtr, const void* src, size_t* srcSizePtr); 82 | 83 | /** ************************************************ 84 | * Streaming decompression 85 | * 86 | * A ZBUFF_DCtx object is required to track streaming operation. 87 | * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. 88 | * Use ZBUFF_decompressInit() to start a new decompression operation. 89 | * ZBUFF_DCtx objects can be reused multiple times. 90 | * 91 | * Optionally, a reference to a static dictionary can be set, using ZBUFF_decompressWithDictionary() 92 | * It must be the same content as the one set during compression phase. 93 | * Dictionary content must remain accessible during the decompression process. 94 | * 95 | * Use ZBUFF_decompressContinue() repetitively to consume your input. 96 | * *srcSizePtr and *maxDstSizePtr can be any size. 97 | * The function will report how many bytes were read or written by modifying *srcSizePtr and *maxDstSizePtr. 98 | * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. 99 | * The content of dst will be overwritten (up to *maxDstSizePtr) at each function call, so save its content if it matters or change dst. 100 | * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) 101 | * or 0 when a frame is completely decoded 102 | * or an error code, which can be tested using ZBUFF_isError(). 103 | * 104 | * Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize / ZBUFF_recommendedDOutSize 105 | * output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when it's decoded. 106 | * input : ZBUFF_recommendedDInSize==128Kb+3; just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . 107 | * **************************************************/ 108 | unsigned ZBUFFv04_isError(size_t errorCode); 109 | const char* ZBUFFv04_getErrorName(size_t errorCode); 110 | 111 | 112 | /** The below functions provide recommended buffer sizes for Compression or Decompression operations. 113 | * These sizes are not compulsory, they just tend to offer better latency */ 114 | size_t ZBUFFv04_recommendedDInSize(void); 115 | size_t ZBUFFv04_recommendedDOutSize(void); 116 | 117 | 118 | /* ************************************* 119 | * Prefix - version detection 120 | ***************************************/ 121 | #define ZSTDv04_magicNumber 0xFD2FB524 /* v0.4 */ 122 | 123 | 124 | #if defined (__cplusplus) 125 | } 126 | #endif 127 | 128 | #endif /* ZSTD_V04_H_91868324769238 */ 129 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/legacy/zstd_v05.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef ZSTDv05_H 11 | #define ZSTDv05_H 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | /*-************************************* 18 | * Dependencies 19 | ***************************************/ 20 | #include /* size_t */ 21 | #include "mem.h" /* U64, U32 */ 22 | 23 | 24 | /* ************************************* 25 | * Simple functions 26 | ***************************************/ 27 | /*! ZSTDv05_decompress() : 28 | `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail. 29 | `dstCapacity` must be large enough, equal or larger than originalSize. 30 | @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), 31 | or an errorCode if it fails (which can be tested using ZSTDv05_isError()) */ 32 | size_t ZSTDv05_decompress( void* dst, size_t dstCapacity, 33 | const void* src, size_t compressedSize); 34 | 35 | 36 | /* ************************************* 37 | * Helper functions 38 | ***************************************/ 39 | /* Error Management */ 40 | unsigned ZSTDv05_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ 41 | const char* ZSTDv05_getErrorName(size_t code); /*!< provides readable string for an error code */ 42 | 43 | 44 | /* ************************************* 45 | * Explicit memory management 46 | ***************************************/ 47 | /** Decompression context */ 48 | typedef struct ZSTDv05_DCtx_s ZSTDv05_DCtx; 49 | ZSTDv05_DCtx* ZSTDv05_createDCtx(void); 50 | size_t ZSTDv05_freeDCtx(ZSTDv05_DCtx* dctx); /*!< @return : errorCode */ 51 | 52 | /** ZSTDv05_decompressDCtx() : 53 | * Same as ZSTDv05_decompress(), but requires an already allocated ZSTDv05_DCtx (see ZSTDv05_createDCtx()) */ 54 | size_t ZSTDv05_decompressDCtx(ZSTDv05_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 55 | 56 | 57 | /*-*********************** 58 | * Simple Dictionary API 59 | *************************/ 60 | /*! ZSTDv05_decompress_usingDict() : 61 | * Decompression using a pre-defined Dictionary content (see dictBuilder). 62 | * Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. 63 | * Note : dict can be NULL, in which case, it's equivalent to ZSTDv05_decompressDCtx() */ 64 | size_t ZSTDv05_decompress_usingDict(ZSTDv05_DCtx* dctx, 65 | void* dst, size_t dstCapacity, 66 | const void* src, size_t srcSize, 67 | const void* dict,size_t dictSize); 68 | 69 | /*-************************ 70 | * Advanced Streaming API 71 | ***************************/ 72 | typedef enum { ZSTDv05_fast, ZSTDv05_greedy, ZSTDv05_lazy, ZSTDv05_lazy2, ZSTDv05_btlazy2, ZSTDv05_opt, ZSTDv05_btopt } ZSTDv05_strategy; 73 | typedef struct { 74 | U64 srcSize; 75 | U32 windowLog; /* the only useful information to retrieve */ 76 | U32 contentLog; U32 hashLog; U32 searchLog; U32 searchLength; U32 targetLength; ZSTDv05_strategy strategy; 77 | } ZSTDv05_parameters; 78 | size_t ZSTDv05_getFrameParams(ZSTDv05_parameters* params, const void* src, size_t srcSize); 79 | 80 | size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, size_t dictSize); 81 | void ZSTDv05_copyDCtx(ZSTDv05_DCtx* dstDCtx, const ZSTDv05_DCtx* srcDCtx); 82 | size_t ZSTDv05_nextSrcSizeToDecompress(ZSTDv05_DCtx* dctx); 83 | size_t ZSTDv05_decompressContinue(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 84 | 85 | 86 | /*-*********************** 87 | * ZBUFF API 88 | *************************/ 89 | typedef struct ZBUFFv05_DCtx_s ZBUFFv05_DCtx; 90 | ZBUFFv05_DCtx* ZBUFFv05_createDCtx(void); 91 | size_t ZBUFFv05_freeDCtx(ZBUFFv05_DCtx* dctx); 92 | 93 | size_t ZBUFFv05_decompressInit(ZBUFFv05_DCtx* dctx); 94 | size_t ZBUFFv05_decompressInitDictionary(ZBUFFv05_DCtx* dctx, const void* dict, size_t dictSize); 95 | 96 | size_t ZBUFFv05_decompressContinue(ZBUFFv05_DCtx* dctx, 97 | void* dst, size_t* dstCapacityPtr, 98 | const void* src, size_t* srcSizePtr); 99 | 100 | /*-*************************************************************************** 101 | * Streaming decompression 102 | * 103 | * A ZBUFFv05_DCtx object is required to track streaming operations. 104 | * Use ZBUFFv05_createDCtx() and ZBUFFv05_freeDCtx() to create/release resources. 105 | * Use ZBUFFv05_decompressInit() to start a new decompression operation, 106 | * or ZBUFFv05_decompressInitDictionary() if decompression requires a dictionary. 107 | * Note that ZBUFFv05_DCtx objects can be reused multiple times. 108 | * 109 | * Use ZBUFFv05_decompressContinue() repetitively to consume your input. 110 | * *srcSizePtr and *dstCapacityPtr can be any size. 111 | * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. 112 | * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. 113 | * The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change @dst. 114 | * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency) 115 | * or 0 when a frame is completely decoded 116 | * or an error code, which can be tested using ZBUFFv05_isError(). 117 | * 118 | * Hint : recommended buffer sizes (not compulsory) : ZBUFFv05_recommendedDInSize() / ZBUFFv05_recommendedDOutSize() 119 | * output : ZBUFFv05_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. 120 | * input : ZBUFFv05_recommendedDInSize==128Kb+3; just follow indications from ZBUFFv05_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . 121 | * *******************************************************************************/ 122 | 123 | 124 | /* ************************************* 125 | * Tool functions 126 | ***************************************/ 127 | unsigned ZBUFFv05_isError(size_t errorCode); 128 | const char* ZBUFFv05_getErrorName(size_t errorCode); 129 | 130 | /** Functions below provide recommended buffer sizes for Compression or Decompression operations. 131 | * These sizes are just hints, and tend to offer better latency */ 132 | size_t ZBUFFv05_recommendedDInSize(void); 133 | size_t ZBUFFv05_recommendedDOutSize(void); 134 | 135 | 136 | 137 | /*-************************************* 138 | * Constants 139 | ***************************************/ 140 | #define ZSTDv05_MAGICNUMBER 0xFD2FB525 /* v0.5 */ 141 | 142 | 143 | 144 | 145 | #if defined (__cplusplus) 146 | } 147 | #endif 148 | 149 | #endif /* ZSTDv0505_H */ 150 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/legacy/zstd_v06.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef ZSTDv06_H 11 | #define ZSTDv06_H 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | /*-************************************* 18 | * Dependencies 19 | ***************************************/ 20 | #include /* size_t */ 21 | 22 | 23 | /*-*************************************************************** 24 | * Export parameters 25 | *****************************************************************/ 26 | /*! 27 | * ZSTDv06_DLL_EXPORT : 28 | * Enable exporting of functions when building a Windows DLL 29 | */ 30 | #if defined(_WIN32) && defined(ZSTDv06_DLL_EXPORT) && (ZSTDv06_DLL_EXPORT==1) 31 | # define ZSTDLIB_API __declspec(dllexport) 32 | #else 33 | # define ZSTDLIB_API 34 | #endif 35 | 36 | 37 | /* ************************************* 38 | * Simple functions 39 | ***************************************/ 40 | /*! ZSTDv06_decompress() : 41 | `compressedSize` : is the _exact_ size of the compressed blob, otherwise decompression will fail. 42 | `dstCapacity` must be large enough, equal or larger than originalSize. 43 | @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), 44 | or an errorCode if it fails (which can be tested using ZSTDv06_isError()) */ 45 | ZSTDLIB_API size_t ZSTDv06_decompress( void* dst, size_t dstCapacity, 46 | const void* src, size_t compressedSize); 47 | 48 | 49 | /* ************************************* 50 | * Helper functions 51 | ***************************************/ 52 | ZSTDLIB_API size_t ZSTDv06_compressBound(size_t srcSize); /*!< maximum compressed size (worst case scenario) */ 53 | 54 | /* Error Management */ 55 | ZSTDLIB_API unsigned ZSTDv06_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ 56 | ZSTDLIB_API const char* ZSTDv06_getErrorName(size_t code); /*!< provides readable string for an error code */ 57 | 58 | 59 | /* ************************************* 60 | * Explicit memory management 61 | ***************************************/ 62 | /** Decompression context */ 63 | typedef struct ZSTDv06_DCtx_s ZSTDv06_DCtx; 64 | ZSTDLIB_API ZSTDv06_DCtx* ZSTDv06_createDCtx(void); 65 | ZSTDLIB_API size_t ZSTDv06_freeDCtx(ZSTDv06_DCtx* dctx); /*!< @return : errorCode */ 66 | 67 | /** ZSTDv06_decompressDCtx() : 68 | * Same as ZSTDv06_decompress(), but requires an already allocated ZSTDv06_DCtx (see ZSTDv06_createDCtx()) */ 69 | ZSTDLIB_API size_t ZSTDv06_decompressDCtx(ZSTDv06_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 70 | 71 | 72 | /*-*********************** 73 | * Dictionary API 74 | *************************/ 75 | /*! ZSTDv06_decompress_usingDict() : 76 | * Decompression using a pre-defined Dictionary content (see dictBuilder). 77 | * Dictionary must be identical to the one used during compression, otherwise regenerated data will be corrupted. 78 | * Note : dict can be NULL, in which case, it's equivalent to ZSTDv06_decompressDCtx() */ 79 | ZSTDLIB_API size_t ZSTDv06_decompress_usingDict(ZSTDv06_DCtx* dctx, 80 | void* dst, size_t dstCapacity, 81 | const void* src, size_t srcSize, 82 | const void* dict,size_t dictSize); 83 | 84 | 85 | /*-************************ 86 | * Advanced Streaming API 87 | ***************************/ 88 | struct ZSTDv06_frameParams_s { unsigned long long frameContentSize; unsigned windowLog; }; 89 | typedef struct ZSTDv06_frameParams_s ZSTDv06_frameParams; 90 | 91 | ZSTDLIB_API size_t ZSTDv06_getFrameParams(ZSTDv06_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ 92 | ZSTDLIB_API size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, size_t dictSize); 93 | ZSTDLIB_API void ZSTDv06_copyDCtx(ZSTDv06_DCtx* dctx, const ZSTDv06_DCtx* preparedDCtx); 94 | 95 | ZSTDLIB_API size_t ZSTDv06_nextSrcSizeToDecompress(ZSTDv06_DCtx* dctx); 96 | ZSTDLIB_API size_t ZSTDv06_decompressContinue(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 97 | 98 | 99 | 100 | /* ************************************* 101 | * ZBUFF API 102 | ***************************************/ 103 | 104 | typedef struct ZBUFFv06_DCtx_s ZBUFFv06_DCtx; 105 | ZSTDLIB_API ZBUFFv06_DCtx* ZBUFFv06_createDCtx(void); 106 | ZSTDLIB_API size_t ZBUFFv06_freeDCtx(ZBUFFv06_DCtx* dctx); 107 | 108 | ZSTDLIB_API size_t ZBUFFv06_decompressInit(ZBUFFv06_DCtx* dctx); 109 | ZSTDLIB_API size_t ZBUFFv06_decompressInitDictionary(ZBUFFv06_DCtx* dctx, const void* dict, size_t dictSize); 110 | 111 | ZSTDLIB_API size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* dctx, 112 | void* dst, size_t* dstCapacityPtr, 113 | const void* src, size_t* srcSizePtr); 114 | 115 | /*-*************************************************************************** 116 | * Streaming decompression howto 117 | * 118 | * A ZBUFFv06_DCtx object is required to track streaming operations. 119 | * Use ZBUFFv06_createDCtx() and ZBUFFv06_freeDCtx() to create/release resources. 120 | * Use ZBUFFv06_decompressInit() to start a new decompression operation, 121 | * or ZBUFFv06_decompressInitDictionary() if decompression requires a dictionary. 122 | * Note that ZBUFFv06_DCtx objects can be re-init multiple times. 123 | * 124 | * Use ZBUFFv06_decompressContinue() repetitively to consume your input. 125 | * *srcSizePtr and *dstCapacityPtr can be any size. 126 | * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. 127 | * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. 128 | * The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. 129 | * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), 130 | * or 0 when a frame is completely decoded, 131 | * or an error code, which can be tested using ZBUFFv06_isError(). 132 | * 133 | * Hint : recommended buffer sizes (not compulsory) : ZBUFFv06_recommendedDInSize() and ZBUFFv06_recommendedDOutSize() 134 | * output : ZBUFFv06_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. 135 | * input : ZBUFFv06_recommendedDInSize == 128KB + 3; 136 | * just follow indications from ZBUFFv06_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . 137 | * *******************************************************************************/ 138 | 139 | 140 | /* ************************************* 141 | * Tool functions 142 | ***************************************/ 143 | ZSTDLIB_API unsigned ZBUFFv06_isError(size_t errorCode); 144 | ZSTDLIB_API const char* ZBUFFv06_getErrorName(size_t errorCode); 145 | 146 | /** Functions below provide recommended buffer sizes for Compression or Decompression operations. 147 | * These sizes are just hints, they tend to offer better latency */ 148 | ZSTDLIB_API size_t ZBUFFv06_recommendedDInSize(void); 149 | ZSTDLIB_API size_t ZBUFFv06_recommendedDOutSize(void); 150 | 151 | 152 | /*-************************************* 153 | * Constants 154 | ***************************************/ 155 | #define ZSTDv06_MAGICNUMBER 0xFD2FB526 /* v0.6 */ 156 | 157 | 158 | 159 | #if defined (__cplusplus) 160 | } 161 | #endif 162 | 163 | #endif /* ZSTDv06_BUFFERED_H */ 164 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/common/zstd_internal.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef ZSTD_CCOMMON_H_MODULE 11 | #define ZSTD_CCOMMON_H_MODULE 12 | 13 | /*-************************************* 14 | * Dependencies 15 | ***************************************/ 16 | #include "mem.h" 17 | #include "error_private.h" 18 | #define ZSTD_STATIC_LINKING_ONLY 19 | #include "zstd.h" 20 | 21 | 22 | /*-************************************* 23 | * Common macros 24 | ***************************************/ 25 | #define MIN(a,b) ((a)<(b) ? (a) : (b)) 26 | #define MAX(a,b) ((a)>(b) ? (a) : (b)) 27 | 28 | 29 | /*-************************************* 30 | * Common constants 31 | ***************************************/ 32 | #define ZSTD_OPT_NUM (1<<12) 33 | #define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ 34 | 35 | #define ZSTD_REP_NUM 3 /* number of repcodes */ 36 | #define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ 37 | #define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) 38 | #define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM) 39 | static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; 40 | 41 | #define KB *(1 <<10) 42 | #define MB *(1 <<20) 43 | #define GB *(1U<<30) 44 | 45 | #define BIT7 128 46 | #define BIT6 64 47 | #define BIT5 32 48 | #define BIT4 16 49 | #define BIT1 2 50 | #define BIT0 1 51 | 52 | #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 53 | static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; 54 | static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; 55 | 56 | #define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ 57 | static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; 58 | typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; 59 | 60 | #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ 61 | #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ 62 | 63 | #define HufLog 12 64 | typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; 65 | 66 | #define LONGNBSEQ 0x7F00 67 | 68 | #define MINMATCH 3 69 | #define EQUAL_READ32 4 70 | 71 | #define Litbits 8 72 | #define MaxLit ((1<= 3) /* GCC Intrinsic */ 214 | return 31 - __builtin_clz(val); 215 | # else /* Software version */ 216 | static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; 217 | U32 v = val; 218 | int r; 219 | v |= v >> 1; 220 | v |= v >> 2; 221 | v |= v >> 4; 222 | v |= v >> 8; 223 | v |= v >> 16; 224 | r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; 225 | return r; 226 | # endif 227 | } 228 | 229 | 230 | #endif /* ZSTD_CCOMMON_H_MODULE */ 231 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/legacy/zstd_v07.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef ZSTDv07_H_235446 11 | #define ZSTDv07_H_235446 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | /*====== Dependency ======*/ 18 | #include /* size_t */ 19 | 20 | 21 | /*====== Export for Windows ======*/ 22 | /*! 23 | * ZSTDv07_DLL_EXPORT : 24 | * Enable exporting of functions when building a Windows DLL 25 | */ 26 | #if defined(_WIN32) && defined(ZSTDv07_DLL_EXPORT) && (ZSTDv07_DLL_EXPORT==1) 27 | # define ZSTDLIB_API __declspec(dllexport) 28 | #else 29 | # define ZSTDLIB_API 30 | #endif 31 | 32 | 33 | 34 | /* ************************************* 35 | * Simple API 36 | ***************************************/ 37 | /*! ZSTDv07_getDecompressedSize() : 38 | * @return : decompressed size if known, 0 otherwise. 39 | note 1 : if `0`, follow up with ZSTDv07_getFrameParams() to know precise failure cause. 40 | note 2 : decompressed size could be wrong or intentionally modified ! 41 | always ensure results fit within application's authorized limits */ 42 | unsigned long long ZSTDv07_getDecompressedSize(const void* src, size_t srcSize); 43 | 44 | /*! ZSTDv07_decompress() : 45 | `compressedSize` : must be _exact_ size of compressed input, otherwise decompression will fail. 46 | `dstCapacity` must be equal or larger than originalSize. 47 | @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), 48 | or an errorCode if it fails (which can be tested using ZSTDv07_isError()) */ 49 | ZSTDLIB_API size_t ZSTDv07_decompress( void* dst, size_t dstCapacity, 50 | const void* src, size_t compressedSize); 51 | 52 | /*====== Helper functions ======*/ 53 | ZSTDLIB_API unsigned ZSTDv07_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ 54 | ZSTDLIB_API const char* ZSTDv07_getErrorName(size_t code); /*!< provides readable string from an error code */ 55 | 56 | 57 | /*-************************************* 58 | * Explicit memory management 59 | ***************************************/ 60 | /** Decompression context */ 61 | typedef struct ZSTDv07_DCtx_s ZSTDv07_DCtx; 62 | ZSTDLIB_API ZSTDv07_DCtx* ZSTDv07_createDCtx(void); 63 | ZSTDLIB_API size_t ZSTDv07_freeDCtx(ZSTDv07_DCtx* dctx); /*!< @return : errorCode */ 64 | 65 | /** ZSTDv07_decompressDCtx() : 66 | * Same as ZSTDv07_decompress(), requires an allocated ZSTDv07_DCtx (see ZSTDv07_createDCtx()) */ 67 | ZSTDLIB_API size_t ZSTDv07_decompressDCtx(ZSTDv07_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); 68 | 69 | 70 | /*-************************ 71 | * Simple dictionary API 72 | ***************************/ 73 | /*! ZSTDv07_decompress_usingDict() : 74 | * Decompression using a pre-defined Dictionary content (see dictBuilder). 75 | * Dictionary must be identical to the one used during compression. 76 | * Note : This function load the dictionary, resulting in a significant startup time */ 77 | ZSTDLIB_API size_t ZSTDv07_decompress_usingDict(ZSTDv07_DCtx* dctx, 78 | void* dst, size_t dstCapacity, 79 | const void* src, size_t srcSize, 80 | const void* dict,size_t dictSize); 81 | 82 | 83 | /*-************************** 84 | * Advanced Dictionary API 85 | ****************************/ 86 | /*! ZSTDv07_createDDict() : 87 | * Create a digested dictionary, ready to start decompression operation without startup delay. 88 | * `dict` can be released after creation */ 89 | typedef struct ZSTDv07_DDict_s ZSTDv07_DDict; 90 | ZSTDLIB_API ZSTDv07_DDict* ZSTDv07_createDDict(const void* dict, size_t dictSize); 91 | ZSTDLIB_API size_t ZSTDv07_freeDDict(ZSTDv07_DDict* ddict); 92 | 93 | /*! ZSTDv07_decompress_usingDDict() : 94 | * Decompression using a pre-digested Dictionary 95 | * Faster startup than ZSTDv07_decompress_usingDict(), recommended when same dictionary is used multiple times. */ 96 | ZSTDLIB_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, 97 | void* dst, size_t dstCapacity, 98 | const void* src, size_t srcSize, 99 | const ZSTDv07_DDict* ddict); 100 | 101 | typedef struct { 102 | unsigned long long frameContentSize; 103 | unsigned windowSize; 104 | unsigned dictID; 105 | unsigned checksumFlag; 106 | } ZSTDv07_frameParams; 107 | 108 | ZSTDLIB_API size_t ZSTDv07_getFrameParams(ZSTDv07_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ 109 | 110 | 111 | 112 | 113 | /* ************************************* 114 | * Streaming functions 115 | ***************************************/ 116 | typedef struct ZBUFFv07_DCtx_s ZBUFFv07_DCtx; 117 | ZSTDLIB_API ZBUFFv07_DCtx* ZBUFFv07_createDCtx(void); 118 | ZSTDLIB_API size_t ZBUFFv07_freeDCtx(ZBUFFv07_DCtx* dctx); 119 | 120 | ZSTDLIB_API size_t ZBUFFv07_decompressInit(ZBUFFv07_DCtx* dctx); 121 | ZSTDLIB_API size_t ZBUFFv07_decompressInitDictionary(ZBUFFv07_DCtx* dctx, const void* dict, size_t dictSize); 122 | 123 | ZSTDLIB_API size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* dctx, 124 | void* dst, size_t* dstCapacityPtr, 125 | const void* src, size_t* srcSizePtr); 126 | 127 | /*-*************************************************************************** 128 | * Streaming decompression howto 129 | * 130 | * A ZBUFFv07_DCtx object is required to track streaming operations. 131 | * Use ZBUFFv07_createDCtx() and ZBUFFv07_freeDCtx() to create/release resources. 132 | * Use ZBUFFv07_decompressInit() to start a new decompression operation, 133 | * or ZBUFFv07_decompressInitDictionary() if decompression requires a dictionary. 134 | * Note that ZBUFFv07_DCtx objects can be re-init multiple times. 135 | * 136 | * Use ZBUFFv07_decompressContinue() repetitively to consume your input. 137 | * *srcSizePtr and *dstCapacityPtr can be any size. 138 | * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. 139 | * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. 140 | * The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. 141 | * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), 142 | * or 0 when a frame is completely decoded, 143 | * or an error code, which can be tested using ZBUFFv07_isError(). 144 | * 145 | * Hint : recommended buffer sizes (not compulsory) : ZBUFFv07_recommendedDInSize() and ZBUFFv07_recommendedDOutSize() 146 | * output : ZBUFFv07_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. 147 | * input : ZBUFFv07_recommendedDInSize == 128KB + 3; 148 | * just follow indications from ZBUFFv07_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . 149 | * *******************************************************************************/ 150 | 151 | 152 | /* ************************************* 153 | * Tool functions 154 | ***************************************/ 155 | ZSTDLIB_API unsigned ZBUFFv07_isError(size_t errorCode); 156 | ZSTDLIB_API const char* ZBUFFv07_getErrorName(size_t errorCode); 157 | 158 | /** Functions below provide recommended buffer sizes for Compression or Decompression operations. 159 | * These sizes are just hints, they tend to offer better latency */ 160 | ZSTDLIB_API size_t ZBUFFv07_recommendedDInSize(void); 161 | ZSTDLIB_API size_t ZBUFFv07_recommendedDOutSize(void); 162 | 163 | 164 | /*-************************************* 165 | * Constants 166 | ***************************************/ 167 | #define ZSTDv07_MAGICNUMBER 0xFD2FB527 /* v0.7 */ 168 | 169 | 170 | #if defined (__cplusplus) 171 | } 172 | #endif 173 | 174 | #endif /* ZSTDv07_H_235446 */ 175 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/common/entropy_common.c: -------------------------------------------------------------------------------- 1 | /* 2 | Common functions of New Generation Entropy library 3 | Copyright (C) 2016, Yann Collet. 4 | 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy 32 | - Public forum : https://groups.google.com/forum/#!forum/lz4c 33 | *************************************************************************** */ 34 | 35 | /* ************************************* 36 | * Dependencies 37 | ***************************************/ 38 | #include "mem.h" 39 | #include "error_private.h" /* ERR_*, ERROR */ 40 | #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ 41 | #include "fse.h" 42 | #define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ 43 | #include "huf.h" 44 | 45 | 46 | /*-**************************************** 47 | * FSE Error Management 48 | ******************************************/ 49 | unsigned FSE_isError(size_t code) { return ERR_isError(code); } 50 | 51 | const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } 52 | 53 | 54 | /* ************************************************************** 55 | * HUF Error Management 56 | ****************************************************************/ 57 | unsigned HUF_isError(size_t code) { return ERR_isError(code); } 58 | 59 | const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } 60 | 61 | 62 | /*-************************************************************** 63 | * FSE NCount encoding-decoding 64 | ****************************************************************/ 65 | static short FSE_abs(short a) { return (short)(a<0 ? -a : a); } 66 | 67 | size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, 68 | const void* headerBuffer, size_t hbSize) 69 | { 70 | const BYTE* const istart = (const BYTE*) headerBuffer; 71 | const BYTE* const iend = istart + hbSize; 72 | const BYTE* ip = istart; 73 | int nbBits; 74 | int remaining; 75 | int threshold; 76 | U32 bitStream; 77 | int bitCount; 78 | unsigned charnum = 0; 79 | int previous0 = 0; 80 | 81 | if (hbSize < 4) return ERROR(srcSize_wrong); 82 | bitStream = MEM_readLE32(ip); 83 | nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ 84 | if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); 85 | bitStream >>= 4; 86 | bitCount = 4; 87 | *tableLogPtr = nbBits; 88 | remaining = (1<1) & (charnum<=*maxSVPtr)) { 93 | if (previous0) { 94 | unsigned n0 = charnum; 95 | while ((bitStream & 0xFFFF) == 0xFFFF) { 96 | n0 += 24; 97 | if (ip < iend-5) { 98 | ip += 2; 99 | bitStream = MEM_readLE32(ip) >> bitCount; 100 | } else { 101 | bitStream >>= 16; 102 | bitCount += 16; 103 | } } 104 | while ((bitStream & 3) == 3) { 105 | n0 += 3; 106 | bitStream >>= 2; 107 | bitCount += 2; 108 | } 109 | n0 += bitStream & 3; 110 | bitCount += 2; 111 | if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); 112 | while (charnum < n0) normalizedCounter[charnum++] = 0; 113 | if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { 114 | ip += bitCount>>3; 115 | bitCount &= 7; 116 | bitStream = MEM_readLE32(ip) >> bitCount; 117 | } else { 118 | bitStream >>= 2; 119 | } } 120 | { short const max = (short)((2*threshold-1)-remaining); 121 | short count; 122 | 123 | if ((bitStream & (threshold-1)) < (U32)max) { 124 | count = (short)(bitStream & (threshold-1)); 125 | bitCount += nbBits-1; 126 | } else { 127 | count = (short)(bitStream & (2*threshold-1)); 128 | if (count >= threshold) count -= max; 129 | bitCount += nbBits; 130 | } 131 | 132 | count--; /* extra accuracy */ 133 | remaining -= FSE_abs(count); 134 | normalizedCounter[charnum++] = count; 135 | previous0 = !count; 136 | while (remaining < threshold) { 137 | nbBits--; 138 | threshold >>= 1; 139 | } 140 | 141 | if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { 142 | ip += bitCount>>3; 143 | bitCount &= 7; 144 | } else { 145 | bitCount -= (int)(8 * (iend - 4 - ip)); 146 | ip = iend - 4; 147 | } 148 | bitStream = MEM_readLE32(ip) >> (bitCount & 31); 149 | } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ 150 | if (remaining != 1) return ERROR(corruption_detected); 151 | if (bitCount > 32) return ERROR(corruption_detected); 152 | *maxSVPtr = charnum-1; 153 | 154 | ip += (bitCount+7)>>3; 155 | return ip-istart; 156 | } 157 | 158 | 159 | /*! HUF_readStats() : 160 | Read compact Huffman tree, saved by HUF_writeCTable(). 161 | `huffWeight` is destination buffer. 162 | @return : size read from `src` , or an error Code . 163 | Note : Needed by HUF_readCTable() and HUF_readDTableX?() . 164 | */ 165 | size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, 166 | U32* nbSymbolsPtr, U32* tableLogPtr, 167 | const void* src, size_t srcSize) 168 | { 169 | U32 weightTotal; 170 | const BYTE* ip = (const BYTE*) src; 171 | size_t iSize = ip[0]; 172 | size_t oSize; 173 | 174 | /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ 175 | 176 | if (iSize >= 128) { /* special header */ 177 | oSize = iSize - 127; 178 | iSize = ((oSize+1)/2); 179 | if (iSize+1 > srcSize) return ERROR(srcSize_wrong); 180 | if (oSize >= hwSize) return ERROR(corruption_detected); 181 | ip += 1; 182 | { U32 n; 183 | for (n=0; n> 4; 185 | huffWeight[n+1] = ip[n/2] & 15; 186 | } } } 187 | else { /* header compressed with FSE (normal case) */ 188 | if (iSize+1 > srcSize) return ERROR(srcSize_wrong); 189 | oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */ 190 | if (FSE_isError(oSize)) return oSize; 191 | } 192 | 193 | /* collect weight stats */ 194 | memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32)); 195 | weightTotal = 0; 196 | { U32 n; for (n=0; n= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); 198 | rankStats[huffWeight[n]]++; 199 | weightTotal += (1 << huffWeight[n]) >> 1; 200 | } } 201 | 202 | /* get last non-null symbol weight (implied, total must be 2^n) */ 203 | { U32 const tableLog = BIT_highbit32(weightTotal) + 1; 204 | if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected); 205 | *tableLogPtr = tableLog; 206 | /* determine last weight */ 207 | { U32 const total = 1 << tableLog; 208 | U32 const rest = total - weightTotal; 209 | U32 const verif = 1 << BIT_highbit32(rest); 210 | U32 const lastWeight = BIT_highbit32(rest) + 1; 211 | if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ 212 | huffWeight[oSize] = (BYTE)lastWeight; 213 | rankStats[lastWeight]++; 214 | } } 215 | 216 | /* check tree construction validity */ 217 | if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ 218 | 219 | /* results */ 220 | *nbSymbolsPtr = (U32)(oSize+1); 221 | return iSize+1; 222 | } 223 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/common/zbuff.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | /* *************************************************************** 11 | * NOTES/WARNINGS 12 | *****************************************************************/ 13 | /* The streaming API defined here will soon be deprecated by the 14 | * new one in 'zstd.h'; consider migrating towards newer streaming 15 | * API. See 'lib/README.md'. 16 | *****************************************************************/ 17 | 18 | #ifndef ZSTD_BUFFERED_H_23987 19 | #define ZSTD_BUFFERED_H_23987 20 | 21 | #if defined (__cplusplus) 22 | extern "C" { 23 | #endif 24 | 25 | /* ************************************* 26 | * Dependencies 27 | ***************************************/ 28 | #include /* size_t */ 29 | 30 | 31 | /* *************************************************************** 32 | * Compiler specifics 33 | *****************************************************************/ 34 | /* ZSTD_DLL_EXPORT : 35 | * Enable exporting of functions when building a Windows DLL */ 36 | #if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) 37 | # define ZSTDLIB_API __declspec(dllexport) 38 | #else 39 | # define ZSTDLIB_API 40 | #endif 41 | 42 | 43 | /* ************************************* 44 | * Streaming functions 45 | ***************************************/ 46 | /* This is the easier "buffered" streaming API, 47 | * using an internal buffer to lift all restrictions on user-provided buffers 48 | * which can be any size, any place, for both input and output. 49 | * ZBUFF and ZSTD are 100% interoperable, 50 | * frames created by one can be decoded by the other one */ 51 | 52 | typedef struct ZBUFF_CCtx_s ZBUFF_CCtx; 53 | ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void); 54 | ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx); 55 | 56 | ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel); 57 | ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); 58 | 59 | ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr); 60 | ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); 61 | ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr); 62 | 63 | /*-************************************************* 64 | * Streaming compression - howto 65 | * 66 | * A ZBUFF_CCtx object is required to track streaming operation. 67 | * Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. 68 | * ZBUFF_CCtx objects can be reused multiple times. 69 | * 70 | * Start by initializing ZBUF_CCtx. 71 | * Use ZBUFF_compressInit() to start a new compression operation. 72 | * Use ZBUFF_compressInitDictionary() for a compression which requires a dictionary. 73 | * 74 | * Use ZBUFF_compressContinue() repetitively to consume input stream. 75 | * *srcSizePtr and *dstCapacityPtr can be any size. 76 | * The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr. 77 | * Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data. 78 | * The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst . 79 | * @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) 80 | * or an error code, which can be tested using ZBUFF_isError(). 81 | * 82 | * At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush(). 83 | * The nb of bytes written into `dst` will be reported into *dstCapacityPtr. 84 | * Note that the function cannot output more than *dstCapacityPtr, 85 | * therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small. 86 | * @return : nb of bytes still present into internal buffer (0 if it's empty) 87 | * or an error code, which can be tested using ZBUFF_isError(). 88 | * 89 | * ZBUFF_compressEnd() instructs to finish a frame. 90 | * It will perform a flush and write frame epilogue. 91 | * The epilogue is required for decoders to consider a frame completed. 92 | * Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. 93 | * In which case, call again ZBUFF_compressFlush() to complete the flush. 94 | * @return : nb of bytes still present into internal buffer (0 if it's empty) 95 | * or an error code, which can be tested using ZBUFF_isError(). 96 | * 97 | * Hint : _recommended buffer_ sizes (not compulsory) : ZBUFF_recommendedCInSize() / ZBUFF_recommendedCOutSize() 98 | * input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, use this value to reduce intermediate stages (better latency) 99 | * output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering. 100 | * By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering. 101 | * **************************************************/ 102 | 103 | 104 | typedef struct ZBUFF_DCtx_s ZBUFF_DCtx; 105 | ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void); 106 | ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx); 107 | 108 | ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx); 109 | ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize); 110 | 111 | ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx, 112 | void* dst, size_t* dstCapacityPtr, 113 | const void* src, size_t* srcSizePtr); 114 | 115 | /*-*************************************************************************** 116 | * Streaming decompression howto 117 | * 118 | * A ZBUFF_DCtx object is required to track streaming operations. 119 | * Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. 120 | * Use ZBUFF_decompressInit() to start a new decompression operation, 121 | * or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. 122 | * Note that ZBUFF_DCtx objects can be re-init multiple times. 123 | * 124 | * Use ZBUFF_decompressContinue() repetitively to consume your input. 125 | * *srcSizePtr and *dstCapacityPtr can be any size. 126 | * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. 127 | * Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. 128 | * The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`. 129 | * @return : 0 when a frame is completely decoded and fully flushed, 130 | * 1 when there is still some data left within internal buffer to flush, 131 | * >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency), 132 | * or an error code, which can be tested using ZBUFF_isError(). 133 | * 134 | * Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() 135 | * output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. 136 | * input : ZBUFF_recommendedDInSize == 128KB + 3; 137 | * just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . 138 | * *******************************************************************************/ 139 | 140 | 141 | /* ************************************* 142 | * Tool functions 143 | ***************************************/ 144 | ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode); 145 | ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode); 146 | 147 | /** Functions below provide recommended buffer sizes for Compression or Decompression operations. 148 | * These sizes are just hints, they tend to offer better latency */ 149 | ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void); 150 | ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void); 151 | ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void); 152 | ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void); 153 | 154 | 155 | #ifdef ZBUFF_STATIC_LINKING_ONLY 156 | 157 | /* ==================================================================================== 158 | * The definitions in this section are considered experimental. 159 | * They should never be used in association with a dynamic library, as they may change in the future. 160 | * They are provided for advanced usages. 161 | * Use them only in association with static linking. 162 | * ==================================================================================== */ 163 | 164 | /*--- Dependency ---*/ 165 | #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */ 166 | #include "zstd.h" 167 | 168 | 169 | /*--- Custom memory allocator ---*/ 170 | /*! ZBUFF_createCCtx_advanced() : 171 | * Create a ZBUFF compression context using external alloc and free functions */ 172 | ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem); 173 | 174 | /*! ZBUFF_createDCtx_advanced() : 175 | * Create a ZBUFF decompression context using external alloc and free functions */ 176 | ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); 177 | 178 | 179 | /*--- Advanced Streaming Initialization ---*/ 180 | ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, 181 | const void* dict, size_t dictSize, 182 | ZSTD_parameters params, unsigned long long pledgedSrcSize); 183 | 184 | #endif /* ZBUFF_STATIC_LINKING_ONLY */ 185 | 186 | 187 | #if defined (__cplusplus) 188 | } 189 | #endif 190 | 191 | #endif /* ZSTD_BUFFERED_H_23987 */ 192 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/legacy/zstd_legacy.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef ZSTD_LEGACY_H 11 | #define ZSTD_LEGACY_H 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | /* ************************************* 18 | * Includes 19 | ***************************************/ 20 | #include "mem.h" /* MEM_STATIC */ 21 | #include "error_private.h" /* ERROR */ 22 | #include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer */ 23 | #include "zstd_v01.h" 24 | #include "zstd_v02.h" 25 | #include "zstd_v03.h" 26 | #include "zstd_v04.h" 27 | #include "zstd_v05.h" 28 | #include "zstd_v06.h" 29 | #include "zstd_v07.h" 30 | 31 | 32 | /** ZSTD_isLegacy() : 33 | @return : > 0 if supported by legacy decoder. 0 otherwise. 34 | return value is the version. 35 | */ 36 | MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize) 37 | { 38 | U32 magicNumberLE; 39 | if (srcSize<4) return 0; 40 | magicNumberLE = MEM_readLE32(src); 41 | switch(magicNumberLE) 42 | { 43 | case ZSTDv01_magicNumberLE:return 1; 44 | case ZSTDv02_magicNumber : return 2; 45 | case ZSTDv03_magicNumber : return 3; 46 | case ZSTDv04_magicNumber : return 4; 47 | case ZSTDv05_MAGICNUMBER : return 5; 48 | case ZSTDv06_MAGICNUMBER : return 6; 49 | case ZSTDv07_MAGICNUMBER : return 7; 50 | default : return 0; 51 | } 52 | } 53 | 54 | 55 | MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize) 56 | { 57 | U32 const version = ZSTD_isLegacy(src, srcSize); 58 | if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ 59 | if (version==5) { 60 | ZSTDv05_parameters fParams; 61 | size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); 62 | if (frResult != 0) return 0; 63 | return fParams.srcSize; 64 | } 65 | if (version==6) { 66 | ZSTDv06_frameParams fParams; 67 | size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); 68 | if (frResult != 0) return 0; 69 | return fParams.frameContentSize; 70 | } 71 | if (version==7) { 72 | ZSTDv07_frameParams fParams; 73 | size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); 74 | if (frResult != 0) return 0; 75 | return fParams.frameContentSize; 76 | } 77 | return 0; /* should not be possible */ 78 | } 79 | 80 | 81 | MEM_STATIC size_t ZSTD_decompressLegacy( 82 | void* dst, size_t dstCapacity, 83 | const void* src, size_t compressedSize, 84 | const void* dict,size_t dictSize) 85 | { 86 | U32 const version = ZSTD_isLegacy(src, compressedSize); 87 | switch(version) 88 | { 89 | case 1 : 90 | return ZSTDv01_decompress(dst, dstCapacity, src, compressedSize); 91 | case 2 : 92 | return ZSTDv02_decompress(dst, dstCapacity, src, compressedSize); 93 | case 3 : 94 | return ZSTDv03_decompress(dst, dstCapacity, src, compressedSize); 95 | case 4 : 96 | return ZSTDv04_decompress(dst, dstCapacity, src, compressedSize); 97 | case 5 : 98 | { size_t result; 99 | ZSTDv05_DCtx* const zd = ZSTDv05_createDCtx(); 100 | if (zd==NULL) return ERROR(memory_allocation); 101 | result = ZSTDv05_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); 102 | ZSTDv05_freeDCtx(zd); 103 | return result; 104 | } 105 | case 6 : 106 | { size_t result; 107 | ZSTDv06_DCtx* const zd = ZSTDv06_createDCtx(); 108 | if (zd==NULL) return ERROR(memory_allocation); 109 | result = ZSTDv06_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); 110 | ZSTDv06_freeDCtx(zd); 111 | return result; 112 | } 113 | case 7 : 114 | { size_t result; 115 | ZSTDv07_DCtx* const zd = ZSTDv07_createDCtx(); 116 | if (zd==NULL) return ERROR(memory_allocation); 117 | result = ZSTDv07_decompress_usingDict(zd, dst, dstCapacity, src, compressedSize, dict, dictSize); 118 | ZSTDv07_freeDCtx(zd); 119 | return result; 120 | } 121 | default : 122 | return ERROR(prefix_unknown); 123 | } 124 | } 125 | 126 | 127 | MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) 128 | { 129 | switch(version) 130 | { 131 | default : 132 | case 1 : 133 | case 2 : 134 | case 3 : 135 | return ERROR(version_unsupported); 136 | case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext); 137 | case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext); 138 | case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext); 139 | case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext); 140 | } 141 | } 142 | 143 | 144 | MEM_STATIC size_t ZSTD_initLegacyStream(void** legacyContext, U32 prevVersion, U32 newVersion, 145 | const void* dict, size_t dictSize) 146 | { 147 | if (prevVersion != newVersion) ZSTD_freeLegacyStreamContext(*legacyContext, prevVersion); 148 | switch(newVersion) 149 | { 150 | default : 151 | case 1 : 152 | case 2 : 153 | case 3 : 154 | return 0; 155 | case 4 : 156 | { 157 | ZBUFFv04_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv04_createDCtx() : (ZBUFFv04_DCtx*)*legacyContext; 158 | if (dctx==NULL) return ERROR(memory_allocation); 159 | ZBUFFv04_decompressInit(dctx); 160 | ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize); 161 | *legacyContext = dctx; 162 | return 0; 163 | } 164 | case 5 : 165 | { 166 | ZBUFFv05_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv05_createDCtx() : (ZBUFFv05_DCtx*)*legacyContext; 167 | if (dctx==NULL) return ERROR(memory_allocation); 168 | ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize); 169 | *legacyContext = dctx; 170 | return 0; 171 | } 172 | case 6 : 173 | { 174 | ZBUFFv06_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv06_createDCtx() : (ZBUFFv06_DCtx*)*legacyContext; 175 | if (dctx==NULL) return ERROR(memory_allocation); 176 | ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize); 177 | *legacyContext = dctx; 178 | return 0; 179 | } 180 | case 7 : 181 | { 182 | ZBUFFv07_DCtx* dctx = (prevVersion != newVersion) ? ZBUFFv07_createDCtx() : (ZBUFFv07_DCtx*)*legacyContext; 183 | if (dctx==NULL) return ERROR(memory_allocation); 184 | ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize); 185 | *legacyContext = dctx; 186 | return 0; 187 | } 188 | } 189 | } 190 | 191 | 192 | 193 | MEM_STATIC size_t ZSTD_decompressLegacyStream(void* legacyContext, U32 version, 194 | ZSTD_outBuffer* output, ZSTD_inBuffer* input) 195 | { 196 | switch(version) 197 | { 198 | default : 199 | case 1 : 200 | case 2 : 201 | case 3 : 202 | return ERROR(version_unsupported); 203 | case 4 : 204 | { 205 | ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; 206 | const void* src = (const char*)input->src + input->pos; 207 | size_t readSize = input->size - input->pos; 208 | void* dst = (char*)output->dst + output->pos; 209 | size_t decodedSize = output->size - output->pos; 210 | size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize); 211 | output->pos += decodedSize; 212 | input->pos += readSize; 213 | return hintSize; 214 | } 215 | case 5 : 216 | { 217 | ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; 218 | const void* src = (const char*)input->src + input->pos; 219 | size_t readSize = input->size - input->pos; 220 | void* dst = (char*)output->dst + output->pos; 221 | size_t decodedSize = output->size - output->pos; 222 | size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize); 223 | output->pos += decodedSize; 224 | input->pos += readSize; 225 | return hintSize; 226 | } 227 | case 6 : 228 | { 229 | ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; 230 | const void* src = (const char*)input->src + input->pos; 231 | size_t readSize = input->size - input->pos; 232 | void* dst = (char*)output->dst + output->pos; 233 | size_t decodedSize = output->size - output->pos; 234 | size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize); 235 | output->pos += decodedSize; 236 | input->pos += readSize; 237 | return hintSize; 238 | } 239 | case 7 : 240 | { 241 | ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; 242 | const void* src = (const char*)input->src + input->pos; 243 | size_t readSize = input->size - input->pos; 244 | void* dst = (char*)output->dst + output->pos; 245 | size_t decodedSize = output->size - output->pos; 246 | size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize); 247 | output->pos += decodedSize; 248 | input->pos += readSize; 249 | return hintSize; 250 | } 251 | } 252 | } 253 | 254 | 255 | #if defined (__cplusplus) 256 | } 257 | #endif 258 | 259 | #endif /* ZSTD_LEGACY_H */ 260 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/decompress/zbuff_decompress.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | 11 | 12 | /* ************************************* 13 | * Dependencies 14 | ***************************************/ 15 | #include 16 | #include "error_private.h" 17 | #include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */ 18 | #define ZBUFF_STATIC_LINKING_ONLY 19 | #include "zbuff.h" 20 | 21 | 22 | typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, 23 | ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; 24 | 25 | /* *** Resource management *** */ 26 | struct ZBUFF_DCtx_s { 27 | ZSTD_DCtx* zd; 28 | ZSTD_frameParams fParams; 29 | ZBUFF_dStage stage; 30 | char* inBuff; 31 | size_t inBuffSize; 32 | size_t inPos; 33 | char* outBuff; 34 | size_t outBuffSize; 35 | size_t outStart; 36 | size_t outEnd; 37 | size_t blockSize; 38 | BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; 39 | size_t lhSize; 40 | ZSTD_customMem customMem; 41 | }; /* typedef'd to ZBUFF_DCtx within "zbuff.h" */ 42 | 43 | 44 | ZBUFF_DCtx* ZBUFF_createDCtx(void) 45 | { 46 | return ZBUFF_createDCtx_advanced(defaultCustomMem); 47 | } 48 | 49 | ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) 50 | { 51 | ZBUFF_DCtx* zbd; 52 | 53 | if (!customMem.customAlloc && !customMem.customFree) 54 | customMem = defaultCustomMem; 55 | 56 | if (!customMem.customAlloc || !customMem.customFree) 57 | return NULL; 58 | 59 | zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx)); 60 | if (zbd==NULL) return NULL; 61 | memset(zbd, 0, sizeof(ZBUFF_DCtx)); 62 | memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem)); 63 | zbd->zd = ZSTD_createDCtx_advanced(customMem); 64 | if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; } 65 | zbd->stage = ZBUFFds_init; 66 | return zbd; 67 | } 68 | 69 | size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) 70 | { 71 | if (zbd==NULL) return 0; /* support free on null */ 72 | ZSTD_freeDCtx(zbd->zd); 73 | if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); 74 | if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); 75 | zbd->customMem.customFree(zbd->customMem.opaque, zbd); 76 | return 0; 77 | } 78 | 79 | 80 | /* *** Initialization *** */ 81 | 82 | size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) 83 | { 84 | zbd->stage = ZBUFFds_loadHeader; 85 | zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0; 86 | return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize); 87 | } 88 | 89 | size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd) 90 | { 91 | return ZBUFF_decompressInitDictionary(zbd, NULL, 0); 92 | } 93 | 94 | 95 | /* internal util function */ 96 | MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) 97 | { 98 | size_t const length = MIN(dstCapacity, srcSize); 99 | memcpy(dst, src, length); 100 | return length; 101 | } 102 | 103 | 104 | /* *** Decompression *** */ 105 | 106 | size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, 107 | void* dst, size_t* dstCapacityPtr, 108 | const void* src, size_t* srcSizePtr) 109 | { 110 | const char* const istart = (const char*)src; 111 | const char* const iend = istart + *srcSizePtr; 112 | const char* ip = istart; 113 | char* const ostart = (char*)dst; 114 | char* const oend = ostart + *dstCapacityPtr; 115 | char* op = ostart; 116 | U32 someMoreWork = 1; 117 | 118 | while (someMoreWork) { 119 | switch(zbd->stage) 120 | { 121 | case ZBUFFds_init : 122 | return ERROR(init_missing); 123 | 124 | case ZBUFFds_loadHeader : 125 | { size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); 126 | if (ZSTD_isError(hSize)) return hSize; 127 | if (hSize != 0) { /* need more input */ 128 | size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ 129 | if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ 130 | memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); 131 | zbd->lhSize += iend-ip; 132 | *dstCapacityPtr = 0; 133 | return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ 134 | } 135 | memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad; 136 | break; 137 | } } 138 | 139 | /* Consume header */ 140 | { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */ 141 | size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size); 142 | if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */ 143 | if (h1Size < zbd->lhSize) { /* long header */ 144 | size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); 145 | size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size); 146 | if (ZSTD_isError(h2Result)) return h2Result; 147 | } } 148 | 149 | zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); 150 | 151 | /* Frame header instruct buffer sizes */ 152 | { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); 153 | size_t const neededOutSize = zbd->fParams.windowSize + blockSize; 154 | zbd->blockSize = blockSize; 155 | if (zbd->inBuffSize < blockSize) { 156 | zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); 157 | zbd->inBuffSize = blockSize; 158 | zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); 159 | if (zbd->inBuff == NULL) return ERROR(memory_allocation); 160 | } 161 | if (zbd->outBuffSize < neededOutSize) { 162 | zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); 163 | zbd->outBuffSize = neededOutSize; 164 | zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize); 165 | if (zbd->outBuff == NULL) return ERROR(memory_allocation); 166 | } } 167 | zbd->stage = ZBUFFds_read; 168 | /* pass-through */ 169 | 170 | case ZBUFFds_read: 171 | { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); 172 | if (neededInSize==0) { /* end of frame */ 173 | zbd->stage = ZBUFFds_init; 174 | someMoreWork = 0; 175 | break; 176 | } 177 | if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ 178 | const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); 179 | size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, 180 | zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart), 181 | ip, neededInSize); 182 | if (ZSTD_isError(decodedSize)) return decodedSize; 183 | ip += neededInSize; 184 | if (!decodedSize && !isSkipFrame) break; /* this was just a header */ 185 | zbd->outEnd = zbd->outStart + decodedSize; 186 | zbd->stage = ZBUFFds_flush; 187 | break; 188 | } 189 | if (ip==iend) { someMoreWork = 0; break; } /* no more input */ 190 | zbd->stage = ZBUFFds_load; 191 | /* pass-through */ 192 | } 193 | 194 | case ZBUFFds_load: 195 | { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); 196 | size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */ 197 | size_t loadedSize; 198 | if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */ 199 | loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip); 200 | ip += loadedSize; 201 | zbd->inPos += loadedSize; 202 | if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ 203 | 204 | /* decode loaded input */ 205 | { const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); 206 | size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, 207 | zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, 208 | zbd->inBuff, neededInSize); 209 | if (ZSTD_isError(decodedSize)) return decodedSize; 210 | zbd->inPos = 0; /* input is consumed */ 211 | if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ 212 | zbd->outEnd = zbd->outStart + decodedSize; 213 | zbd->stage = ZBUFFds_flush; 214 | /* pass-through */ 215 | } } 216 | 217 | case ZBUFFds_flush: 218 | { size_t const toFlushSize = zbd->outEnd - zbd->outStart; 219 | size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize); 220 | op += flushedSize; 221 | zbd->outStart += flushedSize; 222 | if (flushedSize == toFlushSize) { /* flush completed */ 223 | zbd->stage = ZBUFFds_read; 224 | if (zbd->outStart + zbd->blockSize > zbd->outBuffSize) 225 | zbd->outStart = zbd->outEnd = 0; 226 | break; 227 | } 228 | /* cannot flush everything */ 229 | someMoreWork = 0; 230 | break; 231 | } 232 | default: return ERROR(GENERIC); /* impossible */ 233 | } } 234 | 235 | /* result */ 236 | *srcSizePtr = ip-istart; 237 | *dstCapacityPtr = op-ostart; 238 | { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd); 239 | if (!nextSrcSizeHint) return (zbd->outEnd != zbd->outStart); /* return 0 only if fully flushed too */ 240 | nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zbd->zd) == ZSTDnit_block); 241 | if (zbd->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ 242 | nextSrcSizeHint -= zbd->inPos; /* already loaded*/ 243 | return nextSrcSizeHint; 244 | } 245 | } 246 | 247 | 248 | /* ************************************* 249 | * Tool functions 250 | ***************************************/ 251 | size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize /* block header size*/ ; } 252 | size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } 253 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/common/huf.h: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | Huffman coder, part of New Generation Entropy library 3 | header file 4 | Copyright (C) 2013-2016, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 33 | ****************************************************************** */ 34 | #ifndef HUF_H_298734234 35 | #define HUF_H_298734234 36 | 37 | #if defined (__cplusplus) 38 | extern "C" { 39 | #endif 40 | 41 | 42 | /* *** Dependencies *** */ 43 | #include /* size_t */ 44 | 45 | 46 | /* *** simple functions *** */ 47 | /** 48 | HUF_compress() : 49 | Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. 50 | 'dst' buffer must be already allocated. 51 | Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). 52 | `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. 53 | @return : size of compressed data (<= `dstCapacity`). 54 | Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! 55 | if return == 1, srcData is a single repeated byte symbol (RLE compression). 56 | if HUF_isError(return), compression failed (more details using HUF_getErrorName()) 57 | */ 58 | size_t HUF_compress(void* dst, size_t dstCapacity, 59 | const void* src, size_t srcSize); 60 | 61 | /** 62 | HUF_decompress() : 63 | Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', 64 | into already allocated buffer 'dst', of minimum size 'dstSize'. 65 | `dstSize` : **must** be the ***exact*** size of original (uncompressed) data. 66 | Note : in contrast with FSE, HUF_decompress can regenerate 67 | RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, 68 | because it knows size to regenerate. 69 | @return : size of regenerated data (== dstSize), 70 | or an error code, which can be tested using HUF_isError() 71 | */ 72 | size_t HUF_decompress(void* dst, size_t dstSize, 73 | const void* cSrc, size_t cSrcSize); 74 | 75 | 76 | /* **************************************** 77 | * Tool functions 78 | ******************************************/ 79 | #define HUF_BLOCKSIZE_MAX (128 * 1024) 80 | size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ 81 | 82 | /* Error Management */ 83 | unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ 84 | const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ 85 | 86 | 87 | /* *** Advanced function *** */ 88 | 89 | /** HUF_compress2() : 90 | * Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` */ 91 | size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); 92 | 93 | 94 | #ifdef HUF_STATIC_LINKING_ONLY 95 | 96 | /* *** Dependencies *** */ 97 | #include "mem.h" /* U32 */ 98 | 99 | 100 | /* *** Constants *** */ 101 | #define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ 102 | #define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ 103 | #define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ 104 | #define HUF_SYMBOLVALUE_MAX 255 105 | #if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) 106 | # error "HUF_TABLELOG_MAX is too large !" 107 | #endif 108 | 109 | 110 | /* **************************************** 111 | * Static allocation 112 | ******************************************/ 113 | /* HUF buffer bounds */ 114 | #define HUF_CTABLEBOUND 129 115 | #define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ 116 | #define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ 117 | 118 | /* static allocation of HUF's Compression Table */ 119 | #define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ 120 | U32 name##hb[maxSymbolValue+1]; \ 121 | void* name##hv = &(name##hb); \ 122 | HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ 123 | 124 | /* static allocation of HUF's DTable */ 125 | typedef U32 HUF_DTable; 126 | #define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) 127 | #define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ 128 | HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) } 129 | #define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ 130 | HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) } 131 | 132 | 133 | /* **************************************** 134 | * Advanced decompression functions 135 | ******************************************/ 136 | size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ 137 | size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ 138 | 139 | size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ 140 | size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ 141 | size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ 142 | size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ 143 | 144 | size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); 145 | size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ 146 | size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ 147 | 148 | 149 | /* **************************************** 150 | * HUF detailed API 151 | ******************************************/ 152 | /*! 153 | HUF_compress() does the following: 154 | 1. count symbol occurrence from source[] into table count[] using FSE_count() 155 | 2. (optional) refine tableLog using HUF_optimalTableLog() 156 | 3. build Huffman table from count using HUF_buildCTable() 157 | 4. save Huffman table to memory buffer using HUF_writeCTable() 158 | 5. encode the data stream using HUF_compress4X_usingCTable() 159 | 160 | The following API allows targeting specific sub-functions for advanced tasks. 161 | For example, it's possible to compress several blocks using the same 'CTable', 162 | or to save and regenerate 'CTable' using external methods. 163 | */ 164 | /* FSE_count() : find it within "fse.h" */ 165 | unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); 166 | typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ 167 | size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); 168 | size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); 169 | size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); 170 | 171 | 172 | /*! HUF_readStats() : 173 | Read compact Huffman tree, saved by HUF_writeCTable(). 174 | `huffWeight` is destination buffer. 175 | @return : size read from `src` , or an error Code . 176 | Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ 177 | size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, 178 | U32* nbSymbolsPtr, U32* tableLogPtr, 179 | const void* src, size_t srcSize); 180 | 181 | /** HUF_readCTable() : 182 | * Loading a CTable saved with HUF_writeCTable() */ 183 | size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); 184 | 185 | 186 | /* 187 | HUF_decompress() does the following: 188 | 1. select the decompression algorithm (X2, X4) based on pre-computed heuristics 189 | 2. build Huffman table from save, using HUF_readDTableXn() 190 | 3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable 191 | */ 192 | 193 | /** HUF_selectDecoder() : 194 | * Tells which decoder is likely to decode faster, 195 | * based on a set of pre-determined metrics. 196 | * @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . 197 | * Assumption : 0 < cSrcSize < dstSize <= 128 KB */ 198 | U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); 199 | 200 | size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); 201 | size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize); 202 | 203 | size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); 204 | size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); 205 | size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); 206 | 207 | 208 | /* single stream variants */ 209 | 210 | size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); 211 | size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); 212 | 213 | size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ 214 | size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ 215 | 216 | size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); 217 | size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); 218 | size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); 219 | 220 | 221 | #endif /* HUF_STATIC_LINKING_ONLY */ 222 | 223 | 224 | #if defined (__cplusplus) 225 | } 226 | #endif 227 | 228 | #endif /* HUF_H_298734234 */ 229 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/common/mem.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #ifndef MEM_H_MODULE 11 | #define MEM_H_MODULE 12 | 13 | #if defined (__cplusplus) 14 | extern "C" { 15 | #endif 16 | 17 | /*-**************************************** 18 | * Dependencies 19 | ******************************************/ 20 | #include /* size_t, ptrdiff_t */ 21 | #include /* memcpy */ 22 | 23 | 24 | /*-**************************************** 25 | * Compiler specifics 26 | ******************************************/ 27 | #if defined(_MSC_VER) /* Visual Studio */ 28 | # include /* _byteswap_ulong */ 29 | # include /* _byteswap_* */ 30 | #endif 31 | #if defined(__GNUC__) 32 | # define MEM_STATIC static __inline __attribute__((unused)) 33 | #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 34 | # define MEM_STATIC static inline 35 | #elif defined(_MSC_VER) 36 | # define MEM_STATIC static __inline 37 | #else 38 | # define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ 39 | #endif 40 | 41 | /* code only tested on 32 and 64 bits systems */ 42 | #define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } 43 | MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } 44 | 45 | 46 | /*-************************************************************** 47 | * Basic Types 48 | *****************************************************************/ 49 | #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) 50 | # include 51 | typedef uint8_t BYTE; 52 | typedef uint16_t U16; 53 | typedef int16_t S16; 54 | typedef uint32_t U32; 55 | typedef int32_t S32; 56 | typedef uint64_t U64; 57 | typedef int64_t S64; 58 | #else 59 | typedef unsigned char BYTE; 60 | typedef unsigned short U16; 61 | typedef signed short S16; 62 | typedef unsigned int U32; 63 | typedef signed int S32; 64 | typedef unsigned long long U64; 65 | typedef signed long long S64; 66 | #endif 67 | 68 | 69 | /*-************************************************************** 70 | * Memory I/O 71 | *****************************************************************/ 72 | /* MEM_FORCE_MEMORY_ACCESS : 73 | * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. 74 | * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. 75 | * The below switch allow to select different access method for improved performance. 76 | * Method 0 (default) : use `memcpy()`. Safe and portable. 77 | * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). 78 | * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. 79 | * Method 2 : direct access. This method is portable but violate C standard. 80 | * It can generate buggy code on targets depending on alignment. 81 | * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) 82 | * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. 83 | * Prefer these methods in priority order (0 > 1 > 2) 84 | */ 85 | #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ 86 | # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) 87 | # define MEM_FORCE_MEMORY_ACCESS 2 88 | # elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \ 89 | (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) 90 | # define MEM_FORCE_MEMORY_ACCESS 1 91 | # endif 92 | #endif 93 | 94 | MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } 95 | MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } 96 | 97 | MEM_STATIC unsigned MEM_isLittleEndian(void) 98 | { 99 | const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ 100 | return one.c[0]; 101 | } 102 | 103 | #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) 104 | 105 | /* violates C standard, by lying on structure alignment. 106 | Only use if no other choice to achieve best performance on target platform */ 107 | MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } 108 | MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } 109 | MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } 110 | MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } 111 | 112 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } 113 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } 114 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } 115 | 116 | #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) 117 | 118 | /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ 119 | /* currently only defined for gcc and icc */ 120 | #if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) 121 | __pragma( pack(push, 1) ) 122 | typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign; 123 | __pragma( pack(pop) ) 124 | #else 125 | typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; 126 | #endif 127 | 128 | MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } 129 | MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } 130 | MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } 131 | MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } 132 | 133 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } 134 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } 135 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } 136 | 137 | #else 138 | 139 | /* default method, safe and standard. 140 | can sometimes prove slower */ 141 | 142 | MEM_STATIC U16 MEM_read16(const void* memPtr) 143 | { 144 | U16 val; memcpy(&val, memPtr, sizeof(val)); return val; 145 | } 146 | 147 | MEM_STATIC U32 MEM_read32(const void* memPtr) 148 | { 149 | U32 val; memcpy(&val, memPtr, sizeof(val)); return val; 150 | } 151 | 152 | MEM_STATIC U64 MEM_read64(const void* memPtr) 153 | { 154 | U64 val; memcpy(&val, memPtr, sizeof(val)); return val; 155 | } 156 | 157 | MEM_STATIC size_t MEM_readST(const void* memPtr) 158 | { 159 | size_t val; memcpy(&val, memPtr, sizeof(val)); return val; 160 | } 161 | 162 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) 163 | { 164 | memcpy(memPtr, &value, sizeof(value)); 165 | } 166 | 167 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) 168 | { 169 | memcpy(memPtr, &value, sizeof(value)); 170 | } 171 | 172 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) 173 | { 174 | memcpy(memPtr, &value, sizeof(value)); 175 | } 176 | 177 | #endif /* MEM_FORCE_MEMORY_ACCESS */ 178 | 179 | MEM_STATIC U32 MEM_swap32(U32 in) 180 | { 181 | #if defined(_MSC_VER) /* Visual Studio */ 182 | return _byteswap_ulong(in); 183 | #elif defined (__GNUC__) 184 | return __builtin_bswap32(in); 185 | #else 186 | return ((in << 24) & 0xff000000 ) | 187 | ((in << 8) & 0x00ff0000 ) | 188 | ((in >> 8) & 0x0000ff00 ) | 189 | ((in >> 24) & 0x000000ff ); 190 | #endif 191 | } 192 | 193 | MEM_STATIC U64 MEM_swap64(U64 in) 194 | { 195 | #if defined(_MSC_VER) /* Visual Studio */ 196 | return _byteswap_uint64(in); 197 | #elif defined (__GNUC__) 198 | return __builtin_bswap64(in); 199 | #else 200 | return ((in << 56) & 0xff00000000000000ULL) | 201 | ((in << 40) & 0x00ff000000000000ULL) | 202 | ((in << 24) & 0x0000ff0000000000ULL) | 203 | ((in << 8) & 0x000000ff00000000ULL) | 204 | ((in >> 8) & 0x00000000ff000000ULL) | 205 | ((in >> 24) & 0x0000000000ff0000ULL) | 206 | ((in >> 40) & 0x000000000000ff00ULL) | 207 | ((in >> 56) & 0x00000000000000ffULL); 208 | #endif 209 | } 210 | 211 | MEM_STATIC size_t MEM_swapST(size_t in) 212 | { 213 | if (MEM_32bits()) 214 | return (size_t)MEM_swap32((U32)in); 215 | else 216 | return (size_t)MEM_swap64((U64)in); 217 | } 218 | 219 | /*=== Little endian r/w ===*/ 220 | 221 | MEM_STATIC U16 MEM_readLE16(const void* memPtr) 222 | { 223 | if (MEM_isLittleEndian()) 224 | return MEM_read16(memPtr); 225 | else { 226 | const BYTE* p = (const BYTE*)memPtr; 227 | return (U16)(p[0] + (p[1]<<8)); 228 | } 229 | } 230 | 231 | MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) 232 | { 233 | if (MEM_isLittleEndian()) { 234 | MEM_write16(memPtr, val); 235 | } else { 236 | BYTE* p = (BYTE*)memPtr; 237 | p[0] = (BYTE)val; 238 | p[1] = (BYTE)(val>>8); 239 | } 240 | } 241 | 242 | MEM_STATIC U32 MEM_readLE24(const void* memPtr) 243 | { 244 | return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); 245 | } 246 | 247 | MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) 248 | { 249 | MEM_writeLE16(memPtr, (U16)val); 250 | ((BYTE*)memPtr)[2] = (BYTE)(val>>16); 251 | } 252 | 253 | MEM_STATIC U32 MEM_readLE32(const void* memPtr) 254 | { 255 | if (MEM_isLittleEndian()) 256 | return MEM_read32(memPtr); 257 | else 258 | return MEM_swap32(MEM_read32(memPtr)); 259 | } 260 | 261 | MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) 262 | { 263 | if (MEM_isLittleEndian()) 264 | MEM_write32(memPtr, val32); 265 | else 266 | MEM_write32(memPtr, MEM_swap32(val32)); 267 | } 268 | 269 | MEM_STATIC U64 MEM_readLE64(const void* memPtr) 270 | { 271 | if (MEM_isLittleEndian()) 272 | return MEM_read64(memPtr); 273 | else 274 | return MEM_swap64(MEM_read64(memPtr)); 275 | } 276 | 277 | MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) 278 | { 279 | if (MEM_isLittleEndian()) 280 | MEM_write64(memPtr, val64); 281 | else 282 | MEM_write64(memPtr, MEM_swap64(val64)); 283 | } 284 | 285 | MEM_STATIC size_t MEM_readLEST(const void* memPtr) 286 | { 287 | if (MEM_32bits()) 288 | return (size_t)MEM_readLE32(memPtr); 289 | else 290 | return (size_t)MEM_readLE64(memPtr); 291 | } 292 | 293 | MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) 294 | { 295 | if (MEM_32bits()) 296 | MEM_writeLE32(memPtr, (U32)val); 297 | else 298 | MEM_writeLE64(memPtr, (U64)val); 299 | } 300 | 301 | /*=== Big endian r/w ===*/ 302 | 303 | MEM_STATIC U32 MEM_readBE32(const void* memPtr) 304 | { 305 | if (MEM_isLittleEndian()) 306 | return MEM_swap32(MEM_read32(memPtr)); 307 | else 308 | return MEM_read32(memPtr); 309 | } 310 | 311 | MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) 312 | { 313 | if (MEM_isLittleEndian()) 314 | MEM_write32(memPtr, MEM_swap32(val32)); 315 | else 316 | MEM_write32(memPtr, val32); 317 | } 318 | 319 | MEM_STATIC U64 MEM_readBE64(const void* memPtr) 320 | { 321 | if (MEM_isLittleEndian()) 322 | return MEM_swap64(MEM_read64(memPtr)); 323 | else 324 | return MEM_read64(memPtr); 325 | } 326 | 327 | MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) 328 | { 329 | if (MEM_isLittleEndian()) 330 | MEM_write64(memPtr, MEM_swap64(val64)); 331 | else 332 | MEM_write64(memPtr, val64); 333 | } 334 | 335 | MEM_STATIC size_t MEM_readBEST(const void* memPtr) 336 | { 337 | if (MEM_32bits()) 338 | return (size_t)MEM_readBE32(memPtr); 339 | else 340 | return (size_t)MEM_readBE64(memPtr); 341 | } 342 | 343 | MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) 344 | { 345 | if (MEM_32bits()) 346 | MEM_writeBE32(memPtr, (U32)val); 347 | else 348 | MEM_writeBE64(memPtr, (U64)val); 349 | } 350 | 351 | 352 | /* function safe only for comparisons */ 353 | MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) 354 | { 355 | switch (length) 356 | { 357 | default : 358 | case 4 : return MEM_read32(memPtr); 359 | case 3 : if (MEM_isLittleEndian()) 360 | return MEM_read32(memPtr)<<8; 361 | else 362 | return MEM_read32(memPtr)>>8; 363 | } 364 | } 365 | 366 | #if defined (__cplusplus) 367 | } 368 | #endif 369 | 370 | #endif /* MEM_H_MODULE */ 371 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/common/fse_decompress.c: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | FSE : Finite State Entropy decoder 3 | Copyright (C) 2013-2015, Yann Collet. 4 | 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy 32 | - Public forum : https://groups.google.com/forum/#!forum/lz4c 33 | ****************************************************************** */ 34 | 35 | 36 | /* ************************************************************** 37 | * Compiler specifics 38 | ****************************************************************/ 39 | #ifdef _MSC_VER /* Visual Studio */ 40 | # define FORCE_INLINE static __forceinline 41 | # include /* For Visual 2005 */ 42 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 43 | # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ 44 | #else 45 | # ifdef __GNUC__ 46 | # define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) 47 | # define FORCE_INLINE static inline __attribute__((always_inline)) 48 | # else 49 | # define FORCE_INLINE static inline 50 | # endif 51 | #endif 52 | 53 | 54 | /* ************************************************************** 55 | * Includes 56 | ****************************************************************/ 57 | #include /* malloc, free, qsort */ 58 | #include /* memcpy, memset */ 59 | #include /* printf (debug) */ 60 | #include "bitstream.h" 61 | #define FSE_STATIC_LINKING_ONLY 62 | #include "fse.h" 63 | 64 | 65 | /* ************************************************************** 66 | * Error Management 67 | ****************************************************************/ 68 | #define FSE_isError ERR_isError 69 | #define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ 70 | 71 | /* check and forward error code */ 72 | #define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } 73 | 74 | 75 | /* ************************************************************** 76 | * Complex types 77 | ****************************************************************/ 78 | typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; 79 | 80 | 81 | /* ************************************************************** 82 | * Templates 83 | ****************************************************************/ 84 | /* 85 | designed to be included 86 | for type-specific functions (template emulation in C) 87 | Objective is to write these functions only once, for improved maintenance 88 | */ 89 | 90 | /* safety checks */ 91 | #ifndef FSE_FUNCTION_EXTENSION 92 | # error "FSE_FUNCTION_EXTENSION must be defined" 93 | #endif 94 | #ifndef FSE_FUNCTION_TYPE 95 | # error "FSE_FUNCTION_TYPE must be defined" 96 | #endif 97 | 98 | /* Function names */ 99 | #define FSE_CAT(X,Y) X##Y 100 | #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) 101 | #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) 102 | 103 | 104 | /* Function templates */ 105 | FSE_DTable* FSE_createDTable (unsigned tableLog) 106 | { 107 | if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; 108 | return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); 109 | } 110 | 111 | void FSE_freeDTable (FSE_DTable* dt) 112 | { 113 | free(dt); 114 | } 115 | 116 | size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) 117 | { 118 | void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ 119 | FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); 120 | U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; 121 | 122 | U32 const maxSV1 = maxSymbolValue + 1; 123 | U32 const tableSize = 1 << tableLog; 124 | U32 highThreshold = tableSize-1; 125 | 126 | /* Sanity Checks */ 127 | if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); 128 | if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); 129 | 130 | /* Init, lay down lowprob symbols */ 131 | { FSE_DTableHeader DTableH; 132 | DTableH.tableLog = (U16)tableLog; 133 | DTableH.fastMode = 1; 134 | { S16 const largeLimit= (S16)(1 << (tableLog-1)); 135 | U32 s; 136 | for (s=0; s= largeLimit) DTableH.fastMode=0; 142 | symbolNext[s] = normalizedCounter[s]; 143 | } } } 144 | memcpy(dt, &DTableH, sizeof(DTableH)); 145 | } 146 | 147 | /* Spread symbols */ 148 | { U32 const tableMask = tableSize-1; 149 | U32 const step = FSE_TABLESTEP(tableSize); 150 | U32 s, position = 0; 151 | for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ 157 | } } 158 | if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ 159 | } 160 | 161 | /* Build Decoding table */ 162 | { U32 u; 163 | for (u=0; utableLog = 0; 187 | DTableH->fastMode = 0; 188 | 189 | cell->newState = 0; 190 | cell->symbol = symbolValue; 191 | cell->nbBits = 0; 192 | 193 | return 0; 194 | } 195 | 196 | 197 | size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) 198 | { 199 | void* ptr = dt; 200 | FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; 201 | void* dPtr = dt + 1; 202 | FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; 203 | const unsigned tableSize = 1 << nbBits; 204 | const unsigned tableMask = tableSize - 1; 205 | const unsigned maxSV1 = tableMask+1; 206 | unsigned s; 207 | 208 | /* Sanity checks */ 209 | if (nbBits < 1) return ERROR(GENERIC); /* min size */ 210 | 211 | /* Build Decoding Table */ 212 | DTableH->tableLog = (U16)nbBits; 213 | DTableH->fastMode = 1; 214 | for (s=0; s sizeof(bitD.bitContainer)*8) /* This test must be static */ 250 | BIT_reloadDStream(&bitD); 251 | 252 | op[1] = FSE_GETSYMBOL(&state2); 253 | 254 | if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ 255 | { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } 256 | 257 | op[2] = FSE_GETSYMBOL(&state1); 258 | 259 | if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ 260 | BIT_reloadDStream(&bitD); 261 | 262 | op[3] = FSE_GETSYMBOL(&state2); 263 | } 264 | 265 | /* tail */ 266 | /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ 267 | while (1) { 268 | if (op>(omax-2)) return ERROR(dstSize_tooSmall); 269 | *op++ = FSE_GETSYMBOL(&state1); 270 | if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { 271 | *op++ = FSE_GETSYMBOL(&state2); 272 | break; 273 | } 274 | 275 | if (op>(omax-2)) return ERROR(dstSize_tooSmall); 276 | *op++ = FSE_GETSYMBOL(&state2); 277 | if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { 278 | *op++ = FSE_GETSYMBOL(&state1); 279 | break; 280 | } } 281 | 282 | return op-ostart; 283 | } 284 | 285 | 286 | size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, 287 | const void* cSrc, size_t cSrcSize, 288 | const FSE_DTable* dt) 289 | { 290 | const void* ptr = dt; 291 | const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; 292 | const U32 fastMode = DTableH->fastMode; 293 | 294 | /* select fast mode (static) */ 295 | if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); 296 | return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); 297 | } 298 | 299 | 300 | size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize) 301 | { 302 | const BYTE* const istart = (const BYTE*)cSrc; 303 | const BYTE* ip = istart; 304 | short counting[FSE_MAX_SYMBOL_VALUE+1]; 305 | DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ 306 | unsigned tableLog; 307 | unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; 308 | 309 | if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */ 310 | 311 | /* normal FSE decoding mode */ 312 | { size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); 313 | if (FSE_isError(NCountLength)) return NCountLength; 314 | if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */ 315 | ip += NCountLength; 316 | cSrcSize -= NCountLength; 317 | } 318 | 319 | CHECK_F( FSE_buildDTable (dt, counting, maxSymbolValue, tableLog) ); 320 | 321 | return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */ 322 | } 323 | 324 | 325 | 326 | #endif /* FSE_COMMONDEFS_ONLY */ 327 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/common/xxhash.h: -------------------------------------------------------------------------------- 1 | /* 2 | xxHash - Extremely Fast Hash algorithm 3 | Header File 4 | Copyright (C) 2012-2016, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - xxHash source repository : https://github.com/Cyan4973/xxHash 33 | */ 34 | 35 | /* Notice extracted from xxHash homepage : 36 | 37 | xxHash is an extremely fast Hash algorithm, running at RAM speed limits. 38 | It also successfully passes all tests from the SMHasher suite. 39 | 40 | Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) 41 | 42 | Name Speed Q.Score Author 43 | xxHash 5.4 GB/s 10 44 | CrapWow 3.2 GB/s 2 Andrew 45 | MumurHash 3a 2.7 GB/s 10 Austin Appleby 46 | SpookyHash 2.0 GB/s 10 Bob Jenkins 47 | SBox 1.4 GB/s 9 Bret Mulvey 48 | Lookup3 1.2 GB/s 9 Bob Jenkins 49 | SuperFastHash 1.2 GB/s 1 Paul Hsieh 50 | CityHash64 1.05 GB/s 10 Pike & Alakuijala 51 | FNV 0.55 GB/s 5 Fowler, Noll, Vo 52 | CRC32 0.43 GB/s 9 53 | MD5-32 0.33 GB/s 10 Ronald L. Rivest 54 | SHA1-32 0.28 GB/s 10 55 | 56 | Q.Score is a measure of quality of the hash function. 57 | It depends on successfully passing SMHasher test set. 58 | 10 is a perfect score. 59 | 60 | A 64-bits version, named XXH64, is available since r35. 61 | It offers much better speed, but for 64-bits applications only. 62 | Name Speed on 64 bits Speed on 32 bits 63 | XXH64 13.8 GB/s 1.9 GB/s 64 | XXH32 6.8 GB/s 6.0 GB/s 65 | */ 66 | 67 | #ifndef XXHASH_H_5627135585666179 68 | #define XXHASH_H_5627135585666179 1 69 | 70 | #if defined (__cplusplus) 71 | extern "C" { 72 | #endif 73 | 74 | #ifndef XXH_NAMESPACE 75 | # define XXH_NAMESPACE ZSTD_ /* Zstandard specific */ 76 | #endif 77 | 78 | 79 | /* **************************** 80 | * Definitions 81 | ******************************/ 82 | #include /* size_t */ 83 | typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; 84 | 85 | 86 | /* **************************** 87 | * API modifier 88 | ******************************/ 89 | /** XXH_PRIVATE_API 90 | * This is useful if you want to include xxhash functions in `static` mode 91 | * in order to inline them, and remove their symbol from the public list. 92 | * Methodology : 93 | * #define XXH_PRIVATE_API 94 | * #include "xxhash.h" 95 | * `xxhash.c` is automatically included. 96 | * It's not useful to compile and link it as a separate module anymore. 97 | */ 98 | #ifdef XXH_PRIVATE_API 99 | # ifndef XXH_STATIC_LINKING_ONLY 100 | # define XXH_STATIC_LINKING_ONLY 101 | # endif 102 | # if defined(__GNUC__) 103 | # define XXH_PUBLIC_API static __inline __attribute__((unused)) 104 | # elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 105 | # define XXH_PUBLIC_API static inline 106 | # elif defined(_MSC_VER) 107 | # define XXH_PUBLIC_API static __inline 108 | # else 109 | # define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ 110 | # endif 111 | #else 112 | # define XXH_PUBLIC_API /* do nothing */ 113 | #endif /* XXH_PRIVATE_API */ 114 | 115 | /*!XXH_NAMESPACE, aka Namespace Emulation : 116 | 117 | If you want to include _and expose_ xxHash functions from within your own library, 118 | but also want to avoid symbol collisions with another library which also includes xxHash, 119 | 120 | you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library 121 | with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). 122 | 123 | Note that no change is required within the calling program as long as it includes `xxhash.h` : 124 | regular symbol name will be automatically translated by this header. 125 | */ 126 | #ifdef XXH_NAMESPACE 127 | # define XXH_CAT(A,B) A##B 128 | # define XXH_NAME2(A,B) XXH_CAT(A,B) 129 | # define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) 130 | # define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) 131 | # define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) 132 | # define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) 133 | # define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) 134 | # define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) 135 | # define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) 136 | # define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) 137 | # define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) 138 | # define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) 139 | # define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) 140 | # define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) 141 | # define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) 142 | # define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) 143 | # define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) 144 | # define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) 145 | # define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) 146 | # define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) 147 | # define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) 148 | #endif 149 | 150 | 151 | /* ************************************* 152 | * Version 153 | ***************************************/ 154 | #define XXH_VERSION_MAJOR 0 155 | #define XXH_VERSION_MINOR 6 156 | #define XXH_VERSION_RELEASE 2 157 | #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) 158 | XXH_PUBLIC_API unsigned XXH_versionNumber (void); 159 | 160 | 161 | /* **************************** 162 | * Simple Hash Functions 163 | ******************************/ 164 | typedef unsigned int XXH32_hash_t; 165 | typedef unsigned long long XXH64_hash_t; 166 | 167 | XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); 168 | XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); 169 | 170 | /*! 171 | XXH32() : 172 | Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". 173 | The memory between input & input+length must be valid (allocated and read-accessible). 174 | "seed" can be used to alter the result predictably. 175 | Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s 176 | XXH64() : 177 | Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". 178 | "seed" can be used to alter the result predictably. 179 | This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). 180 | */ 181 | 182 | 183 | /* **************************** 184 | * Streaming Hash Functions 185 | ******************************/ 186 | typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ 187 | typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ 188 | 189 | /*! State allocation, compatible with dynamic libraries */ 190 | 191 | XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); 192 | XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); 193 | 194 | XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); 195 | XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); 196 | 197 | 198 | /* hash streaming */ 199 | 200 | XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); 201 | XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); 202 | XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); 203 | 204 | XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); 205 | XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); 206 | XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); 207 | 208 | /* 209 | These functions generate the xxHash of an input provided in multiple segments. 210 | Note that, for small input, they are slower than single-call functions, due to state management. 211 | For small input, prefer `XXH32()` and `XXH64()` . 212 | 213 | XXH state must first be allocated, using XXH*_createState() . 214 | 215 | Start a new hash by initializing state with a seed, using XXH*_reset(). 216 | 217 | Then, feed the hash state by calling XXH*_update() as many times as necessary. 218 | Obviously, input must be allocated and read accessible. 219 | The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. 220 | 221 | Finally, a hash value can be produced anytime, by using XXH*_digest(). 222 | This function returns the nn-bits hash as an int or long long. 223 | 224 | It's still possible to continue inserting input into the hash state after a digest, 225 | and generate some new hashes later on, by calling again XXH*_digest(). 226 | 227 | When done, free XXH state space if it was allocated dynamically. 228 | */ 229 | 230 | 231 | /* ************************** 232 | * Utils 233 | ****************************/ 234 | #if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ 235 | # define restrict /* disable restrict */ 236 | #endif 237 | 238 | XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); 239 | XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); 240 | 241 | 242 | /* ************************** 243 | * Canonical representation 244 | ****************************/ 245 | typedef struct { unsigned char digest[4]; } XXH32_canonical_t; 246 | typedef struct { unsigned char digest[8]; } XXH64_canonical_t; 247 | 248 | XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); 249 | XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); 250 | 251 | XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); 252 | XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); 253 | 254 | /* Default result type for XXH functions are primitive unsigned 32 and 64 bits. 255 | * The canonical representation uses human-readable write convention, aka big-endian (large digits first). 256 | * These functions allow transformation of hash result into and from its canonical format. 257 | * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. 258 | */ 259 | 260 | 261 | #ifdef XXH_STATIC_LINKING_ONLY 262 | 263 | /* ================================================================================================ 264 | This section contains definitions which are not guaranteed to remain stable. 265 | They may change in future versions, becoming incompatible with a different version of the library. 266 | They shall only be used with static linking. 267 | Never use these definitions in association with dynamic linking ! 268 | =================================================================================================== */ 269 | 270 | /* These definitions are only meant to allow allocation of XXH state 271 | statically, on stack, or in a struct for example. 272 | Do not use members directly. */ 273 | 274 | struct XXH32_state_s { 275 | unsigned total_len_32; 276 | unsigned large_len; 277 | unsigned v1; 278 | unsigned v2; 279 | unsigned v3; 280 | unsigned v4; 281 | unsigned mem32[4]; /* buffer defined as U32 for alignment */ 282 | unsigned memsize; 283 | unsigned reserved; /* never read nor write, will be removed in a future version */ 284 | }; /* typedef'd to XXH32_state_t */ 285 | 286 | struct XXH64_state_s { 287 | unsigned long long total_len; 288 | unsigned long long v1; 289 | unsigned long long v2; 290 | unsigned long long v3; 291 | unsigned long long v4; 292 | unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ 293 | unsigned memsize; 294 | unsigned reserved[2]; /* never read nor write, will be removed in a future version */ 295 | }; /* typedef'd to XXH64_state_t */ 296 | 297 | 298 | # ifdef XXH_PRIVATE_API 299 | # include "xxhash.c" /* include xxhash functions as `static`, for inlining */ 300 | # endif 301 | 302 | #endif /* XXH_STATIC_LINKING_ONLY */ 303 | 304 | 305 | #if defined (__cplusplus) 306 | } 307 | #endif 308 | 309 | #endif /* XXHASH_H_5627135585666179 */ 310 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/compress/zbuff_compress.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | 11 | 12 | /* ************************************* 13 | * Dependencies 14 | ***************************************/ 15 | #include 16 | #include "error_private.h" 17 | #include "zstd_internal.h" /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */ 18 | #define ZBUFF_STATIC_LINKING_ONLY 19 | #include "zbuff.h" 20 | 21 | 22 | /* ************************************* 23 | * Constants 24 | ***************************************/ 25 | static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE; 26 | 27 | 28 | /*-*********************************************************** 29 | * Streaming compression 30 | * 31 | * A ZBUFF_CCtx object is required to track streaming operation. 32 | * Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. 33 | * Use ZBUFF_compressInit() to start a new compression operation. 34 | * ZBUFF_CCtx objects can be reused multiple times. 35 | * 36 | * Use ZBUFF_compressContinue() repetitively to consume your input. 37 | * *srcSizePtr and *dstCapacityPtr can be any size. 38 | * The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. 39 | * Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. 40 | * The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst . 41 | * @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) 42 | * or an error code, which can be tested using ZBUFF_isError(). 43 | * 44 | * ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer. 45 | * Note that it will not output more than *dstCapacityPtr. 46 | * Therefore, some content might still be left into its internal buffer if dst buffer is too small. 47 | * @return : nb of bytes still present into internal buffer (0 if it's empty) 48 | * or an error code, which can be tested using ZBUFF_isError(). 49 | * 50 | * ZBUFF_compressEnd() instructs to finish a frame. 51 | * It will perform a flush and write frame epilogue. 52 | * Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. 53 | * @return : nb of bytes still present into internal buffer (0 if it's empty) 54 | * or an error code, which can be tested using ZBUFF_isError(). 55 | * 56 | * Hint : recommended buffer sizes (not compulsory) 57 | * input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value. 58 | * output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed. 59 | * ***********************************************************/ 60 | 61 | typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage; 62 | 63 | /* *** Resources *** */ 64 | struct ZBUFF_CCtx_s { 65 | ZSTD_CCtx* zc; 66 | char* inBuff; 67 | size_t inBuffSize; 68 | size_t inToCompress; 69 | size_t inBuffPos; 70 | size_t inBuffTarget; 71 | size_t blockSize; 72 | char* outBuff; 73 | size_t outBuffSize; 74 | size_t outBuffContentSize; 75 | size_t outBuffFlushedSize; 76 | ZBUFF_cStage stage; 77 | U32 checksum; 78 | U32 frameEnded; 79 | ZSTD_customMem customMem; 80 | }; /* typedef'd tp ZBUFF_CCtx within "zbuff.h" */ 81 | 82 | ZBUFF_CCtx* ZBUFF_createCCtx(void) 83 | { 84 | return ZBUFF_createCCtx_advanced(defaultCustomMem); 85 | } 86 | 87 | ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) 88 | { 89 | ZBUFF_CCtx* zbc; 90 | 91 | if (!customMem.customAlloc && !customMem.customFree) 92 | customMem = defaultCustomMem; 93 | 94 | if (!customMem.customAlloc || !customMem.customFree) 95 | return NULL; 96 | 97 | zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx)); 98 | if (zbc==NULL) return NULL; 99 | memset(zbc, 0, sizeof(ZBUFF_CCtx)); 100 | memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem)); 101 | zbc->zc = ZSTD_createCCtx_advanced(customMem); 102 | if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; } 103 | return zbc; 104 | } 105 | 106 | size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) 107 | { 108 | if (zbc==NULL) return 0; /* support free on NULL */ 109 | ZSTD_freeCCtx(zbc->zc); 110 | if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); 111 | if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); 112 | zbc->customMem.customFree(zbc->customMem.opaque, zbc); 113 | return 0; 114 | } 115 | 116 | 117 | /* ====== Initialization ====== */ 118 | 119 | size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, 120 | const void* dict, size_t dictSize, 121 | ZSTD_parameters params, unsigned long long pledgedSrcSize) 122 | { 123 | /* allocate buffers */ 124 | { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; 125 | if (zbc->inBuffSize < neededInBuffSize) { 126 | zbc->inBuffSize = neededInBuffSize; 127 | zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */ 128 | zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize); 129 | if (zbc->inBuff == NULL) return ERROR(memory_allocation); 130 | } 131 | zbc->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); 132 | } 133 | if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { 134 | zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; 135 | zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */ 136 | zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize); 137 | if (zbc->outBuff == NULL) return ERROR(memory_allocation); 138 | } 139 | 140 | { size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize); 141 | if (ZSTD_isError(errorCode)) return errorCode; } 142 | 143 | zbc->inToCompress = 0; 144 | zbc->inBuffPos = 0; 145 | zbc->inBuffTarget = zbc->blockSize; 146 | zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0; 147 | zbc->stage = ZBUFFcs_load; 148 | zbc->checksum = params.fParams.checksumFlag > 0; 149 | zbc->frameEnded = 0; 150 | return 0; /* ready to go */ 151 | } 152 | 153 | 154 | size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) 155 | { 156 | ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); 157 | return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0); 158 | } 159 | 160 | size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) 161 | { 162 | return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel); 163 | } 164 | 165 | 166 | /* internal util function */ 167 | MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) 168 | { 169 | size_t const length = MIN(dstCapacity, srcSize); 170 | memcpy(dst, src, length); 171 | return length; 172 | } 173 | 174 | 175 | /* ====== Compression ====== */ 176 | 177 | typedef enum { zbf_gather, zbf_flush, zbf_end } ZBUFF_flush_e; 178 | 179 | static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, 180 | void* dst, size_t* dstCapacityPtr, 181 | const void* src, size_t* srcSizePtr, 182 | ZBUFF_flush_e const flush) 183 | { 184 | U32 someMoreWork = 1; 185 | const char* const istart = (const char*)src; 186 | const char* const iend = istart + *srcSizePtr; 187 | const char* ip = istart; 188 | char* const ostart = (char*)dst; 189 | char* const oend = ostart + *dstCapacityPtr; 190 | char* op = ostart; 191 | 192 | while (someMoreWork) { 193 | switch(zbc->stage) 194 | { 195 | case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ 196 | 197 | case ZBUFFcs_load: 198 | /* complete inBuffer */ 199 | { size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos; 200 | size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip); 201 | zbc->inBuffPos += loaded; 202 | ip += loaded; 203 | if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) { 204 | someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */ 205 | } } 206 | /* compress current block (note : this stage cannot be stopped in the middle) */ 207 | { void* cDst; 208 | size_t cSize; 209 | size_t const iSize = zbc->inBuffPos - zbc->inToCompress; 210 | size_t oSize = oend-op; 211 | if (oSize >= ZSTD_compressBound(iSize)) 212 | cDst = op; /* compress directly into output buffer (avoid flush stage) */ 213 | else 214 | cDst = zbc->outBuff, oSize = zbc->outBuffSize; 215 | cSize = (flush == zbf_end) ? 216 | ZSTD_compressEnd(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize) : 217 | ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize); 218 | if (ZSTD_isError(cSize)) return cSize; 219 | if (flush == zbf_end) zbc->frameEnded = 1; 220 | /* prepare next block */ 221 | zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize; 222 | if (zbc->inBuffTarget > zbc->inBuffSize) 223 | zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */ 224 | zbc->inToCompress = zbc->inBuffPos; 225 | if (cDst == op) { op += cSize; break; } /* no need to flush */ 226 | zbc->outBuffContentSize = cSize; 227 | zbc->outBuffFlushedSize = 0; 228 | zbc->stage = ZBUFFcs_flush; /* continue to flush stage */ 229 | } 230 | 231 | case ZBUFFcs_flush: 232 | { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; 233 | size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); 234 | op += flushed; 235 | zbc->outBuffFlushedSize += flushed; 236 | if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */ 237 | zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0; 238 | zbc->stage = ZBUFFcs_load; 239 | break; 240 | } 241 | 242 | case ZBUFFcs_final: 243 | someMoreWork = 0; /* do nothing */ 244 | break; 245 | 246 | default: 247 | return ERROR(GENERIC); /* impossible */ 248 | } 249 | } 250 | 251 | *srcSizePtr = ip - istart; 252 | *dstCapacityPtr = op - ostart; 253 | if (zbc->frameEnded) return 0; 254 | { size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos; 255 | if (hintInSize==0) hintInSize = zbc->blockSize; 256 | return hintInSize; 257 | } 258 | } 259 | 260 | size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc, 261 | void* dst, size_t* dstCapacityPtr, 262 | const void* src, size_t* srcSizePtr) 263 | { 264 | return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, zbf_gather); 265 | } 266 | 267 | 268 | 269 | /* ====== Finalize ====== */ 270 | 271 | size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) 272 | { 273 | size_t srcSize = 0; 274 | ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, zbf_flush); /* use a valid src address instead of NULL */ 275 | return zbc->outBuffContentSize - zbc->outBuffFlushedSize; 276 | } 277 | 278 | 279 | size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) 280 | { 281 | BYTE* const ostart = (BYTE*)dst; 282 | BYTE* const oend = ostart + *dstCapacityPtr; 283 | BYTE* op = ostart; 284 | 285 | if (zbc->stage != ZBUFFcs_final) { 286 | /* flush whatever remains */ 287 | size_t outSize = *dstCapacityPtr; 288 | size_t srcSize = 0; 289 | size_t const notEnded = ZBUFF_compressContinue_generic(zbc, dst, &outSize, &srcSize, &srcSize, zbf_end); /* use a valid address instead of NULL */ 290 | size_t const remainingToFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; 291 | op += outSize; 292 | if (remainingToFlush) { 293 | *dstCapacityPtr = op-ostart; 294 | return remainingToFlush + ZBUFF_endFrameSize + (zbc->checksum * 4); 295 | } 296 | /* create epilogue */ 297 | zbc->stage = ZBUFFcs_final; 298 | zbc->outBuffContentSize = !notEnded ? 0 : 299 | ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize, NULL, 0); /* write epilogue into outBuff */ 300 | } 301 | 302 | /* flush epilogue */ 303 | { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; 304 | size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); 305 | op += flushed; 306 | zbc->outBuffFlushedSize += flushed; 307 | *dstCapacityPtr = op-ostart; 308 | if (toFlush==flushed) zbc->stage = ZBUFFcs_init; /* end reached */ 309 | return toFlush - flushed; 310 | } 311 | } 312 | 313 | 314 | 315 | /* ************************************* 316 | * Tool functions 317 | ***************************************/ 318 | size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } 319 | size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; } 320 | -------------------------------------------------------------------------------- /ext/zstd/libzstd/common/bitstream.h: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | bitstream 3 | Part of FSE library 4 | header file (to include) 5 | Copyright (C) 2013-2016, Yann Collet. 6 | 7 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions are 11 | met: 12 | 13 | * Redistributions of source code must retain the above copyright 14 | notice, this list of conditions and the following disclaimer. 15 | * Redistributions in binary form must reproduce the above 16 | copyright notice, this list of conditions and the following disclaimer 17 | in the documentation and/or other materials provided with the 18 | distribution. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | You can contact the author at : 33 | - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 34 | ****************************************************************** */ 35 | #ifndef BITSTREAM_H_MODULE 36 | #define BITSTREAM_H_MODULE 37 | 38 | #if defined (__cplusplus) 39 | extern "C" { 40 | #endif 41 | 42 | 43 | /* 44 | * This API consists of small unitary functions, which must be inlined for best performance. 45 | * Since link-time-optimization is not available for all compilers, 46 | * these functions are defined into a .h to be included. 47 | */ 48 | 49 | /*-**************************************** 50 | * Dependencies 51 | ******************************************/ 52 | #include "mem.h" /* unaligned access routines */ 53 | #include "error_private.h" /* error codes and messages */ 54 | 55 | 56 | /*========================================= 57 | * Target specific 58 | =========================================*/ 59 | #if defined(__BMI__) && defined(__GNUC__) 60 | # include /* support for bextr (experimental) */ 61 | #endif 62 | 63 | 64 | /*-****************************************** 65 | * bitStream encoding API (write forward) 66 | ********************************************/ 67 | /* bitStream can mix input from multiple sources. 68 | * A critical property of these streams is that they encode and decode in **reverse** direction. 69 | * So the first bit sequence you add will be the last to be read, like a LIFO stack. 70 | */ 71 | typedef struct 72 | { 73 | size_t bitContainer; 74 | int bitPos; 75 | char* startPtr; 76 | char* ptr; 77 | char* endPtr; 78 | } BIT_CStream_t; 79 | 80 | MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); 81 | MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); 82 | MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); 83 | MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); 84 | 85 | /* Start with initCStream, providing the size of buffer to write into. 86 | * bitStream will never write outside of this buffer. 87 | * `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. 88 | * 89 | * bits are first added to a local register. 90 | * Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. 91 | * Writing data into memory is an explicit operation, performed by the flushBits function. 92 | * Hence keep track how many bits are potentially stored into local register to avoid register overflow. 93 | * After a flushBits, a maximum of 7 bits might still be stored into local register. 94 | * 95 | * Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. 96 | * 97 | * Last operation is to close the bitStream. 98 | * The function returns the final size of CStream in bytes. 99 | * If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) 100 | */ 101 | 102 | 103 | /*-******************************************** 104 | * bitStream decoding API (read backward) 105 | **********************************************/ 106 | typedef struct 107 | { 108 | size_t bitContainer; 109 | unsigned bitsConsumed; 110 | const char* ptr; 111 | const char* start; 112 | } BIT_DStream_t; 113 | 114 | typedef enum { BIT_DStream_unfinished = 0, 115 | BIT_DStream_endOfBuffer = 1, 116 | BIT_DStream_completed = 2, 117 | BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ 118 | /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ 119 | 120 | MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); 121 | MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); 122 | MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); 123 | MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); 124 | 125 | 126 | /* Start by invoking BIT_initDStream(). 127 | * A chunk of the bitStream is then stored into a local register. 128 | * Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). 129 | * You can then retrieve bitFields stored into the local register, **in reverse order**. 130 | * Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. 131 | * A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. 132 | * Otherwise, it can be less than that, so proceed accordingly. 133 | * Checking if DStream has reached its end can be performed with BIT_endOfDStream(). 134 | */ 135 | 136 | 137 | /*-**************************************** 138 | * unsafe API 139 | ******************************************/ 140 | MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); 141 | /* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ 142 | 143 | MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); 144 | /* unsafe version; does not check buffer overflow */ 145 | 146 | MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); 147 | /* faster, but works only if nbBits >= 1 */ 148 | 149 | 150 | 151 | /*-************************************************************** 152 | * Internal functions 153 | ****************************************************************/ 154 | MEM_STATIC unsigned BIT_highbit32 (register U32 val) 155 | { 156 | # if defined(_MSC_VER) /* Visual */ 157 | unsigned long r=0; 158 | _BitScanReverse ( &r, val ); 159 | return (unsigned) r; 160 | # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ 161 | return 31 - __builtin_clz (val); 162 | # else /* Software version */ 163 | static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; 164 | U32 v = val; 165 | v |= v >> 1; 166 | v |= v >> 2; 167 | v |= v >> 4; 168 | v |= v >> 8; 169 | v |= v >> 16; 170 | return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; 171 | # endif 172 | } 173 | 174 | /*===== Local Constants =====*/ 175 | static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ 176 | 177 | 178 | /*-************************************************************** 179 | * bitStream encoding 180 | ****************************************************************/ 181 | /*! BIT_initCStream() : 182 | * `dstCapacity` must be > sizeof(void*) 183 | * @return : 0 if success, 184 | otherwise an error code (can be tested using ERR_isError() ) */ 185 | MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity) 186 | { 187 | bitC->bitContainer = 0; 188 | bitC->bitPos = 0; 189 | bitC->startPtr = (char*)startPtr; 190 | bitC->ptr = bitC->startPtr; 191 | bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); 192 | if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall); 193 | return 0; 194 | } 195 | 196 | /*! BIT_addBits() : 197 | can add up to 26 bits into `bitC`. 198 | Does not check for register overflow ! */ 199 | MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) 200 | { 201 | bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; 202 | bitC->bitPos += nbBits; 203 | } 204 | 205 | /*! BIT_addBitsFast() : 206 | * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ 207 | MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits) 208 | { 209 | bitC->bitContainer |= value << bitC->bitPos; 210 | bitC->bitPos += nbBits; 211 | } 212 | 213 | /*! BIT_flushBitsFast() : 214 | * unsafe version; does not check buffer overflow */ 215 | MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) 216 | { 217 | size_t const nbBytes = bitC->bitPos >> 3; 218 | MEM_writeLEST(bitC->ptr, bitC->bitContainer); 219 | bitC->ptr += nbBytes; 220 | bitC->bitPos &= 7; 221 | bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ 222 | } 223 | 224 | /*! BIT_flushBits() : 225 | * safe version; check for buffer overflow, and prevents it. 226 | * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ 227 | MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) 228 | { 229 | size_t const nbBytes = bitC->bitPos >> 3; 230 | MEM_writeLEST(bitC->ptr, bitC->bitContainer); 231 | bitC->ptr += nbBytes; 232 | if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; 233 | bitC->bitPos &= 7; 234 | bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ 235 | } 236 | 237 | /*! BIT_closeCStream() : 238 | * @return : size of CStream, in bytes, 239 | or 0 if it could not fit into dstBuffer */ 240 | MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) 241 | { 242 | BIT_addBitsFast(bitC, 1, 1); /* endMark */ 243 | BIT_flushBits(bitC); 244 | 245 | if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */ 246 | 247 | return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); 248 | } 249 | 250 | 251 | /*-******************************************************** 252 | * bitStream decoding 253 | **********************************************************/ 254 | /*! BIT_initDStream() : 255 | * Initialize a BIT_DStream_t. 256 | * `bitD` : a pointer to an already allocated BIT_DStream_t structure. 257 | * `srcSize` must be the *exact* size of the bitStream, in bytes. 258 | * @return : size of stream (== srcSize) or an errorCode if a problem is detected 259 | */ 260 | MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) 261 | { 262 | if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } 263 | 264 | if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ 265 | bitD->start = (const char*)srcBuffer; 266 | bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); 267 | bitD->bitContainer = MEM_readLEST(bitD->ptr); 268 | { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; 269 | bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; 270 | if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } 271 | } else { 272 | bitD->start = (const char*)srcBuffer; 273 | bitD->ptr = bitD->start; 274 | bitD->bitContainer = *(const BYTE*)(bitD->start); 275 | switch(srcSize) 276 | { 277 | case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); 278 | case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); 279 | case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); 280 | case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; 281 | case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; 282 | case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; 283 | default:; 284 | } 285 | { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; 286 | bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; 287 | if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } 288 | bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; 289 | } 290 | 291 | return srcSize; 292 | } 293 | 294 | MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) 295 | { 296 | return bitContainer >> start; 297 | } 298 | 299 | MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) 300 | { 301 | #if defined(__BMI__) && defined(__GNUC__) /* experimental */ 302 | # if defined(__x86_64__) 303 | if (sizeof(bitContainer)==8) 304 | return _bextr_u64(bitContainer, start, nbBits); 305 | else 306 | # endif 307 | return _bextr_u32(bitContainer, start, nbBits); 308 | #else 309 | return (bitContainer >> start) & BIT_mask[nbBits]; 310 | #endif 311 | } 312 | 313 | MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) 314 | { 315 | return bitContainer & BIT_mask[nbBits]; 316 | } 317 | 318 | /*! BIT_lookBits() : 319 | * Provides next n bits from local register. 320 | * local register is not modified. 321 | * On 32-bits, maxNbBits==24. 322 | * On 64-bits, maxNbBits==56. 323 | * @return : value extracted 324 | */ 325 | MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) 326 | { 327 | #if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ 328 | return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); 329 | #else 330 | U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; 331 | return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); 332 | #endif 333 | } 334 | 335 | /*! BIT_lookBitsFast() : 336 | * unsafe version; only works only if nbBits >= 1 */ 337 | MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) 338 | { 339 | U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; 340 | return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); 341 | } 342 | 343 | MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) 344 | { 345 | bitD->bitsConsumed += nbBits; 346 | } 347 | 348 | /*! BIT_readBits() : 349 | * Read (consume) next n bits from local register and update. 350 | * Pay attention to not read more than nbBits contained into local register. 351 | * @return : extracted value. 352 | */ 353 | MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) 354 | { 355 | size_t const value = BIT_lookBits(bitD, nbBits); 356 | BIT_skipBits(bitD, nbBits); 357 | return value; 358 | } 359 | 360 | /*! BIT_readBitsFast() : 361 | * unsafe version; only works only if nbBits >= 1 */ 362 | MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) 363 | { 364 | size_t const value = BIT_lookBitsFast(bitD, nbBits); 365 | BIT_skipBits(bitD, nbBits); 366 | return value; 367 | } 368 | 369 | /*! BIT_reloadDStream() : 370 | * Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ). 371 | * This function is safe, it guarantees it will not read beyond src buffer. 372 | * @return : status of `BIT_DStream_t` internal register. 373 | if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ 374 | MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) 375 | { 376 | if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ 377 | return BIT_DStream_overflow; 378 | 379 | if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { 380 | bitD->ptr -= bitD->bitsConsumed >> 3; 381 | bitD->bitsConsumed &= 7; 382 | bitD->bitContainer = MEM_readLEST(bitD->ptr); 383 | return BIT_DStream_unfinished; 384 | } 385 | if (bitD->ptr == bitD->start) { 386 | if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; 387 | return BIT_DStream_completed; 388 | } 389 | { U32 nbBytes = bitD->bitsConsumed >> 3; 390 | BIT_DStream_status result = BIT_DStream_unfinished; 391 | if (bitD->ptr - nbBytes < bitD->start) { 392 | nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ 393 | result = BIT_DStream_endOfBuffer; 394 | } 395 | bitD->ptr -= nbBytes; 396 | bitD->bitsConsumed -= nbBytes*8; 397 | bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ 398 | return result; 399 | } 400 | } 401 | 402 | /*! BIT_endOfDStream() : 403 | * @return Tells if DStream has exactly reached its end (all bits consumed). 404 | */ 405 | MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) 406 | { 407 | return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); 408 | } 409 | 410 | #if defined (__cplusplus) 411 | } 412 | #endif 413 | 414 | #endif /* BITSTREAM_H_MODULE */ 415 | --------------------------------------------------------------------------------