├── .gitattributes ├── doc └── images │ └── bench_mt2.png ├── data_block.h ├── test ├── Makefile ├── file_test.c └── file_test.vcxproj ├── fuzzer ├── Makefile ├── datagen.h ├── datagen.c └── fuzzer.vcxproj ├── bench ├── Makefile ├── bench.vcxproj └── bench.c ├── LICENSE ├── lzma2_enc.h ├── fl2_pool.h ├── radix_bitpack.c ├── atomic.h ├── Makefile ├── radix_mf.h ├── fl2_errors.h ├── fl2_compress_internal.h ├── fl2_threading.c ├── dict_buffer.h ├── radix_struct.c ├── fl2_internal.h ├── fl2_common.c ├── range_enc.h ├── count.h ├── radix_internal.h ├── dll ├── fast-lzma2.vcxproj.filters └── fast-lzma2.vcxproj ├── README.md ├── compiler.h ├── fl2_threading.h ├── .gitignore ├── lzma2_dec.h ├── radix_get.h ├── range_enc.c ├── fl2_pool.c ├── dict_buffer.c ├── util.h ├── platform.h ├── mem.h └── xxhash.h /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /doc/images/bench_mt2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/conor42/fast-lzma2/HEAD/doc/images/bench_mt2.png -------------------------------------------------------------------------------- /data_block.h: -------------------------------------------------------------------------------- 1 | #include "mem.h" 2 | 3 | #ifndef FL2_DATA_BLOCK_H_ 4 | #define FL2_DATA_BLOCK_H_ 5 | 6 | #if defined (__cplusplus) 7 | extern "C" { 8 | #endif 9 | 10 | typedef struct { 11 | const BYTE* data; 12 | size_t start; 13 | size_t end; 14 | } FL2_dataBlock; 15 | 16 | #if defined (__cplusplus) 17 | } 18 | #endif 19 | 20 | #endif /* FL2_DATA_BLOCK_H_ */ -------------------------------------------------------------------------------- /test/Makefile: -------------------------------------------------------------------------------- 1 | OBJ=file_test.o 2 | LIB:=-static -L../ -lfast-lzma2 3 | 4 | CFLAGS:=-Wall -O1 -pthread 5 | CC:=gcc 6 | 7 | EXT:= 8 | ifeq ($(OS),Windows_NT) 9 | EXT:=.exe 10 | LIB:=../libfast-lzma2.dll 11 | endif 12 | CFLAGS+=-I../ 13 | 14 | file_test : $(OBJ) 15 | $(CC) -pthread -o file_test$(EXT) $(OBJ) $(LIB) 16 | 17 | clean: 18 | rm -f file_test$(EXT) $(OBJ) 19 | -------------------------------------------------------------------------------- /fuzzer/Makefile: -------------------------------------------------------------------------------- 1 | SRC = $(wildcard ../*.c) datagen.c fuzzer.c 2 | OBJ = $(SRC:.c=.o) 3 | DEP = $(OBJ:.o=.d) 4 | 5 | # Use -fPIC for obj compatibility with shared lib build 6 | CFLAGS:=-Wall -O2 -pthread -fPIC 7 | CC:=gcc 8 | 9 | ASFLAGS:= 10 | 11 | x86_64:=0 12 | EXT:= 13 | 14 | ifeq ($(OS),Windows_NT) 15 | CFLAGS+=-DFL2_DLL_EXPORT=1 16 | EXT:=.exe 17 | ifeq ($(PROCESSOR_ARCHITECTURE),AMD64) 18 | ASFLAGS+=-DMS_x64_CALL=1 19 | x86_64:=1 20 | endif 21 | else 22 | PROC_ARCH:=$(shell uname -p) 23 | ifneq ($(PROC_ARCH),x86_64) 24 | PROC_ARCH:=$(shell uname -m) 25 | endif 26 | ifeq ($(PROC_ARCH),x86_64) 27 | ASFLAGS+=-DMS_x64_CALL=0 28 | x86_64:=1 29 | endif 30 | endif 31 | 32 | ifeq ($(x86_64),1) 33 | CFLAGS+=-DLZMA2_DEC_OPT 34 | OBJ+=../lzma_dec_x86_64.o 35 | endif 36 | 37 | fuzzer : $(OBJ) 38 | $(CC) -pthread -o fuzzer$(EXT) $(OBJ) 39 | 40 | -include $(DEP) 41 | 42 | %.d: %.c 43 | @$(CC) $(CFLAGS) $< -MM -MT $(@:.d=.o) >$@ 44 | 45 | clean: 46 | rm -f fuzzer$(EXT) $(OBJ) $(DEP) 47 | -------------------------------------------------------------------------------- /bench/Makefile: -------------------------------------------------------------------------------- 1 | SRC = $(wildcard ../*.c) ../fuzzer/datagen.c bench.c 2 | OBJ = $(SRC:.c=.o) 3 | DEP = $(OBJ:.o=.d) 4 | 5 | # Use -fPIC for obj compatibility with shared lib build 6 | CFLAGS:=-Wall -O2 -pthread -fPIC 7 | CC:=gcc 8 | 9 | ASFLAGS:= 10 | 11 | x86_64:=0 12 | EXT:= 13 | 14 | ifeq ($(OS),Windows_NT) 15 | CFLAGS+=-DFL2_DLL_EXPORT=1 16 | EXT:=.exe 17 | ifeq ($(PROCESSOR_ARCHITECTURE),AMD64) 18 | ASFLAGS+=-DMS_x64_CALL=1 19 | x86_64:=1 20 | endif 21 | else 22 | PROC_ARCH:=$(shell uname -p) 23 | ifneq ($(PROC_ARCH),x86_64) 24 | PROC_ARCH:=$(shell uname -m) 25 | endif 26 | ifeq ($(PROC_ARCH),x86_64) 27 | ASFLAGS+=-DMS_x64_CALL=0 28 | x86_64:=1 29 | endif 30 | endif 31 | 32 | ifeq ($(x86_64),1) 33 | CFLAGS+=-DLZMA2_DEC_OPT 34 | OBJ+=../lzma_dec_x86_64.o 35 | endif 36 | 37 | bench : $(OBJ) 38 | $(CC) -pthread -o bench$(EXT) $(OBJ) 39 | 40 | -include $(DEP) 41 | 42 | %.d: %.c 43 | @$(CC) $(CFLAGS) $< -MM -MT $(@:.d=.o) >$@ 44 | 45 | clean: 46 | rm -f bench$(EXT) $(OBJ) $(DEP) 47 | -------------------------------------------------------------------------------- /fuzzer/datagen.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | 12 | #ifndef DATAGEN_H 13 | #define DATAGEN_H 14 | 15 | #include /* size_t */ 16 | 17 | void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed); 18 | void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed); 19 | /*!RDG_genBuffer 20 | Generate 'size' bytes of compressible data into 'buffer'. 21 | Compressibility can be controlled using 'matchProba', which is floating point value between 0 and 1. 22 | 'LitProba' is optional, it affect variability of individual bytes. If litProba==0.0, default value will be used. 23 | Generated data pattern can be modified using different 'seed'. 24 | For a triplet (matchProba, litProba, seed), the function always generate the same content. 25 | 26 | RDG_genStdout 27 | Same as RDG_genBuffer, but generates data into stdout 28 | */ 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018-present Conor McCarthy 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /lzma2_enc.h: -------------------------------------------------------------------------------- 1 | /* lzma2_enc.h -- LZMA2 Encoder 2 | Based on LzmaEnc.h and Lzma2Enc.h : Igor Pavlov 3 | Modified for FL2 by Conor McCarthy 4 | Public domain 5 | */ 6 | 7 | #ifndef RADYX_LZMA2_ENCODER_H 8 | #define RADYX_LZMA2_ENCODER_H 9 | 10 | #include "mem.h" 11 | #include "data_block.h" 12 | #include "radix_mf.h" 13 | #include "atomic.h" 14 | 15 | #if defined (__cplusplus) 16 | extern "C" { 17 | #endif 18 | 19 | #define kFastDistBits 12U 20 | 21 | #define LZMA2_END_MARKER '\0' 22 | #define ENC_MIN_BYTES_PER_THREAD 0x1C000 /* Enough for 8 threads, 1 Mb dict, 2/16 overlap */ 23 | 24 | 25 | typedef struct LZMA2_ECtx_s LZMA2_ECtx; 26 | 27 | typedef struct 28 | { 29 | unsigned lc; 30 | unsigned lp; 31 | unsigned pb; 32 | unsigned fast_length; 33 | unsigned match_cycles; 34 | FL2_strategy strategy; 35 | unsigned second_dict_bits; 36 | unsigned reset_interval; 37 | } FL2_lzma2Parameters; 38 | 39 | 40 | LZMA2_ECtx* LZMA2_createECtx(void); 41 | 42 | void LZMA2_freeECtx(LZMA2_ECtx *const enc); 43 | 44 | int LZMA2_hashAlloc(LZMA2_ECtx *const enc, const FL2_lzma2Parameters* const options); 45 | 46 | size_t LZMA2_encode(LZMA2_ECtx *const enc, 47 | FL2_matchTable* const tbl, 48 | FL2_dataBlock const block, 49 | const FL2_lzma2Parameters* const options, 50 | int stream_prop, 51 | FL2_atomic *const progress_in, 52 | FL2_atomic *const progress_out, 53 | int *const canceled); 54 | 55 | BYTE LZMA2_getDictSizeProp(size_t const dictionary_size); 56 | 57 | size_t LZMA2_compressBound(size_t src_size); 58 | 59 | size_t LZMA2_encMemoryUsage(unsigned const chain_log, FL2_strategy const strategy, unsigned const thread_count); 60 | 61 | #if defined (__cplusplus) 62 | } 63 | #endif 64 | 65 | #endif /* RADYX_LZMA2_ENCODER_H */ -------------------------------------------------------------------------------- /fl2_pool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * Modified for FL2 by Conor McCarthy 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #ifndef FL2POOL_H 13 | #define FL2POOL_H 14 | 15 | #if defined (__cplusplus) 16 | extern "C" { 17 | #endif 18 | 19 | 20 | #include /* size_t */ 21 | 22 | typedef struct FL2POOL_ctx_s FL2POOL_ctx; 23 | 24 | /*! FL2POOL_create() : 25 | * Create a thread pool with at most `numThreads` threads. 26 | * `numThreads` must be at least 1. 27 | * @return : FL2POOL_ctx pointer on success, else NULL. 28 | */ 29 | FL2POOL_ctx *FL2POOL_create(size_t numThreads); 30 | 31 | 32 | /*! FL2POOL_free() : 33 | Free a thread pool returned by FL2POOL_create(). 34 | */ 35 | void FL2POOL_free(FL2POOL_ctx *ctx); 36 | 37 | /*! FL2POOL_sizeof() : 38 | return memory usage of pool returned by FL2POOL_create(). 39 | */ 40 | size_t FL2POOL_sizeof(FL2POOL_ctx *ctx); 41 | 42 | /*! FL2POOL_function : 43 | The function type that can be added to a thread pool. 44 | */ 45 | typedef void(*FL2POOL_function)(void *, ptrdiff_t); 46 | 47 | /*! FL2POOL_add() : 48 | Add the job `function(opaque)` to the thread pool. 49 | FL2POOL_addRange adds multiple jobs with size_t parameter from first to less than end. 50 | Possibly blocks until there is room in the queue. 51 | Note : The function may be executed asynchronously, so `opaque` must live until the function has been completed. 52 | */ 53 | void FL2POOL_add(void* ctxVoid, FL2POOL_function function, void *opaque, ptrdiff_t n); 54 | void FL2POOL_addRange(void *ctx, FL2POOL_function function, void *opaque, ptrdiff_t first, ptrdiff_t end); 55 | 56 | int FL2POOL_waitAll(void *ctx, unsigned timeout); 57 | 58 | size_t FL2POOL_threadsBusy(void *ctx); 59 | 60 | #if defined (__cplusplus) 61 | } 62 | #endif 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /radix_bitpack.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018, Conor McCarthy 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #include "mem.h" /* U32, U64 */ 12 | #include "fl2_threading.h" 13 | #include "fl2_internal.h" 14 | #include "radix_internal.h" 15 | 16 | #undef MIN 17 | #define MIN(a,b) ((a) < (b) ? (a) : (b)) 18 | 19 | #define RMF_BITPACK 20 | 21 | #define RADIX_MAX_LENGTH BITPACK_MAX_LENGTH 22 | 23 | #define InitMatchLink(pos, link) tbl->table[pos] = link 24 | 25 | #define GetMatchLink(link) (tbl->table[link] & RADIX_LINK_MASK) 26 | 27 | #define GetInitialMatchLink(pos) tbl->table[pos] 28 | 29 | #define GetMatchLength(pos) (tbl->table[pos] >> RADIX_LINK_BITS) 30 | 31 | #define SetMatchLink(pos, link, length) tbl->table[pos] = (link) | ((U32)(length) << RADIX_LINK_BITS) 32 | 33 | #define SetMatchLength(pos, link, length) tbl->table[pos] = (link) | ((U32)(length) << RADIX_LINK_BITS) 34 | 35 | #define SetMatchLinkAndLength(pos, link, length) tbl->table[pos] = (link) | ((U32)(length) << RADIX_LINK_BITS) 36 | 37 | #define SetNull(pos) tbl->table[pos] = RADIX_NULL_LINK 38 | 39 | #define IsNull(pos) (tbl->table[pos] == RADIX_NULL_LINK) 40 | 41 | BYTE* RMF_bitpackAsOutputBuffer(FL2_matchTable* const tbl, size_t const pos) 42 | { 43 | return (BYTE*)(tbl->table + pos); 44 | } 45 | 46 | /* Restrict the match lengths so that they don't reach beyond pos */ 47 | void RMF_bitpackLimitLengths(FL2_matchTable* const tbl, size_t const pos) 48 | { 49 | DEBUGLOG(5, "RMF_limitLengths : end %u, max length %u", (U32)pos, RADIX_MAX_LENGTH); 50 | SetNull(pos - 1); 51 | for (U32 length = 2; length < RADIX_MAX_LENGTH && length <= pos; ++length) { 52 | U32 const link = tbl->table[pos - length]; 53 | if (link != RADIX_NULL_LINK) 54 | tbl->table[pos - length] = (MIN(length, link >> RADIX_LINK_BITS) << RADIX_LINK_BITS) | (link & RADIX_LINK_MASK); 55 | } 56 | } 57 | 58 | #include "radix_engine.h" -------------------------------------------------------------------------------- /atomic.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018 Conor McCarthy 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * 9 | */ 10 | 11 | #ifndef FL2_ATOMIC_H 12 | #define FL2_ATOMIC_H 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | /* atomic add */ 19 | 20 | #if !defined(FL2_SINGLETHREAD) && defined(_WIN32) 21 | 22 | #ifdef WINVER 23 | #undef WINVER 24 | #endif 25 | #define WINVER 0x0600 26 | 27 | #ifdef _WIN32_WINNT 28 | #undef _WIN32_WINNT 29 | #endif 30 | #define _WIN32_WINNT 0x0600 31 | 32 | #ifndef WIN32_LEAN_AND_MEAN 33 | #define WIN32_LEAN_AND_MEAN 34 | #endif 35 | 36 | #include 37 | 38 | 39 | typedef LONG volatile FL2_atomic; 40 | #define ATOMIC_INITIAL_VALUE -1 41 | #define FL2_atomic_increment(n) InterlockedIncrement(&n) 42 | #define FL2_atomic_add(n, a) InterlockedAdd(&n, a) 43 | #define FL2_nonAtomic_increment(n) (++n) 44 | 45 | #elif !defined(FL2_SINGLETHREAD) && defined(__GNUC__) 46 | 47 | typedef long FL2_atomic; 48 | #define ATOMIC_INITIAL_VALUE 0 49 | #define FL2_atomic_increment(n) __sync_fetch_and_add(&n, 1) 50 | #define FL2_atomic_add(n, a) __sync_fetch_and_add(&n, a) 51 | #define FL2_nonAtomic_increment(n) (n++) 52 | 53 | #elif !defined(FL2_SINGLETHREAD) && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_ATOMICS__) /* C11 */ 54 | 55 | #include 56 | 57 | typedef _Atomic long FL2_atomic; 58 | #define ATOMIC_INITIAL_VALUE 0 59 | #define FL2_atomic_increment(n) atomic_fetch_add(&n, 1) 60 | #define FL2_atomic_add(n, a) atomic_fetch_add(&n, a) 61 | #define FL2_nonAtomic_increment(n) (n++) 62 | 63 | #else /* No atomics */ 64 | 65 | # ifndef FL2_SINGLETHREAD 66 | # error No atomic operations available. Change compiler config or define FL2_SINGLETHREAD for the entire build. 67 | # endif 68 | 69 | typedef long FL2_atomic; 70 | #define ATOMIC_INITIAL_VALUE 0 71 | #define FL2_atomic_increment(n) (n++) 72 | #define FL2_atomic_add(n, a) (n += (a)) 73 | #define FL2_nonAtomic_increment(n) (n++) 74 | 75 | #endif /* FL2_SINGLETHREAD */ 76 | 77 | 78 | #if defined (__cplusplus) 79 | } 80 | #endif 81 | 82 | #endif /* FL2_ATOMIC_H */ 83 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SRC = $(wildcard *.c) 2 | OBJ = $(SRC:.c=.o) 3 | DEP = $(OBJ:.o=.d) 4 | 5 | CFLAGS:=-Wall -O2 -pthread -fPIC 6 | CC:=gcc 7 | AR:=ar -rcs 8 | RM:=rm -rf 9 | 10 | ASFLAGS := 11 | 12 | SONAME:=libfast-lzma2.so.1 13 | REAL_NAME:=libfast-lzma2.so.1.0 14 | LINKER_NAME=libfast-lzma2.so 15 | STATIC_LIBNAME=libfast-lzma2.a 16 | 17 | x86_64:=0 18 | 19 | ifeq ($(OS),Windows_NT) 20 | CFLAGS+=-DFL2_DLL_EXPORT=1 21 | LINKER_NAME=libfast-lzma2.dll 22 | SONAME:=$(LINKER_NAME) 23 | REAL_NAME:=$(LINKER_NAME) 24 | ifeq ($(PROCESSOR_ARCHITECTURE),AMD64) 25 | ASFLAGS+=-DMS_x64_CALL=1 26 | x86_64:=1 27 | endif 28 | else 29 | PROC_ARCH:=$(shell uname -p) 30 | ifneq ($(PROC_ARCH),x86_64) 31 | PROC_ARCH:=$(shell uname -m) 32 | endif 33 | ifeq ($(PROC_ARCH),x86_64) 34 | ASFLAGS+=-DMS_x64_CALL=0 35 | x86_64:=1 36 | endif 37 | endif 38 | 39 | ifeq ($(x86_64),1) 40 | CFLAGS+=-DLZMA2_DEC_OPT 41 | OBJ+=lzma_dec_x86_64.o 42 | endif 43 | 44 | libfast-lzma2 : $(OBJ) 45 | @echo "Build static & dynamic library." 46 | $(CC) -shared -pthread -Wl,-soname,$(SONAME) -o $(REAL_NAME) $(OBJ) 47 | $(AR) $(STATIC_LIBNAME) $(OBJ) 48 | @echo "Library build SUCCESS." 49 | 50 | -include $(DEP) 51 | 52 | %.d: %.c 53 | @$(CC) $(CFLAGS) $< -MM -MT $(@:.d=.o) >$@ 54 | 55 | DESTDIR:= 56 | PREFIX:=/usr/local 57 | LIBDIR:=$(DESTDIR)$(PREFIX)/lib 58 | 59 | .PHONY: install 60 | install: 61 | ifeq ($(OS),Windows_NT) 62 | strip -g $(REAL_NAME) 63 | else 64 | mkdir -p $(LIBDIR) 65 | cp $(REAL_NAME) $(LIBDIR)/$(REAL_NAME) 66 | strip -g $(LIBDIR)/$(REAL_NAME) 67 | chmod 0755 $(LIBDIR)/$(REAL_NAME) 68 | cd $(LIBDIR) && ln -sf $(REAL_NAME) $(LINKER_NAME) 69 | ldconfig $(LIBDIR) 70 | mkdir -p $(DESTDIR)$(PREFIX)/include 71 | cp fast-lzma2.h $(DESTDIR)$(PREFIX)/include/ 72 | cp fl2_errors.h $(DESTDIR)$(PREFIX)/include/ 73 | endif 74 | 75 | .PHONY: uninstall 76 | uninstall: 77 | ifeq ($(OS),Windows_NT) 78 | rm -f libfast-lzma2.dll 79 | else 80 | rm -f $(LIBDIR)/$(LINKER_NAME) 81 | rm -f $(LIBDIR)/$(REAL_NAME) 82 | ldconfig $(LIBDIR) 83 | rm -f $(DESTDIR)$(PREFIX)/include/fast-lzma2.h 84 | rm -f $(DESTDIR)$(PREFIX)/include/fl2_errors.h 85 | endif 86 | 87 | .PHONY: test 88 | test:libfast-lzma2 89 | $(MAKE) -C ./test file_test 90 | test/file_test radix_engine.h 91 | @echo "File compression/decompression test completed." 92 | 93 | .PHONY: clean 94 | clean: 95 | $(RM) $(REAL_NAME) $(STATIC_LIBNAME) $(OBJ) $(DEP) 96 | $(MAKE) -C ./test clean 97 | -------------------------------------------------------------------------------- /radix_mf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018, Conor McCarthy 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef RADIX_MF_H 12 | #define RADIX_MF_H 13 | 14 | #include "fast-lzma2.h" 15 | #include "data_block.h" 16 | 17 | #if defined (__cplusplus) 18 | extern "C" { 19 | #endif 20 | 21 | typedef struct FL2_matchTable_s FL2_matchTable; 22 | 23 | #define OVERLAP_FROM_DICT_SIZE(d, o) (((d) >> 4) * (o)) 24 | 25 | #define RMF_MIN_BYTES_PER_THREAD 1024 26 | 27 | typedef struct 28 | { 29 | size_t dictionary_size; 30 | unsigned match_buffer_resize; 31 | unsigned overlap_fraction; 32 | unsigned divide_and_conquer; 33 | unsigned depth; 34 | #ifdef RMF_REFERENCE 35 | unsigned use_ref_mf; 36 | #endif 37 | } RMF_parameters; 38 | 39 | FL2_matchTable* RMF_createMatchTable(const RMF_parameters* const params, size_t const dict_reduce, unsigned const thread_count); 40 | void RMF_freeMatchTable(FL2_matchTable* const tbl); 41 | BYTE RMF_compatibleParameters(const FL2_matchTable* const tbl, const RMF_parameters* const params, size_t const dict_reduce); 42 | size_t RMF_applyParameters(FL2_matchTable* const tbl, const RMF_parameters* const params, size_t const dict_reduce); 43 | size_t RMF_threadCount(const FL2_matchTable * const tbl); 44 | void RMF_initProgress(FL2_matchTable * const tbl); 45 | void RMF_initTable(FL2_matchTable* const tbl, const void* const data, size_t const end); 46 | int RMF_buildTable(FL2_matchTable* const tbl, 47 | size_t const job, 48 | unsigned const multi_thread, 49 | FL2_dataBlock const block); 50 | void RMF_cancelBuild(FL2_matchTable* const tbl); 51 | void RMF_resetIncompleteBuild(FL2_matchTable* const tbl); 52 | int RMF_integrityCheck(const FL2_matchTable* const tbl, const BYTE* const data, size_t const pos, size_t const end, unsigned const max_depth); 53 | void RMF_limitLengths(FL2_matchTable* const tbl, size_t const pos); 54 | BYTE* RMF_getTableAsOutputBuffer(FL2_matchTable* const tbl, size_t const pos); 55 | size_t RMF_memoryUsage(size_t const dict_size, unsigned const buffer_resize, unsigned const thread_count); 56 | 57 | #if defined (__cplusplus) 58 | } 59 | #endif 60 | 61 | #endif /* RADIX_MF_H */ -------------------------------------------------------------------------------- /fl2_errors.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * Modified for FL2 by Conor McCarthy 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #ifndef FL2_ERRORS_H_398273423 13 | #define FL2_ERRORS_H_398273423 14 | 15 | #if defined (__cplusplus) 16 | extern "C" { 17 | #endif 18 | 19 | /*===== dependency =====*/ 20 | #include /* size_t */ 21 | 22 | #include "fast-lzma2.h" 23 | 24 | /*-**************************************** 25 | * error codes list 26 | * note : this API is still considered unstable 27 | * and shall not be used with a dynamic library. 28 | * only static linking is allowed 29 | ******************************************/ 30 | typedef enum { 31 | FL2_error_no_error = 0, 32 | FL2_error_GENERIC = 1, 33 | FL2_error_internal = 2, 34 | FL2_error_corruption_detected = 3, 35 | FL2_error_checksum_wrong = 4, 36 | FL2_error_parameter_unsupported = 5, 37 | FL2_error_parameter_outOfBound = 6, 38 | FL2_error_lclpMax_exceeded = 7, 39 | FL2_error_stage_wrong = 8, 40 | FL2_error_init_missing = 9, 41 | FL2_error_memory_allocation = 10, 42 | FL2_error_dstSize_tooSmall = 11, 43 | FL2_error_srcSize_wrong = 12, 44 | FL2_error_canceled = 13, 45 | FL2_error_buffer = 14, 46 | FL2_error_timedOut = 15, 47 | FL2_error_maxCode = 20 /* never EVER use this value directly, it can change in future versions! Use FL2_isError() instead */ 48 | } FL2_ErrorCode; 49 | 50 | /*! FL2_getErrorCode() : 51 | convert a `size_t` function result into a `FL2_ErrorCode` enum type, 52 | which can be used to compare with enum list published above */ 53 | FL2LIB_API FL2_ErrorCode FL2LIB_CALL FL2_getErrorCode(size_t functionResult); 54 | FL2LIB_API const char* FL2LIB_CALL FL2_getErrorString(FL2_ErrorCode code); /**< Same as FL2_getErrorName, but using a `FL2_ErrorCode` enum argument */ 55 | 56 | 57 | #if defined (__cplusplus) 58 | } 59 | #endif 60 | 61 | #endif /* FL2_ERRORS_H_398273423 */ 62 | -------------------------------------------------------------------------------- /fl2_compress_internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018, Conor McCarthy 3 | * All rights reserved. 4 | * Parts based on zstd_compress_internal.h copyright Yann Collet 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #ifndef FL2_COMPRESS_H 13 | #define FL2_COMPRESS_H 14 | 15 | /*-************************************* 16 | * Dependencies 17 | ***************************************/ 18 | #include "mem.h" 19 | #include "data_block.h" 20 | #include "radix_internal.h" 21 | #include "lzma2_enc.h" 22 | #include "fast-lzma2.h" 23 | #include "fl2_threading.h" 24 | #include "fl2_pool.h" 25 | #include "dict_buffer.h" 26 | #ifndef NO_XXHASH 27 | # include "xxhash.h" 28 | #endif 29 | 30 | #if defined (__cplusplus) 31 | extern "C" { 32 | #endif 33 | 34 | /*-************************************* 35 | * Context memory management 36 | ***************************************/ 37 | 38 | typedef struct { 39 | FL2_lzma2Parameters cParams; 40 | RMF_parameters rParams; 41 | unsigned compressionLevel; 42 | BYTE highCompression; 43 | #ifndef NO_XXHASH 44 | BYTE doXXH; 45 | #endif 46 | BYTE omitProp; 47 | } FL2_CCtx_params; 48 | 49 | typedef struct { 50 | FL2_CCtx* cctx; 51 | LZMA2_ECtx* enc; 52 | FL2_dataBlock block; 53 | size_t cSize; 54 | } FL2_job; 55 | 56 | struct FL2_CCtx_s { 57 | DICT_buffer buf; 58 | FL2_CCtx_params params; 59 | #ifndef FL2_SINGLETHREAD 60 | FL2POOL_ctx* factory; 61 | FL2POOL_ctx* compressThread; 62 | #endif 63 | FL2_dataBlock curBlock; 64 | size_t asyncRes; 65 | size_t threadCount; 66 | size_t outThread; 67 | size_t outPos; 68 | size_t dictMax; 69 | U64 streamTotal; 70 | U64 streamCsize; 71 | FL2_matchTable* matchTable; 72 | #ifndef FL2_SINGLETHREAD 73 | U32 timeout; 74 | #endif 75 | U32 rmfWeight; 76 | U32 encWeight; 77 | FL2_atomic progressIn; 78 | FL2_atomic progressOut; 79 | int canceled; 80 | BYTE wroteProp; 81 | BYTE endMarked; 82 | BYTE loopCount; 83 | BYTE lockParams; 84 | unsigned jobCount; 85 | FL2_job jobs[1]; 86 | }; 87 | 88 | #if defined (__cplusplus) 89 | } 90 | #endif 91 | 92 | 93 | #endif /* FL2_COMPRESS_H */ 94 | -------------------------------------------------------------------------------- /fl2_threading.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016 Tino Reichardt 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * 9 | * You can contact the author at: 10 | * - zstdmt source repository: https://github.com/mcmilk/zstdmt 11 | */ 12 | 13 | /** 14 | * This file will hold wrapper for systems, which do not support pthreads 15 | */ 16 | 17 | /* create fake symbol to avoid empty translation unit warning */ 18 | int g_ZSTD_threading_useles_symbol; 19 | 20 | #include "fast-lzma2.h" 21 | #include "fl2_threading.h" 22 | #include "util.h" 23 | 24 | #if !defined(FL2_SINGLETHREAD) && defined(_WIN32) 25 | 26 | /** 27 | * Windows minimalist Pthread Wrapper, based on : 28 | * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html 29 | */ 30 | 31 | 32 | /* === Dependencies === */ 33 | #include 34 | #include 35 | 36 | 37 | /* === Implementation === */ 38 | 39 | static unsigned __stdcall worker(void *arg) 40 | { 41 | FL2_pthread_t* const thread = (FL2_pthread_t*) arg; 42 | thread->arg = thread->start_routine(thread->arg); 43 | return 0; 44 | } 45 | 46 | int FL2_pthread_create(FL2_pthread_t* thread, const void* unused, 47 | void* (*start_routine) (void*), void* arg) 48 | { 49 | (void)unused; 50 | thread->arg = arg; 51 | thread->start_routine = start_routine; 52 | thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL); 53 | 54 | if (!thread->handle) 55 | return errno; 56 | else 57 | return 0; 58 | } 59 | 60 | int FL2_pthread_join(FL2_pthread_t thread, void **value_ptr) 61 | { 62 | DWORD result; 63 | 64 | if (!thread.handle) return 0; 65 | 66 | result = WaitForSingleObject(thread.handle, INFINITE); 67 | switch (result) { 68 | case WAIT_OBJECT_0: 69 | if (value_ptr) *value_ptr = thread.arg; 70 | return 0; 71 | case WAIT_ABANDONED: 72 | return EINVAL; 73 | default: 74 | return GetLastError(); 75 | } 76 | } 77 | 78 | #endif /* FL2_SINGLETHREAD */ 79 | 80 | unsigned FL2_checkNbThreads(unsigned nbThreads) 81 | { 82 | #ifndef FL2_SINGLETHREAD 83 | if (nbThreads == 0) { 84 | nbThreads = UTIL_countPhysicalCores(); 85 | nbThreads += !nbThreads; 86 | } 87 | if (nbThreads > FL2_MAXTHREADS) { 88 | nbThreads = FL2_MAXTHREADS; 89 | } 90 | #else 91 | nbThreads = 1; 92 | #endif 93 | return nbThreads; 94 | } 95 | 96 | -------------------------------------------------------------------------------- /dict_buffer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018, Conor McCarthy 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #include "fast-lzma2.h" 12 | #include "mem.h" 13 | #include "data_block.h" 14 | #ifndef NO_XXHASH 15 | # include "xxhash.h" 16 | #endif 17 | 18 | #ifndef FL2_DICT_BUFFER_H_ 19 | #define FL2_DICT_BUFFER_H_ 20 | 21 | #if defined (__cplusplus) 22 | extern "C" { 23 | #endif 24 | 25 | /* DICT_buffer structure. 26 | * Maintains one or two dictionary buffers. In a dual dict configuration (asyc==1), when the 27 | * current buffer is full, the overlap region will be copied to the other buffer and it 28 | * becomes the destination for input while the first is compressed. This is useful when I/O 29 | * is much slower than compression. */ 30 | typedef struct { 31 | BYTE* data[2]; 32 | size_t index; 33 | size_t async; 34 | size_t overlap; 35 | size_t start; /* start = 0 (first block) or overlap */ 36 | size_t end; /* never < overlap */ 37 | size_t size; /* allocation size */ 38 | size_t total; /* total size compressed after last dict reset */ 39 | size_t reset_interval; 40 | #ifndef NO_XXHASH 41 | XXH32_state_t *xxh; 42 | #endif 43 | } DICT_buffer; 44 | 45 | int DICT_construct(DICT_buffer *const buf, int const async); 46 | 47 | int DICT_init(DICT_buffer *const buf, size_t const dict_size, size_t const overlap, unsigned const reset_multiplier, int const do_hash); 48 | 49 | void DICT_destruct(DICT_buffer *const buf); 50 | 51 | size_t DICT_size(const DICT_buffer *const buf); 52 | 53 | size_t DICT_get(DICT_buffer *const buf, void **const dict); 54 | 55 | int DICT_update(DICT_buffer *const buf, size_t const added_size); 56 | 57 | void DICT_put(DICT_buffer *const buf, FL2_inBuffer* const input); 58 | 59 | size_t DICT_availSpace(const DICT_buffer *const buf); 60 | 61 | int DICT_hasUnprocessed(const DICT_buffer *const buf); 62 | 63 | void DICT_getBlock(DICT_buffer *const buf, FL2_dataBlock *const block); 64 | 65 | int DICT_needShift(DICT_buffer *const buf); 66 | 67 | int DICT_async(const DICT_buffer *const buf); 68 | 69 | void DICT_shift(DICT_buffer *const buf); 70 | 71 | #ifndef NO_XXHASH 72 | XXH32_hash_t DICT_getDigest(const DICT_buffer *const buf); 73 | #endif 74 | 75 | size_t DICT_memUsage(const DICT_buffer *const buf); 76 | 77 | #if defined (__cplusplus) 78 | } 79 | #endif 80 | 81 | #endif /* FL2_DICT_BUFFER_H_ */ -------------------------------------------------------------------------------- /radix_struct.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018, Conor McCarthy 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #include "mem.h" /* U32, U64 */ 12 | #include "fl2_threading.h" 13 | #include "fl2_internal.h" 14 | #include "radix_internal.h" 15 | 16 | #undef MIN 17 | #define MIN(a,b) ((a) < (b) ? (a) : (b)) 18 | 19 | #define RMF_STRUCTURED 20 | 21 | #define RADIX_MAX_LENGTH STRUCTURED_MAX_LENGTH 22 | 23 | #define InitMatchLink(pos, link) ((RMF_unit*)tbl->table)[(pos) >> UNIT_BITS].links[(pos) & UNIT_MASK] = (U32)(link) 24 | 25 | #define GetMatchLink(pos) ((RMF_unit*)tbl->table)[(pos) >> UNIT_BITS].links[(pos) & UNIT_MASK] 26 | 27 | #define GetInitialMatchLink(pos) ((RMF_unit*)tbl->table)[(pos) >> UNIT_BITS].links[(pos) & UNIT_MASK] 28 | 29 | #define GetMatchLength(pos) ((RMF_unit*)tbl->table)[(pos) >> UNIT_BITS].lengths[(pos) & UNIT_MASK] 30 | 31 | #define SetMatchLink(pos, link, length) ((RMF_unit*)tbl->table)[(pos) >> UNIT_BITS].links[(pos) & UNIT_MASK] = (U32)(link) 32 | 33 | #define SetMatchLength(pos, link, length) ((RMF_unit*)tbl->table)[(pos) >> UNIT_BITS].lengths[(pos) & UNIT_MASK] = (BYTE)(length) 34 | 35 | #define SetMatchLinkAndLength(pos, link, length) do { size_t i_ = (pos) >> UNIT_BITS, u_ = (pos) & UNIT_MASK; ((RMF_unit*)tbl->table)[i_].links[u_] = (U32)(link); ((RMF_unit*)tbl->table)[i_].lengths[u_] = (BYTE)(length); } while(0) 36 | 37 | #define SetNull(pos) ((RMF_unit*)tbl->table)[(pos) >> UNIT_BITS].links[(pos) & UNIT_MASK] = RADIX_NULL_LINK 38 | 39 | #define IsNull(pos) (((RMF_unit*)tbl->table)[(pos) >> UNIT_BITS].links[(pos) & UNIT_MASK] == RADIX_NULL_LINK) 40 | 41 | BYTE* RMF_structuredAsOutputBuffer(FL2_matchTable* const tbl, size_t const pos) 42 | { 43 | return (BYTE*)((RMF_unit*)tbl->table + (pos >> UNIT_BITS) + ((pos & UNIT_MASK) != 0)); 44 | } 45 | 46 | /* Restrict the match lengths so that they don't reach beyond pos */ 47 | void RMF_structuredLimitLengths(FL2_matchTable* const tbl, size_t const pos) 48 | { 49 | DEBUGLOG(5, "RMF_limitLengths : end %u, max length %u", (U32)pos, RADIX_MAX_LENGTH); 50 | SetNull(pos - 1); 51 | for (size_t length = 2; length < RADIX_MAX_LENGTH && length <= pos; ++length) { 52 | size_t const i = (pos - length) >> UNIT_BITS; 53 | size_t const u = (pos - length) & UNIT_MASK; 54 | if (((RMF_unit*)tbl->table)[i].links[u] != RADIX_NULL_LINK) { 55 | ((RMF_unit*)tbl->table)[i].lengths[u] = MIN((BYTE)length, ((RMF_unit*)tbl->table)[i].lengths[u]); 56 | } 57 | } 58 | } 59 | 60 | #include "radix_engine.h" -------------------------------------------------------------------------------- /fl2_internal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * Modified for FL2 by Conor McCarthy 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #ifndef FL2_INTERNAL_H_ 13 | #define FL2_INTERNAL_H_ 14 | 15 | 16 | /*-************************************* 17 | * Dependencies 18 | ***************************************/ 19 | #include "mem.h" 20 | #include "compiler.h" 21 | 22 | 23 | #if defined (__cplusplus) 24 | extern "C" { 25 | #endif 26 | 27 | 28 | /*-**************************************** 29 | * Error codes handling 30 | ******************************************/ 31 | #define PREFIX(name) FL2_error_##name 32 | #define FL2_ERROR(name) ((size_t)-PREFIX(name)) 33 | 34 | 35 | /*-************************************* 36 | * Stream properties 37 | ***************************************/ 38 | #define FL2_PROP_HASH_BIT 7 39 | #define FL2_LZMA_PROP_MASK 0x3FU 40 | #ifndef NO_XXHASH 41 | # define XXHASH_SIZEOF sizeof(XXH32_canonical_t) 42 | #endif 43 | 44 | 45 | /*-************************************* 46 | * Debug 47 | ***************************************/ 48 | #if defined(FL2_DEBUG) && (FL2_DEBUG>=1) 49 | # include 50 | #else 51 | # ifndef assert 52 | # define assert(condition) ((void)0) 53 | # endif 54 | #endif 55 | 56 | #define FL2_STATIC_ASSERT(c) { enum { FL2_static_assert = 1/(int)(!!(c)) }; } 57 | 58 | #if defined(FL2_DEBUG) && (FL2_DEBUG>=2) 59 | # include 60 | extern int g_debuglog_enable; 61 | /* recommended values for FL2_DEBUG display levels : 62 | * 1 : no display, enables assert() only 63 | * 2 : reserved for currently active debugging path 64 | * 3 : events once per object lifetime (CCtx, CDict) 65 | * 4 : events once per frame 66 | * 5 : events once per block 67 | * 6 : events once per sequence (*very* verbose) */ 68 | # define RAWLOG(l, ...) { \ 69 | if ((g_debuglog_enable) & (l<=FL2_DEBUG)) { \ 70 | fprintf(stderr, __VA_ARGS__); \ 71 | } } 72 | # define DEBUGLOG(l, ...) { \ 73 | if ((g_debuglog_enable) & (l<=FL2_DEBUG)) { \ 74 | fprintf(stderr, __FILE__ ": "); \ 75 | fprintf(stderr, __VA_ARGS__); \ 76 | fprintf(stderr, " \n"); \ 77 | } } 78 | #else 79 | # define RAWLOG(l, ...) {} /* disabled */ 80 | # define DEBUGLOG(l, ...) {} /* disabled */ 81 | #endif 82 | 83 | 84 | /*-************************************* 85 | * shared macros 86 | ***************************************/ 87 | #undef MIN 88 | #undef MAX 89 | #define MIN(a,b) ((a)<(b) ? (a) : (b)) 90 | #define MAX(a,b) ((a)>(b) ? (a) : (b)) 91 | #define CHECK_F(f) do { size_t const errcod = f; if (FL2_isError(errcod)) return errcod; } while(0) /* check and Forward error code */ 92 | #define CHECK_E(f, e) do { size_t const errcod = f; if (FL2_isError(errcod)) return FL2_ERROR(e); } while(0) /* check and send Error code */ 93 | 94 | MEM_STATIC U32 ZSTD_highbit32(U32 val) 95 | { 96 | assert(val != 0); 97 | { 98 | # if defined(_MSC_VER) /* Visual */ 99 | unsigned long r=0; 100 | _BitScanReverse(&r, val); 101 | return (unsigned)r; 102 | # elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ 103 | return 31 - __builtin_clz(val); 104 | # else /* Software version */ 105 | static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; 106 | U32 v = val; 107 | int r; 108 | v |= v >> 1; 109 | v |= v >> 2; 110 | v |= v >> 4; 111 | v |= v >> 8; 112 | v |= v >> 16; 113 | r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; 114 | return r; 115 | # endif 116 | } 117 | } 118 | 119 | 120 | #if defined (__cplusplus) 121 | } 122 | #endif 123 | 124 | #endif /* FL2_INTERNAL_H_ */ 125 | -------------------------------------------------------------------------------- /fl2_common.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * Modified for FL2 by Conor McCarthy 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | 13 | 14 | /*-************************************* 15 | * Dependencies 16 | ***************************************/ 17 | #include "fast-lzma2.h" 18 | #include "fl2_errors.h" 19 | #include "fl2_internal.h" 20 | #include "lzma2_enc.h" 21 | 22 | 23 | /*-**************************************** 24 | * Version 25 | ******************************************/ 26 | FL2LIB_API unsigned FL2LIB_CALL FL2_versionNumber(void) { return FL2_VERSION_NUMBER; } 27 | 28 | FL2LIB_API const char* FL2LIB_CALL FL2_versionString(void) { return FL2_VERSION_STRING; } 29 | 30 | 31 | /*-**************************************** 32 | * Compression helpers 33 | ******************************************/ 34 | FL2LIB_API size_t FL2LIB_CALL FL2_compressBound(size_t srcSize) 35 | { 36 | return LZMA2_compressBound(srcSize); 37 | } 38 | 39 | /*-**************************************** 40 | * FL2 Error Management 41 | ******************************************/ 42 | HINT_INLINE 43 | unsigned IsError(size_t code) 44 | { 45 | return (code > FL2_ERROR(maxCode)); 46 | } 47 | 48 | /*! FL2_isError() : 49 | * tells if a return value is an error code */ 50 | FL2LIB_API unsigned FL2LIB_CALL FL2_isError(size_t code) 51 | { 52 | return IsError(code); 53 | } 54 | 55 | /*! FL2_isTimedOut() : 56 | * tells if a return value is the timeout code */ 57 | FL2LIB_API unsigned FL2LIB_CALL FL2_isTimedOut(size_t code) 58 | { 59 | return (code == FL2_ERROR(timedOut)); 60 | } 61 | 62 | /*! FL2_getErrorName() : 63 | * provides error code string from function result (useful for debugging) */ 64 | FL2LIB_API const char* FL2LIB_CALL FL2_getErrorName(size_t code) 65 | { 66 | return FL2_getErrorString(FL2_getErrorCode(code)); 67 | } 68 | 69 | /*! FL2_getError() : 70 | * convert a `size_t` function result into a proper FL2_errorCode enum */ 71 | FL2LIB_API FL2_ErrorCode FL2LIB_CALL FL2_getErrorCode(size_t code) 72 | { 73 | if (!IsError(code)) 74 | return (FL2_ErrorCode)0; 75 | 76 | return (FL2_ErrorCode)(0 - code); 77 | } 78 | 79 | /*! FL2_getErrorString() : 80 | * provides error code string from enum */ 81 | FL2LIB_API const char* FL2LIB_CALL FL2_getErrorString(FL2_ErrorCode code) 82 | { 83 | static const char* const notErrorCode = "Unspecified error code"; 84 | switch (code) 85 | { 86 | case PREFIX(no_error): return "No error detected"; 87 | case PREFIX(GENERIC): return "Error (generic)"; 88 | case PREFIX(internal): return "Internal error (bug)"; 89 | case PREFIX(corruption_detected): return "Corrupted block detected"; 90 | case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; 91 | case PREFIX(parameter_unsupported): return "Unsupported parameter"; 92 | case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; 93 | case PREFIX(lclpMax_exceeded): return "Parameters lc+lp > 4"; 94 | case PREFIX(stage_wrong): return "Not possible at this stage of encoding"; 95 | case PREFIX(init_missing): return "Context should be init first"; 96 | case PREFIX(memory_allocation): return "Allocation error : not enough memory"; 97 | case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; 98 | case PREFIX(srcSize_wrong): return "Src size is incorrect"; 99 | case PREFIX(canceled): return "Processing was canceled by a call to FL2_cancelCStream() or FL2_cancelDStream()"; 100 | case PREFIX(buffer): return "Streaming progress halted due to buffer(s) full/empty"; 101 | case PREFIX(timedOut): return "Wait timed out. Timeouts should be handled before errors using FL2_isTimedOut()"; 102 | /* following error codes are not stable and may be removed or changed in a future version */ 103 | case PREFIX(maxCode): 104 | default: return notErrorCode; 105 | } 106 | } 107 | 108 | /*! g_debuglog_enable : 109 | * turn on/off debug traces (global switch) */ 110 | #if defined(FL2_DEBUG) && (FL2_DEBUG >= 2) 111 | int g_debuglog_enable = 1; 112 | #endif 113 | 114 | -------------------------------------------------------------------------------- /range_enc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Bitwise range encoder by Igor Pavlov 3 | * Modified by Conor McCarthy 4 | * 5 | * Public domain 6 | */ 7 | 8 | #ifndef RANGE_ENCODER_H 9 | #define RANGE_ENCODER_H 10 | 11 | #include "mem.h" 12 | #include "compiler.h" 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | #ifdef LZMA_ENC_PROB32 19 | typedef U32 LZMA2_prob; 20 | #else 21 | typedef U16 LZMA2_prob; 22 | #endif 23 | 24 | #define kNumTopBits 24U 25 | #define kTopValue (1UL << kNumTopBits) 26 | #define kNumBitModelTotalBits 11U 27 | #define kBitModelTotal (1 << kNumBitModelTotalBits) 28 | #define kNumMoveBits 5U 29 | #define kProbInitValue (kBitModelTotal >> 1U) 30 | #define kNumMoveReducingBits 4U 31 | #define kNumBitPriceShiftBits 5U 32 | #define kPriceTableSize (kBitModelTotal >> kNumMoveReducingBits) 33 | 34 | extern BYTE price_table[2][kPriceTableSize]; 35 | #if 0 36 | void RC_printPriceTable(); 37 | #endif 38 | 39 | typedef struct 40 | { 41 | BYTE *out_buffer; 42 | size_t out_index; 43 | U64 cache_size; 44 | U64 low; 45 | U32 range; 46 | BYTE cache; 47 | } RC_encoder; 48 | 49 | void RC_reset(RC_encoder* const rc); 50 | 51 | void RC_setOutputBuffer(RC_encoder* const rc, BYTE *const out_buffer); 52 | 53 | void FORCE_NOINLINE RC_shiftLow(RC_encoder* const rc); 54 | 55 | void RC_encodeBitTree(RC_encoder* const rc, LZMA2_prob *const probs, unsigned bit_count, unsigned symbol); 56 | 57 | void RC_encodeBitTreeReverse(RC_encoder* const rc, LZMA2_prob *const probs, unsigned bit_count, unsigned symbol); 58 | 59 | void FORCE_NOINLINE RC_encodeDirect(RC_encoder* const rc, unsigned value, unsigned bit_count); 60 | 61 | HINT_INLINE 62 | void RC_encodeBit0(RC_encoder* const rc, LZMA2_prob *const rprob) 63 | { 64 | unsigned prob = *rprob; 65 | rc->range = (rc->range >> kNumBitModelTotalBits) * prob; 66 | prob += (kBitModelTotal - prob) >> kNumMoveBits; 67 | *rprob = (LZMA2_prob)prob; 68 | if (rc->range < kTopValue) { 69 | rc->range <<= 8; 70 | RC_shiftLow(rc); 71 | } 72 | } 73 | 74 | HINT_INLINE 75 | void RC_encodeBit1(RC_encoder* const rc, LZMA2_prob *const rprob) 76 | { 77 | unsigned prob = *rprob; 78 | U32 new_bound = (rc->range >> kNumBitModelTotalBits) * prob; 79 | rc->low += new_bound; 80 | rc->range -= new_bound; 81 | prob -= prob >> kNumMoveBits; 82 | *rprob = (LZMA2_prob)prob; 83 | if (rc->range < kTopValue) { 84 | rc->range <<= 8; 85 | RC_shiftLow(rc); 86 | } 87 | } 88 | 89 | HINT_INLINE 90 | void RC_encodeBit(RC_encoder* const rc, LZMA2_prob *const rprob, unsigned const bit) 91 | { 92 | unsigned prob = *rprob; 93 | if (bit != 0) { 94 | U32 const new_bound = (rc->range >> kNumBitModelTotalBits) * prob; 95 | rc->low += new_bound; 96 | rc->range -= new_bound; 97 | prob -= prob >> kNumMoveBits; 98 | } 99 | else { 100 | rc->range = (rc->range >> kNumBitModelTotalBits) * prob; 101 | prob += (kBitModelTotal - prob) >> kNumMoveBits; 102 | } 103 | *rprob = (LZMA2_prob)prob; 104 | if (rc->range < kTopValue) { 105 | rc->range <<= 8; 106 | RC_shiftLow(rc); 107 | } 108 | } 109 | 110 | #define GET_PRICE(prob, symbol) \ 111 | price_table[symbol][(prob) >> kNumMoveReducingBits] 112 | 113 | #define GET_PRICE_0(prob) price_table[0][(prob) >> kNumMoveReducingBits] 114 | 115 | #define GET_PRICE_1(prob) price_table[1][(prob) >> kNumMoveReducingBits] 116 | 117 | #define kMinLitPrice 8U 118 | 119 | HINT_INLINE 120 | unsigned RC_getTreePrice(const LZMA2_prob* const prob_table, unsigned bit_count, size_t symbol) 121 | { 122 | unsigned price = 0; 123 | symbol |= ((size_t)1 << bit_count); 124 | do { 125 | size_t const next_symbol = symbol >> 1; 126 | unsigned prob = prob_table[next_symbol]; 127 | size_t bit = symbol & 1; 128 | price += GET_PRICE(prob, bit); 129 | symbol = next_symbol; 130 | } while (symbol != 1); 131 | return price; 132 | } 133 | 134 | HINT_INLINE 135 | unsigned RC_getReverseTreePrice(const LZMA2_prob* const prob_table, unsigned bit_count, size_t symbol) 136 | { 137 | unsigned prob = prob_table[1]; 138 | size_t bit = symbol & 1; 139 | unsigned price = GET_PRICE(prob, bit); 140 | size_t m = 1; 141 | while (--bit_count != 0) { 142 | m = (m << 1) | bit; 143 | symbol >>= 1; 144 | prob = prob_table[m]; 145 | bit = symbol & 1; 146 | price += GET_PRICE(prob, bit); 147 | } 148 | return price; 149 | } 150 | 151 | HINT_INLINE 152 | void RC_flush(RC_encoder* const rc) 153 | { 154 | for (int i = 0; i < 5; ++i) 155 | RC_shiftLow(rc); 156 | } 157 | 158 | #if defined (__cplusplus) 159 | } 160 | #endif 161 | 162 | #endif /* RANGE_ENCODER_H */ -------------------------------------------------------------------------------- /count.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_COUNT_H_ 12 | #define ZSTD_COUNT_H_ 13 | 14 | #include "mem.h" 15 | 16 | #if defined (__cplusplus) 17 | extern "C" { 18 | #endif 19 | 20 | /*-************************************* 21 | * Match length counter 22 | ***************************************/ 23 | static unsigned ZSTD_NbCommonBytes(register size_t val) 24 | { 25 | if (MEM_isLittleEndian()) { 26 | if (MEM_64bits()) { 27 | # if defined(_MSC_VER) && defined(_WIN64) 28 | unsigned long r = 0; 29 | _BitScanForward64(&r, (U64)val); 30 | return (unsigned)(r >> 3); 31 | # elif defined(__GNUC__) && (__GNUC__ >= 4) 32 | return (__builtin_ctzll((U64)val) >> 3); 33 | # else 34 | static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 35 | 0, 3, 1, 3, 1, 4, 2, 7, 36 | 0, 2, 3, 6, 1, 5, 3, 5, 37 | 1, 3, 4, 4, 2, 5, 6, 7, 38 | 7, 0, 1, 2, 3, 3, 4, 6, 39 | 2, 6, 5, 5, 3, 4, 5, 6, 40 | 7, 1, 2, 4, 6, 4, 4, 5, 41 | 7, 2, 6, 5, 7, 6, 7, 7 }; 42 | return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; 43 | # endif 44 | } 45 | else { /* 32 bits */ 46 | # if defined(_MSC_VER) 47 | unsigned long r = 0; 48 | _BitScanForward(&r, (U32)val); 49 | return (unsigned)(r >> 3); 50 | # elif defined(__GNUC__) && (__GNUC__ >= 3) 51 | return (__builtin_ctz((U32)val) >> 3); 52 | # else 53 | static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 54 | 3, 2, 2, 1, 3, 2, 0, 1, 55 | 3, 3, 1, 2, 2, 2, 2, 0, 56 | 3, 1, 2, 0, 1, 0, 1, 1 }; 57 | return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; 58 | # endif 59 | } 60 | } 61 | else { /* Big Endian CPU */ 62 | if (MEM_64bits()) { 63 | # if defined(_MSC_VER) && defined(_WIN64) 64 | unsigned long r = 0; 65 | _BitScanReverse64(&r, val); 66 | return (unsigned)(r >> 3); 67 | # elif defined(__GNUC__) && (__GNUC__ >= 4) 68 | return (__builtin_clzll(val) >> 3); 69 | # else 70 | unsigned r; 71 | const unsigned n32 = sizeof(size_t) * 4; /* calculate this way due to compiler complaining in 32-bits mode */ 72 | if (!(val >> n32)) { r = 4; } 73 | else { r = 0; val >>= n32; } 74 | if (!(val >> 16)) { r += 2; val >>= 8; } 75 | else { val >>= 24; } 76 | r += (!val); 77 | return r; 78 | # endif 79 | } 80 | else { /* 32 bits */ 81 | # if defined(_MSC_VER) 82 | unsigned long r = 0; 83 | _BitScanReverse(&r, (unsigned long)val); 84 | return (unsigned)(r >> 3); 85 | # elif defined(__GNUC__) && (__GNUC__ >= 3) 86 | return (__builtin_clz((U32)val) >> 3); 87 | # else 88 | unsigned r; 89 | if (!(val >> 16)) { r = 2; val >>= 8; } 90 | else { r = 0; val >>= 24; } 91 | r += (!val); 92 | return r; 93 | # endif 94 | } 95 | } 96 | } 97 | 98 | 99 | static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) 100 | { 101 | const BYTE* const pStart = pIn; 102 | const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t) - 1); 103 | 104 | if (pIn < pInLoopLimit) { 105 | { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); 106 | if (diff) return ZSTD_NbCommonBytes(diff); } 107 | pIn += sizeof(size_t); pMatch += sizeof(size_t); 108 | while (pIn < pInLoopLimit) { 109 | size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); 110 | if (!diff) { pIn += sizeof(size_t); pMatch += sizeof(size_t); continue; } 111 | pIn += ZSTD_NbCommonBytes(diff); 112 | return (size_t)(pIn - pStart); 113 | } 114 | } 115 | if (MEM_64bits() && (pIn<(pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn += 4; pMatch += 4; } 116 | if ((pIn<(pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn += 2; pMatch += 2; } 117 | if ((pIn 2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav 15 | 16 | 17 | 18 | 19 | Source Files 20 | 21 | 22 | Source Files 23 | 24 | 25 | Source Files 26 | 27 | 28 | Source Files 29 | 30 | 31 | Source Files 32 | 33 | 34 | Source Files 35 | 36 | 37 | Source Files 38 | 39 | 40 | Source Files 41 | 42 | 43 | Source Files 44 | 45 | 46 | Source Files 47 | 48 | 49 | Source Files 50 | 51 | 52 | Source Files 53 | 54 | 55 | Source Files 56 | 57 | 58 | Source Files 59 | 60 | 61 | 62 | 63 | Header Files 64 | 65 | 66 | Header Files 67 | 68 | 69 | Header Files 70 | 71 | 72 | Header Files 73 | 74 | 75 | Header Files 76 | 77 | 78 | Header Files 79 | 80 | 81 | Header Files 82 | 83 | 84 | Header Files 85 | 86 | 87 | Header Files 88 | 89 | 90 | Header Files 91 | 92 | 93 | Header Files 94 | 95 | 96 | Header Files 97 | 98 | 99 | Header Files 100 | 101 | 102 | Header Files 103 | 104 | 105 | Header Files 106 | 107 | 108 | Header Files 109 | 110 | 111 | Header Files 112 | 113 | 114 | Header Files 115 | 116 | 117 | Header Files 118 | 119 | 120 | Header Files 121 | 122 | 123 | Header Files 124 | 125 | 126 | Header Files 127 | 128 | 129 | Header Files 130 | 131 | 132 | 133 | 134 | Source Files 135 | 136 | 137 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | The __Fast LZMA2 Library__ is a lossless high-ratio data compression library based on Igor Pavlov's LZMA2 codec from 7-zip. 2 | 3 | Binaries of 7-Zip forks which use the algorithm are available in the [7-Zip-FL2 project], the [7-Zip-zstd project], and the active fork of [p7zip]. The library 4 | is also embedded in a fork of XZ Utils, named [FXZ Utils]. 5 | 6 | [7-Zip-FL2 project]: https://github.com/conor42/7-Zip-FL2/releases/ 7 | [7-Zip-zstd project]: https://github.com/mcmilk/7-Zip-zstd/releases/ 8 | [p7zip]: https://github.com/szcnick/p7zip/releases/ 9 | [FXZ Utils]: https://github.com/conor42/fxz 10 | 11 | The library uses a parallel buffered radix match-finder and some optimizations from Zstandard to achieve a 20% to 100% 12 | speed gain at the higher levels over the default LZMA2 algorithm used in 7-zip, for a small loss in compression ratio. Speed gains 13 | depend on the nature of the source data. The library also uses some threading, portability, and testing code from Zstandard. 14 | 15 | Use of the radix match-finder allows multi-threaded execution employing a simple threading model and with low memory usage. The 16 | library can compress using many threads without dividing the input into large chunks which require the duplication of the 17 | match-finder tables and chains. Extra memory used per thread is typically no more than a few megabytes. 18 | 19 | The largest caveat is that the match-finder is a block algorithm, and to achieve about the same ratio as 7-Zip requires double the 20 | dictionary size, which raises the decompression memory usage. By default it uses the same dictionary size as 7-Zip, resulting in 21 | output that is larger by about 1%-5% of the compressed size. A high-compression option is provided to select parameters which 22 | achieve higher compression on smaller dictionaries. The speed/ratio tradeoff is less optimal with this enabled. 23 | 24 | Here are the results of an in-memory benchmark using two threads on the [Silesia compression corpus] vs the 7-zip 19.00 LZMA2 25 | encoder. The design goal for the encoder and compression level parameters was to move the line as far as possible toward the top 26 | left of the graph. This provides an optimal speed/ratio tradeoff. 27 | 28 | 29 | [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia 30 | 31 | Compression data rate vs ratio 32 | ------------------------------ 33 | ![Compression data rate vs ratio](doc/images/bench_mt2.png "Compression data rate vs ratio") 34 | 35 | ## Build 36 | 37 | ### Windows 38 | 39 | The build\VS folder contains a solution for VS2015. It includes projects for a benchmark program, fuzz tester, file compression 40 | tester, and DLL. 41 | 42 | ### POSIX 43 | 44 | Run `make` in the root directory to build the shared library, then `make install` to allow other programs to use the headers and 45 | libfast-lzma2. Use `make test` to build the file compression tester and run it on a test file. 46 | 47 | The bench, fuzzer and test directories have makefiles for these programs. The CMake file present in earlier releases does not 48 | have an installation script so is not currently included. 49 | 50 | If a build fails on any system please open an issue on github. 51 | 52 | [FXZ Utils] provides a derivative of liblzma from XZ Utils as a wrapper for Fast LZMA2. It is built using GNU autotools. 53 | 54 | ## Status 55 | 56 | The library has passed long periods of fuzz testing, and testing on file sets selected at random in the Radyx file archiver. An 57 | earlier version was released in the 7-Zip forks linked above. The library is considered suitable for production environments. 58 | However, no warranty or fitness for a particular purpose is expressed or implied. 59 | 60 | 61 | Changes in v1.0.1: 62 | 63 | - The root makefile for GNU make now builds and installs a shared library and headers. 64 | - Fixed a potential crash on memory allocation failure during structure allocations for multi-threaded decoding. 65 | - Added a file compression test program. 66 | - Renamed the VS DLL project and some structures / types. 67 | - Removed some duplicated typedefs. 68 | 69 | 70 | Changes in v1.0.0: 71 | 72 | - Breaking changes have been made to the API functions. 73 | - Some of the options have been renamed. 74 | - Optimized the encoder, incorporating some of Igor Pavlov's improvements to 7-Zip 18.05, and some novel ones. The speed increase is 75 | about 5% - 8%. 76 | - Moved detection of repeats from the single-threaded initialization stage to a later, multi-threaded stage to increase speed. 77 | - Removed two compression levels, reducing the total to 10, and tweaked the parameters. 78 | - Improved calculation of the match buffer size. It can still be changed in the options, but the meaning of the value is different. 79 | - Replaced the callbacks for writing and progress with timeouts and new functions to gain direct access to the dictionary buffer and 80 | the compressed data buffers. 81 | - Added Igor Pavlov's assembler-optimized decoder. 82 | - Multi-threaded decompression. 83 | 84 | 85 | Changes in v0.9.2: 86 | 87 | - Fixed excess memory allocation when the dictionary size is > 64Mb 88 | 89 | 90 | Changes in v0.9.1: 91 | 92 | - Fixed a bug in compression of very small files when using a high search depth. 93 | - Added an incompressibility checker which processes high-entropy (e.g. encrypted or already compressed) data about twice as fast 94 | as before. 95 | 96 | ## License 97 | 98 | Fast LZMA2 is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING). 99 | -------------------------------------------------------------------------------- /compiler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * Modified for FL2 by Conor McCarthy 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #ifndef FL2_COMPILER_H 13 | #define FL2_COMPILER_H 14 | 15 | /*-******************************************************* 16 | * Compiler specifics 17 | *********************************************************/ 18 | /* force inlining */ 19 | 20 | #if !defined(FL2_NO_INLINE) 21 | #if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ 22 | # define INLINE_KEYWORD inline 23 | #else 24 | # define INLINE_KEYWORD 25 | #endif 26 | 27 | #if defined(__GNUC__) 28 | # define FORCE_INLINE_ATTR __attribute__((always_inline)) 29 | #elif defined(_MSC_VER) 30 | # define FORCE_INLINE_ATTR __forceinline 31 | #else 32 | # define FORCE_INLINE_ATTR 33 | #endif 34 | 35 | #else 36 | 37 | #define INLINE_KEYWORD 38 | #define FORCE_INLINE_ATTR 39 | 40 | #endif 41 | 42 | /** 43 | * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant 44 | * parameters. They must be inlined for the compiler to eliminate the constant 45 | * branches. 46 | */ 47 | #define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR 48 | /** 49 | * HINT_INLINE is used to help the compiler generate better code. It is *not* 50 | * used for "templates", so it can be tweaked based on the compilers 51 | * performance. 52 | * 53 | * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the 54 | * always_inline attribute. 55 | * 56 | * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline 57 | * attribute. 58 | */ 59 | #if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 60 | # define HINT_INLINE static INLINE_KEYWORD 61 | #else 62 | # define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR 63 | #endif 64 | 65 | /* force no inlining */ 66 | #ifdef _MSC_VER 67 | # define FORCE_NOINLINE __declspec(noinline) 68 | #else 69 | # ifdef __GNUC__ 70 | # define FORCE_NOINLINE __attribute__((__noinline__)) 71 | # else 72 | # define FORCE_NOINLINE 73 | # endif 74 | #endif 75 | 76 | /* target attribute */ 77 | #ifndef __has_attribute 78 | #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ 79 | #endif 80 | #if defined(__GNUC__) 81 | # define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) 82 | #else 83 | # define TARGET_ATTRIBUTE(target) 84 | #endif 85 | 86 | /* Enable runtime BMI2 dispatch based on the CPU. 87 | * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. 88 | */ 89 | #ifndef DYNAMIC_BMI2 90 | #if ((defined(__clang__) && __has_attribute(__target__)) \ 91 | || (defined(__GNUC__) \ 92 | && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ 93 | && (defined(__x86_64__) || defined(_M_X86)) \ 94 | && !defined(__BMI2__) 95 | # define DYNAMIC_BMI2 1 96 | #else 97 | # define DYNAMIC_BMI2 0 98 | #endif 99 | #endif 100 | 101 | /* prefetch 102 | * can be disabled, by declaring NO_PREFETCH build macro */ 103 | #if defined(NO_PREFETCH) 104 | # define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ 105 | # define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ 106 | #else 107 | # if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ 108 | # include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ 109 | # define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) 110 | # define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) 111 | # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) 112 | # define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) 113 | # define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) 114 | # else 115 | # define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ 116 | # define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ 117 | # endif 118 | #endif /* NO_PREFETCH */ 119 | 120 | #define CACHELINE_SIZE 64 121 | 122 | #define PREFETCH_AREA(p, s) { \ 123 | const char* const _ptr = (const char*)(p); \ 124 | size_t const _size = (size_t)(s); \ 125 | size_t _pos; \ 126 | for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ 127 | PREFETCH_L2(_ptr + _pos); \ 128 | } \ 129 | } 130 | 131 | /* disable warnings */ 132 | #ifdef _MSC_VER /* Visual Studio */ 133 | # include /* For Visual 2005 */ 134 | # pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ 135 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ 136 | # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ 137 | # pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ 138 | # pragma warning(disable : 4324) /* disable: C4324: padded structure */ 139 | #endif 140 | 141 | #endif /* FL2_COMPILER_H */ 142 | -------------------------------------------------------------------------------- /test/file_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018, Conor McCarthy 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #include 12 | #include 13 | #include /* Assumes that libfast-lzma2 was installed using 'make install' */ 14 | 15 | static FILE *fin; 16 | static FILE *fout; 17 | static char out_name[4096]; 18 | static FL2_CStream *fcs; 19 | static FL2_DStream *fds; 20 | 21 | static void exit_fail(const char *msg) 22 | { 23 | fputs(msg, stderr); 24 | exit(1); 25 | } 26 | 27 | static int compress_file(FL2_CStream *fcs) 28 | { 29 | unsigned char in_buffer[8 * 1024]; 30 | unsigned char out_buffer[4 * 1024]; 31 | FL2_inBuffer in_buf = { in_buffer, sizeof(in_buffer), sizeof(in_buffer) }; 32 | FL2_outBuffer out_buf = { out_buffer, sizeof(out_buffer), 0 }; 33 | size_t res = 0; 34 | size_t in_size = 0; 35 | size_t out_size = 0; 36 | do { 37 | if (in_buf.pos == in_buf.size) { 38 | in_buf.size = fread(in_buffer, 1, sizeof(in_buffer), fin); 39 | in_size += in_buf.size; 40 | in_buf.pos = 0; 41 | } 42 | res = FL2_compressStream(fcs, &out_buf, &in_buf); 43 | if (FL2_isError(res)) 44 | goto error_out; 45 | 46 | fwrite(out_buf.dst, 1, out_buf.pos, fout); 47 | out_size += out_buf.pos; 48 | out_buf.pos = 0; 49 | 50 | } while (in_buf.size == sizeof(in_buffer)); 51 | do { 52 | res = FL2_endStream(fcs, &out_buf); 53 | if (FL2_isError(res)) 54 | goto error_out; 55 | 56 | fwrite(out_buf.dst, 1, out_buf.pos, fout); 57 | out_size += out_buf.pos; 58 | out_buf.pos = 0; 59 | } while (res); 60 | fprintf(stdout, "\t%ld -> %ld\n", in_size, out_size); 61 | 62 | return 0; 63 | 64 | error_out: 65 | fprintf(stderr, "Error: %s\n", FL2_getErrorName(res)); 66 | return 1; 67 | } 68 | 69 | static int decompress_file(FL2_DStream *fds) 70 | { 71 | unsigned char in_buffer[4 * 1024]; 72 | unsigned char out_buffer[8 * 1024]; 73 | FL2_inBuffer in_buf = { in_buffer, sizeof(in_buffer), sizeof(in_buffer) }; 74 | FL2_outBuffer out_buf = { out_buffer, sizeof(out_buffer), 0 }; 75 | size_t res; 76 | size_t in_size = 0; 77 | size_t out_size = 0; 78 | do { 79 | if (in_buf.pos == in_buf.size) { 80 | in_buf.size = fread(in_buffer, 1, sizeof(in_buffer), fout); 81 | in_size += in_buf.size; 82 | in_buf.pos = 0; 83 | } 84 | res = FL2_decompressStream(fds, &out_buf, &in_buf); 85 | if (FL2_isError(res)) 86 | goto error_out; 87 | /* Discard the output. XXhash will verify the integrity. */ 88 | out_size += out_buf.pos; 89 | out_buf.pos = 0; 90 | } while (res && in_buf.size); 91 | 92 | fprintf(stdout, "\t%ld -> %ld\n", in_size, out_size); 93 | 94 | return 0; 95 | 96 | error_out: 97 | fprintf(stderr, "Error: %s\n", FL2_getErrorName(res)); 98 | return 1; 99 | } 100 | 101 | static void open_files(const char *name) 102 | { 103 | fin = fopen(name, "rb"); 104 | if (fin == NULL) 105 | exit_fail("Cannot open input file.\n"); 106 | 107 | snprintf(out_name, sizeof(out_name), "%s.lzma2", name); 108 | out_name[sizeof(out_name) - 1] = '\0'; 109 | 110 | fout = fopen(out_name, "w+b"); 111 | if (fout == NULL) 112 | exit_fail("Cannot open output file.\n"); 113 | } 114 | 115 | static void create_init_fl2_streams(int preset) 116 | { 117 | fcs = FL2_createCStreamMt(2, 0); 118 | if (fcs == NULL) 119 | exit_fail("Cannot allocate compression context.\n"); 120 | 121 | fds = FL2_createDStreamMt(2); 122 | if (fds == NULL) 123 | exit_fail("Cannot allocate decompression context.\n"); 124 | 125 | size_t res = FL2_initCStream(fcs, preset); 126 | if(!res) 127 | res = FL2_initDStream(fds); 128 | if (FL2_isError(res)) { 129 | fprintf(stderr, "Error: %s\n", FL2_getErrorName(res)); 130 | exit(1); 131 | } 132 | } 133 | 134 | int main(int argc, char **argv) 135 | { 136 | int ret = 1; 137 | static const char *usage = "Usage: file_test [-1..10] FILE\n"; 138 | 139 | if (argc < 2) 140 | exit_fail(usage); 141 | 142 | int preset = 6; 143 | for (int i = 1; i < argc; ++i) 144 | if (argv[i][0] == '-') 145 | preset = atoi(argv[i] + 1); 146 | 147 | const char *name = NULL; 148 | for (int i = 1; i < argc; ++i) 149 | if (argv[i][0] != '-') { 150 | name = argv[i]; 151 | break; 152 | } 153 | if (name == NULL) 154 | exit_fail(usage); 155 | 156 | create_init_fl2_streams(preset); 157 | open_files(name); 158 | fprintf(stdout, "Compress %s to %s:\n", name, out_name); 159 | 160 | if (compress_file(fcs)) 161 | goto cleanup; 162 | 163 | fseek(fout, 0, SEEK_SET); 164 | fprintf(stdout, "Decompress %s (Discard the output. XXhash will verify the integrity.):\n", out_name); 165 | ret = decompress_file(fds); 166 | if(ret == 0){ 167 | fprintf(stdout, "Compress & decompress SUCCESS.\n"); 168 | } 169 | cleanup: 170 | fclose(fout); 171 | fclose(fin); 172 | remove(out_name); 173 | FL2_freeCStream(fcs); 174 | FL2_freeDStream(fds); 175 | return ret; 176 | } 177 | -------------------------------------------------------------------------------- /fl2_threading.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2016 Tino Reichardt 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * 9 | * You can contact the author at: 10 | * - zstdmt source repository: https://github.com/mcmilk/zstdmt 11 | */ 12 | 13 | #ifndef THREADING_H_938743 14 | #define THREADING_H_938743 15 | 16 | #include "mem.h" 17 | 18 | #ifndef FL2_XZ_BUILD 19 | # ifdef _WIN32 20 | # define MYTHREAD_VISTA 21 | # else 22 | # define MYTHREAD_POSIX /* posix assumed ; need a better detection method */ 23 | # endif 24 | #elif defined(HAVE_CONFIG_H) 25 | # include 26 | #endif 27 | 28 | #if defined (__cplusplus) 29 | extern "C" { 30 | #endif 31 | 32 | unsigned FL2_checkNbThreads(unsigned nbThreads); 33 | 34 | 35 | #if !defined(FL2_SINGLETHREAD) && defined(MYTHREAD_VISTA) 36 | 37 | /** 38 | * Windows minimalist Pthread Wrapper, based on : 39 | * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html 40 | */ 41 | #ifdef WINVER 42 | # undef WINVER 43 | #endif 44 | #define WINVER 0x0600 45 | 46 | #ifdef _WIN32_WINNT 47 | # undef _WIN32_WINNT 48 | #endif 49 | #define _WIN32_WINNT 0x0600 50 | 51 | #ifndef WIN32_LEAN_AND_MEAN 52 | # define WIN32_LEAN_AND_MEAN 53 | #endif 54 | 55 | #include 56 | #include 57 | 58 | 59 | /* mutex */ 60 | #define FL2_pthread_mutex_t CRITICAL_SECTION 61 | #define FL2_pthread_mutex_init(a, b) (InitializeCriticalSection((a)), 0) 62 | #define FL2_pthread_mutex_destroy(a) DeleteCriticalSection((a)) 63 | #define FL2_pthread_mutex_lock(a) EnterCriticalSection((a)) 64 | #define FL2_pthread_mutex_unlock(a) LeaveCriticalSection((a)) 65 | 66 | /* condition variable */ 67 | #define FL2_pthread_cond_t CONDITION_VARIABLE 68 | #define FL2_pthread_cond_init(a, b) (InitializeConditionVariable((a)), 0) 69 | #define FL2_pthread_cond_destroy(a) /* No delete */ 70 | #define FL2_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE) 71 | #define FL2_pthread_cond_timedwait(a, b, c) SleepConditionVariableCS((a), (b), (c)) 72 | #define FL2_pthread_cond_signal(a) WakeConditionVariable((a)) 73 | #define FL2_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) 74 | 75 | /* FL2_pthread_create() and FL2_pthread_join() */ 76 | typedef struct { 77 | HANDLE handle; 78 | void* (*start_routine)(void*); 79 | void* arg; 80 | } FL2_pthread_t; 81 | 82 | int FL2_pthread_create(FL2_pthread_t* thread, const void* unused, 83 | void* (*start_routine) (void*), void* arg); 84 | 85 | int FL2_pthread_join(FL2_pthread_t thread, void** value_ptr); 86 | 87 | /** 88 | * add here more wrappers as required 89 | */ 90 | 91 | 92 | #elif !defined(FL2_SINGLETHREAD) && defined(MYTHREAD_POSIX) 93 | /* === POSIX Systems === */ 94 | # include 95 | # include 96 | 97 | #define FL2_pthread_mutex_t pthread_mutex_t 98 | #define FL2_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) 99 | #define FL2_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) 100 | #define FL2_pthread_mutex_lock(a) pthread_mutex_lock((a)) 101 | #define FL2_pthread_mutex_unlock(a) pthread_mutex_unlock((a)) 102 | 103 | #define FL2_pthread_cond_t pthread_cond_t 104 | #define FL2_pthread_cond_init(a, b) pthread_cond_init((a), (b)) 105 | #define FL2_pthread_cond_destroy(a) pthread_cond_destroy((a)) 106 | #define FL2_pthread_cond_wait(a, b) pthread_cond_wait((a), (b)) 107 | #define FL2_pthread_cond_signal(a) pthread_cond_signal((a)) 108 | #define FL2_pthread_cond_broadcast(a) pthread_cond_broadcast((a)) 109 | 110 | #define FL2_pthread_t pthread_t 111 | #define FL2_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) 112 | #define FL2_pthread_join(a, b) pthread_join((a),(b)) 113 | 114 | /* Timed wait functions from XZ by Lasse Collin 115 | */ 116 | 117 | /* Sets condtime to the absolute time that is timeout_ms milliseconds 118 | * in the future. 119 | */ 120 | static inline void 121 | mythread_condtime_set(struct timespec *condtime, U32 timeout_ms) 122 | { 123 | condtime->tv_sec = timeout_ms / 1000; 124 | condtime->tv_nsec = (timeout_ms % 1000) * 1000000; 125 | 126 | struct timeval now; 127 | gettimeofday(&now, NULL); 128 | 129 | condtime->tv_sec += now.tv_sec; 130 | condtime->tv_nsec += now.tv_usec * 1000L; 131 | 132 | /* tv_nsec must stay in the range [0, 999_999_999]. */ 133 | if (condtime->tv_nsec >= 1000000000L) { 134 | condtime->tv_nsec -= 1000000000L; 135 | ++condtime->tv_sec; 136 | } 137 | } 138 | 139 | /* Waits on a condition or until a timeout expires. If the timeout expires, 140 | * non-zero is returned, otherwise zero is returned. 141 | */ 142 | static inline void 143 | FL2_pthread_cond_timedwait(FL2_pthread_cond_t *cond, FL2_pthread_mutex_t *mutex, 144 | U32 timeout_ms) 145 | { 146 | struct timespec condtime; 147 | mythread_condtime_set(&condtime, timeout_ms); 148 | pthread_cond_timedwait(cond, mutex, &condtime); 149 | } 150 | 151 | 152 | #elif defined(FL2_SINGLETHREAD) 153 | /* No multithreading support */ 154 | 155 | typedef int FL2_pthread_mutex_t; 156 | #define FL2_pthread_mutex_init(a, b) ((void)a, 0) 157 | #define FL2_pthread_mutex_destroy(a) 158 | #define FL2_pthread_mutex_lock(a) 159 | #define FL2_pthread_mutex_unlock(a) 160 | 161 | typedef int FL2_pthread_cond_t; 162 | #define FL2_pthread_cond_init(a, b) ((void)a, 0) 163 | #define FL2_pthread_cond_destroy(a) 164 | #define FL2_pthread_cond_wait(a, b) 165 | #define FL2_pthread_cond_signal(a) 166 | #define FL2_pthread_cond_broadcast(a) 167 | 168 | /* do not use FL2_pthread_t */ 169 | 170 | #else 171 | # error FL2_SINGLETHREAD not defined but no threading support found 172 | #endif /* FL2_SINGLETHREAD */ 173 | 174 | #if defined (__cplusplus) 175 | } 176 | #endif 177 | 178 | #endif /* THREADING_H_938743 */ 179 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.suo 8 | *.user 9 | *.userosscache 10 | *.sln.docstates 11 | 12 | # User-specific files (MonoDevelop/Xamarin Studio) 13 | *.userprefs 14 | 15 | # Build results 16 | [Dd]ebug/ 17 | [Dd]ebugPublic/ 18 | [Rr]elease/ 19 | [Rr]eleases/ 20 | x64/ 21 | x86/ 22 | bld/ 23 | [Bb]in/ 24 | [Oo]bj/ 25 | [Ll]og/ 26 | 27 | # Visual Studio 2015 cache/options directory 28 | .vs/ 29 | # Uncomment if you have tasks that create the project's static files in wwwroot 30 | #wwwroot/ 31 | 32 | # MSTest test Results 33 | [Tt]est[Rr]esult*/ 34 | [Bb]uild[Ll]og.* 35 | 36 | # NUNIT 37 | *.VisualState.xml 38 | TestResult.xml 39 | 40 | # Build Results of an ATL Project 41 | [Dd]ebugPS/ 42 | [Rr]eleasePS/ 43 | dlldata.c 44 | 45 | # Benchmark Results 46 | BenchmarkDotNet.Artifacts/ 47 | 48 | # .NET Core 49 | project.lock.json 50 | project.fragment.lock.json 51 | artifacts/ 52 | **/Properties/launchSettings.json 53 | 54 | *_i.c 55 | *_p.c 56 | *_i.h 57 | *.ilk 58 | *.meta 59 | *.obj 60 | *.pch 61 | *.pdb 62 | *.pgc 63 | *.pgd 64 | *.rsp 65 | *.sbr 66 | *.tlb 67 | *.tli 68 | *.tlh 69 | *.tmp 70 | *.tmp_proj 71 | *.log 72 | *.vspscc 73 | *.vssscc 74 | .builds 75 | *.pidb 76 | *.svclog 77 | *.scc 78 | *.dll 79 | *.exe 80 | *.o 81 | 82 | # Chutzpah Test files 83 | _Chutzpah* 84 | 85 | # Visual C++ cache files 86 | ipch/ 87 | *.aps 88 | *.ncb 89 | *.opendb 90 | *.opensdf 91 | *.sdf 92 | *.cachefile 93 | *.VC.db 94 | *.VC.VC.opendb 95 | 96 | # Visual Studio profiler 97 | *.psess 98 | *.vsp 99 | *.vspx 100 | *.sap 101 | 102 | # TFS 2012 Local Workspace 103 | $tf/ 104 | 105 | # Guidance Automation Toolkit 106 | *.gpState 107 | 108 | # ReSharper is a .NET coding add-in 109 | _ReSharper*/ 110 | *.[Rr]e[Ss]harper 111 | *.DotSettings.user 112 | 113 | # JustCode is a .NET coding add-in 114 | .JustCode 115 | 116 | # TeamCity is a build add-in 117 | _TeamCity* 118 | 119 | # DotCover is a Code Coverage Tool 120 | *.dotCover 121 | 122 | # AxoCover is a Code Coverage Tool 123 | .axoCover/* 124 | !.axoCover/settings.json 125 | 126 | # Visual Studio code coverage results 127 | *.coverage 128 | *.coveragexml 129 | 130 | # NCrunch 131 | _NCrunch_* 132 | .*crunch*.local.xml 133 | nCrunchTemp_* 134 | 135 | # MightyMoose 136 | *.mm.* 137 | AutoTest.Net/ 138 | 139 | # Web workbench (sass) 140 | .sass-cache/ 141 | 142 | # Installshield output folder 143 | [Ee]xpress/ 144 | 145 | # DocProject is a documentation generator add-in 146 | DocProject/buildhelp/ 147 | DocProject/Help/*.HxT 148 | DocProject/Help/*.HxC 149 | DocProject/Help/*.hhc 150 | DocProject/Help/*.hhk 151 | DocProject/Help/*.hhp 152 | DocProject/Help/Html2 153 | DocProject/Help/html 154 | 155 | # Click-Once directory 156 | publish/ 157 | 158 | # Publish Web Output 159 | *.[Pp]ublish.xml 160 | *.azurePubxml 161 | # Note: Comment the next line if you want to checkin your web deploy settings, 162 | # but database connection strings (with potential passwords) will be unencrypted 163 | *.pubxml 164 | *.publishproj 165 | 166 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 167 | # checkin your Azure Web App publish settings, but sensitive information contained 168 | # in these scripts will be unencrypted 169 | PublishScripts/ 170 | 171 | # NuGet Packages 172 | *.nupkg 173 | # The packages folder can be ignored because of Package Restore 174 | **/packages/* 175 | # except build/, which is used as an MSBuild target. 176 | !**/packages/build/ 177 | # Uncomment if necessary however generally it will be regenerated when needed 178 | #!**/packages/repositories.config 179 | # NuGet v3's project.json files produces more ignorable files 180 | *.nuget.props 181 | *.nuget.targets 182 | 183 | # Microsoft Azure Build Output 184 | csx/ 185 | *.build.csdef 186 | 187 | # Microsoft Azure Emulator 188 | ecf/ 189 | rcf/ 190 | 191 | # Windows Store app package directories and files 192 | AppPackages/ 193 | BundleArtifacts/ 194 | Package.StoreAssociation.xml 195 | _pkginfo.txt 196 | *.appx 197 | 198 | # Visual Studio cache files 199 | # files ending in .cache can be ignored 200 | *.[Cc]ache 201 | # but keep track of directories ending in .cache 202 | !*.[Cc]ache/ 203 | 204 | # Others 205 | ClientBin/ 206 | ~$* 207 | *~ 208 | *.dbmdl 209 | *.dbproj.schemaview 210 | *.jfm 211 | *.pfx 212 | *.publishsettings 213 | orleans.codegen.cs 214 | 215 | # Since there are multiple workflows, uncomment next line to ignore bower_components 216 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 217 | #bower_components/ 218 | 219 | # RIA/Silverlight projects 220 | Generated_Code/ 221 | 222 | # Backup & report files from converting an old project file 223 | # to a newer Visual Studio version. Backup files are not needed, 224 | # because we have git ;-) 225 | _UpgradeReport_Files/ 226 | Backup*/ 227 | UpgradeLog*.XML 228 | UpgradeLog*.htm 229 | 230 | # SQL Server files 231 | *.mdf 232 | *.ldf 233 | *.ndf 234 | 235 | # Business Intelligence projects 236 | *.rdl.data 237 | *.bim.layout 238 | *.bim_*.settings 239 | 240 | # Microsoft Fakes 241 | FakesAssemblies/ 242 | 243 | # GhostDoc plugin setting file 244 | *.GhostDoc.xml 245 | 246 | # Node.js Tools for Visual Studio 247 | .ntvs_analysis.dat 248 | node_modules/ 249 | 250 | # Typescript v1 declaration files 251 | typings/ 252 | 253 | # Visual Studio 6 build log 254 | *.plg 255 | 256 | # Visual Studio 6 workspace options file 257 | *.opt 258 | 259 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 260 | *.vbw 261 | 262 | # Visual Studio LightSwitch build output 263 | **/*.HTMLClient/GeneratedArtifacts 264 | **/*.DesktopClient/GeneratedArtifacts 265 | **/*.DesktopClient/ModelManifest.xml 266 | **/*.Server/GeneratedArtifacts 267 | **/*.Server/ModelManifest.xml 268 | _Pvt_Extensions 269 | 270 | # Paket dependency manager 271 | .paket/paket.exe 272 | paket-files/ 273 | 274 | # FAKE - F# Make 275 | .fake/ 276 | 277 | # JetBrains Rider 278 | .idea/ 279 | *.sln.iml 280 | 281 | # CodeRush 282 | .cr/ 283 | 284 | # Python Tools for Visual Studio (PTVS) 285 | __pycache__/ 286 | *.pyc 287 | 288 | # Cake - Uncomment if you are using it 289 | # tools/** 290 | # !tools/packages.config 291 | 292 | # Tabs Studio 293 | *.tss 294 | 295 | # Telerik's JustMock configuration file 296 | *.jmconfig 297 | 298 | # BizTalk build output 299 | *.btp.cs 300 | *.btm.cs 301 | *.odx.cs 302 | *.xsd.cs 303 | 304 | *.map 305 | *.d 306 | -------------------------------------------------------------------------------- /lzma2_dec.h: -------------------------------------------------------------------------------- 1 | /* lzma2_dec.h -- LZMA2 Decoder 2 | 2017-04-03 : Igor Pavlov : Public domain 3 | Modified for FL2 by Conor McCarthy */ 4 | 5 | #ifndef __LZMA_DEC_H 6 | #define __LZMA_DEC_H 7 | 8 | #include "mem.h" 9 | 10 | #if defined (__cplusplus) 11 | extern "C" { 12 | #endif 13 | 14 | /* #define LZMA_DEC_PROB16 */ 15 | /* 32-bit probs can increase the speed on some CPUs, 16 | but memory usage for LZMA2_DCtx::probs will be doubled in that case */ 17 | 18 | #ifdef LZMA_DEC_PROB16 19 | typedef U16 LZMA2_prob; 20 | #else 21 | typedef U32 LZMA2_prob; 22 | #endif 23 | 24 | 25 | /* ---------- LZMA Properties ---------- */ 26 | 27 | typedef struct 28 | { 29 | BYTE lc; 30 | BYTE lp; 31 | BYTE pb; 32 | BYTE pad_; 33 | U32 dic_size; 34 | } LZMA2_props; 35 | 36 | /* LzmaProps_Decode - decodes properties 37 | Returns: 38 | SZ_OK 39 | SZ_ERROR_UNSUPPORTED - Unsupported properties 40 | */ 41 | 42 | 43 | /* ---------- LZMA Decoder state ---------- */ 44 | 45 | /* LZMA_REQUIRED_INPUT_MAX = number of required input bytes for worst case. 46 | Num bits = log2((2^11 / 31) ^ 22) + 26 < 134 + 26 = 160; */ 47 | 48 | #define LZMA_REQUIRED_INPUT_MAX 20 49 | 50 | #define kNumPosBitsMax 4 51 | #define kNumPosStatesMax (1 << kNumPosBitsMax) 52 | 53 | #define kLenNumLowBits 3 54 | #define kLenNumLowSymbols (1 << kLenNumLowBits) 55 | #define kLenNumHighBits 8 56 | #define kLenNumHighSymbols (1 << kLenNumHighBits) 57 | 58 | #define LenLow 0 59 | #define LenHigh (LenLow + 2 * (kNumPosStatesMax << kLenNumLowBits)) 60 | #define kNumLenProbs (LenHigh + kLenNumHighSymbols) 61 | 62 | #define LenChoice LenLow 63 | #define LenChoice2 (LenLow + (1 << kLenNumLowBits)) 64 | 65 | #define kNumStates 12 66 | #define kNumStates2 16 67 | #define kNumLitStates 7 68 | 69 | #define kStartPosModelIndex 4 70 | #define kEndPosModelIndex 14 71 | #define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) 72 | 73 | #define kNumPosSlotBits 6 74 | #define kNumLenToPosStates 4 75 | 76 | #define kNumAlignBits 4 77 | #define kAlignTableSize (1 << kNumAlignBits) 78 | 79 | #define kMatchMinLen 2 80 | #define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) 81 | 82 | /* External ASM code needs same CLzmaProb array layout. So don't change it. */ 83 | 84 | /* (probs_1664) is faster and better for code size at some platforms */ 85 | /* 86 | #ifdef MY_CPU_X86_OR_AMD64 87 | */ 88 | #define kStartOffset 1664 89 | #define GET_PROBS p->probs_1664 90 | /* 91 | #define GET_PROBS p->probs + kStartOffset 92 | #else 93 | #define kStartOffset 0 94 | #define GET_PROBS p->probs 95 | #endif 96 | */ 97 | 98 | #define SpecPos (-kStartOffset) 99 | #define IsRep0Long (SpecPos + kNumFullDistances) 100 | #define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) 101 | #define LenCoder (RepLenCoder + kNumLenProbs) 102 | #define IsMatch (LenCoder + kNumLenProbs) 103 | #define Align (IsMatch + (kNumStates2 << kNumPosBitsMax)) 104 | #define IsRep (Align + kAlignTableSize) 105 | #define IsRepG0 (IsRep + kNumStates) 106 | #define IsRepG1 (IsRepG0 + kNumStates) 107 | #define IsRepG2 (IsRepG1 + kNumStates) 108 | #define PosSlot (IsRepG2 + kNumStates) 109 | #define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) 110 | #define NUM_BASE_PROBS (Literal + kStartOffset) 111 | 112 | #if Align != 0 && kStartOffset != 0 113 | #error Stop_Compiling_Bad_LZMA_kAlign 114 | #endif 115 | 116 | #if NUM_BASE_PROBS != 1984 117 | #error Stop_Compiling_Bad_LZMA_PROBS 118 | #endif 119 | 120 | 121 | #define kLzmaLitSize 0x300 122 | 123 | #define LzmaProps_GetNumProbs(p) (NUM_BASE_PROBS + ((U32)kLzmaLitSize << ((p)->lc + (p)->lp))) 124 | 125 | 126 | #define CALC_POS_STATE(processed_pos, pb_mask) (((processed_pos) & (pb_mask)) << 4) 127 | #define COMBINED_PS_STATE (pos_state + state) 128 | #define GET_LEN_STATE (pos_state) 129 | 130 | #define kLzma2LcLpMax 4U 131 | 132 | 133 | typedef struct 134 | { 135 | LZMA2_props prop; 136 | BYTE *dic; 137 | size_t dic_pos; 138 | size_t dic_buf_size; 139 | const BYTE *buf; 140 | LZMA2_prob *probs_1664; 141 | U32 range; 142 | U32 code; 143 | U32 processed_pos; 144 | U32 check_dic_size; 145 | U32 reps[4]; 146 | unsigned state; 147 | unsigned state2; 148 | unsigned remain_len; 149 | size_t pack_size; 150 | size_t unpack_size; 151 | BYTE control; 152 | BYTE need_init_dic; 153 | BYTE need_init_state; 154 | BYTE need_init_state2; 155 | BYTE need_init_prop; 156 | BYTE need_flush; 157 | BYTE ext_dic; 158 | BYTE pad_; 159 | LZMA2_prob probs[NUM_BASE_PROBS + ((U32)kLzmaLitSize << kLzma2LcLpMax)]; 160 | } LZMA2_DCtx; 161 | 162 | void LZMA_constructDCtx(LZMA2_DCtx *p); 163 | 164 | typedef enum 165 | { 166 | LZMA_FINISH_ANY, /* finish at any point */ 167 | LZMA_FINISH_END /* block must be finished at the end */ 168 | } LZMA2_finishMode; 169 | 170 | typedef enum 171 | { 172 | LZMA_STATUS_NOT_SPECIFIED, /* use main error code instead */ 173 | LZMA_STATUS_FINISHED, /* stream was finished */ 174 | LZMA_STATUS_NOT_FINISHED, /* stream was not finished */ 175 | LZMA_STATUS_NEEDS_MORE_INPUT, /* you must provide more input bytes */ 176 | LZMA_STATUS_OUTPUT_FULL /* not finished; output buffer is full */ 177 | } LZMA2_status; 178 | 179 | void LZMA_destructDCtx(LZMA2_DCtx *const p); 180 | 181 | size_t LZMA2_getDictSizeFromProp(BYTE const dict_prop); 182 | 183 | #define LZMA2_CONTENTSIZE_ERROR (size_t)-1 184 | 185 | U64 LZMA2_getUnpackSize(const BYTE *const src, size_t const src_len); 186 | 187 | size_t LZMA2_decMemoryUsage(size_t const dict_size); 188 | 189 | size_t LZMA2_initDecoder(LZMA2_DCtx *const p, BYTE const dict_prop, BYTE *const dic, size_t dic_buf_size); 190 | 191 | size_t LZMA2_decodeToDic(LZMA2_DCtx *const p, size_t const dic_limit, 192 | const BYTE *const src, size_t *const src_len, LZMA2_finishMode const finish_mode); 193 | 194 | size_t LZMA2_decodeToBuf(LZMA2_DCtx *const p, BYTE *dest, size_t *const dest_len, 195 | const BYTE *src, size_t *const src_len, LZMA2_finishMode const finish_mode); 196 | 197 | typedef enum 198 | { 199 | CHUNK_MORE_DATA, 200 | CHUNK_CONTINUE, 201 | CHUNK_DICT_RESET, 202 | CHUNK_FINAL, 203 | CHUNK_ERROR 204 | } LZMA2_parseRes; 205 | 206 | typedef struct 207 | { 208 | size_t pack_size; 209 | size_t unpack_size; 210 | } LZMA2_chunk; 211 | 212 | #if defined(FL2_DEBUG) && (FL2_DEBUG>=1) 213 | # define LZMA2_MT_INPUT_SIZE 0x400 214 | #else 215 | # define LZMA2_MT_INPUT_SIZE 0x40000 216 | #endif 217 | 218 | LZMA2_parseRes LZMA2_parseInput(const BYTE* const in_buf, size_t const pos, ptrdiff_t const len, LZMA2_chunk *const inf); 219 | 220 | #if defined (__cplusplus) 221 | } 222 | #endif 223 | 224 | #endif 225 | -------------------------------------------------------------------------------- /fuzzer/datagen.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | 12 | 13 | /*-************************************ 14 | * Dependencies 15 | **************************************/ 16 | #include "../platform.h" /* SET_BINARY_MODE */ 17 | #include /* malloc, free */ 18 | #include /* FILE, fwrite, fprintf */ 19 | #include /* memcpy */ 20 | #include "../mem.h" /* U32 */ 21 | 22 | 23 | /*-************************************ 24 | * Macros 25 | **************************************/ 26 | #define KB *(1 <<10) 27 | #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) 28 | 29 | #define RDG_DEBUG 0 30 | #define TRACE(...) if (RDG_DEBUG) fprintf(stderr, __VA_ARGS__ ) 31 | 32 | 33 | /*-************************************ 34 | * Local constants 35 | **************************************/ 36 | #define LTLOG 13 37 | #define LTSIZE (1<> (32 - r))) 45 | static U32 RDG_rand(U32* src) 46 | { 47 | static const U32 prime1 = 2654435761U; 48 | static const U32 prime2 = 2246822519U; 49 | U32 rand32 = *src; 50 | rand32 *= prime1; 51 | rand32 ^= prime2; 52 | rand32 = RDG_rotl32(rand32, 13); 53 | *src = rand32; 54 | return rand32 >> 5; 55 | } 56 | 57 | 58 | static void RDG_fillLiteralDistrib(BYTE* ldt, double ld) 59 | { 60 | BYTE const firstChar = (ld<=0.0) ? 0 : '('; 61 | BYTE const lastChar = (ld<=0.0) ? 255 : '}'; 62 | BYTE character = (ld<=0.0) ? 0 : '0'; 63 | U32 u; 64 | 65 | if (ld<=0.0) ld = 0.0; 66 | for (u=0; u lastChar) character = firstChar; 72 | } 73 | } 74 | 75 | 76 | static BYTE RDG_genChar(U32* seed, const BYTE* ldt) 77 | { 78 | U32 const id = RDG_rand(seed) & LTMASK; 79 | return ldt[id]; /* memory-sanitizer fails here, stating "uninitialized value" when table initialized with P==0.0. Checked : table is fully initialized */ 80 | } 81 | 82 | 83 | static U32 RDG_rand15Bits (unsigned* seedPtr) 84 | { 85 | return RDG_rand(seedPtr) & 0x7FFF; 86 | } 87 | 88 | static U32 RDG_randLength(unsigned* seedPtr) 89 | { 90 | if (RDG_rand(seedPtr) & 7) return (RDG_rand(seedPtr) & 0xF); /* small length */ 91 | return (RDG_rand(seedPtr) & 0x1FF) + 0xF; 92 | } 93 | 94 | void RDG_genBlock(void* buffer, size_t buffSize, size_t prefixSize, double matchProba, const BYTE* ldt, unsigned* seedPtr) 95 | { 96 | BYTE* const buffPtr = (BYTE*)buffer; 97 | U32 const matchProba32 = (U32)(32768 * matchProba); 98 | size_t pos = prefixSize; 99 | U32 prevOffset = 1; 100 | 101 | /* special case : sparse content */ 102 | while (matchProba >= 1.0) { 103 | size_t size0 = RDG_rand(seedPtr) & 3; 104 | size0 = (size_t)1 << (16 + size0 * 2); 105 | size0 += RDG_rand(seedPtr) & (size0-1); /* because size0 is power of 2*/ 106 | if (buffSize < pos + size0) { 107 | memset(buffPtr+pos, 0, buffSize-pos); 108 | return; 109 | } 110 | memset(buffPtr+pos, 0, size0); 111 | pos += size0; 112 | buffPtr[pos-1] = RDG_genChar(seedPtr, ldt); 113 | continue; 114 | } 115 | 116 | /* init */ 117 | if (pos==0) buffPtr[0] = RDG_genChar(seedPtr, ldt), pos=1; 118 | 119 | /* Generate compressible data */ 120 | while (pos < buffSize) { 121 | /* Select : Literal (char) or Match (within 32K) */ 122 | if (RDG_rand15Bits(seedPtr) < matchProba32) { 123 | /* Copy (within 32K) */ 124 | U32 const length = RDG_randLength(seedPtr) + 4; 125 | U32 const d = (U32) MIN(pos + length , buffSize); 126 | U32 const repeatOffset = (RDG_rand(seedPtr) & 15) == 2; 127 | U32 const randOffset = RDG_rand15Bits(seedPtr) + 1; 128 | U32 const offset = repeatOffset ? prevOffset : (U32) MIN(randOffset , pos); 129 | size_t match = pos - offset; 130 | while (pos < d) buffPtr[pos++] = buffPtr[match++]; /* correctly manages overlaps */ 131 | prevOffset = offset; 132 | } else { 133 | /* Literal (noise) */ 134 | U32 const length = RDG_randLength(seedPtr); 135 | U32 const d = (U32) MIN(pos + length, buffSize); 136 | while (pos < d) buffPtr[pos++] = RDG_genChar(seedPtr, ldt); 137 | } } 138 | } 139 | 140 | 141 | void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed) 142 | { 143 | BYTE ldt[LTSIZE]; 144 | memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */ 145 | if (litProba<=0.0) litProba = matchProba / 4.5; 146 | RDG_fillLiteralDistrib(ldt, litProba); 147 | RDG_genBlock(buffer, size, 0, matchProba, ldt, &seed); 148 | } 149 | 150 | 151 | void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed) 152 | { 153 | size_t const stdBlockSize = 128 KB; 154 | size_t const stdDictSize = 32 KB; 155 | BYTE* const buff = (BYTE*)malloc(stdDictSize + stdBlockSize); 156 | U64 total = 0; 157 | BYTE ldt[LTSIZE]; /* literals distribution table */ 158 | 159 | /* init */ 160 | if (buff==NULL) { perror("datagen"); exit(1); } 161 | if (litProba<=0.0) litProba = matchProba / 4.5; 162 | memset(ldt, '0', sizeof(ldt)); /* yes, character '0', this is intentional */ 163 | RDG_fillLiteralDistrib(ldt, litProba); 164 | SET_BINARY_MODE(stdout); 165 | 166 | /* Generate initial dict */ 167 | RDG_genBlock(buff, stdDictSize, 0, matchProba, ldt, &seed); 168 | 169 | /* Generate compressible data */ 170 | while (total < size) { 171 | size_t const genBlockSize = (size_t) (MIN (stdBlockSize, size-total)); 172 | RDG_genBlock(buff, stdDictSize+stdBlockSize, stdDictSize, matchProba, ldt, &seed); 173 | total += genBlockSize; 174 | { size_t const unused = fwrite(buff, 1, genBlockSize, stdout); (void)unused; } 175 | /* update dict */ 176 | memcpy(buff, buff + stdBlockSize, stdDictSize); 177 | } 178 | 179 | /* cleanup */ 180 | free(buff); 181 | } 182 | -------------------------------------------------------------------------------- /radix_get.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2018, Conor McCarthy 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef FL2_RADIX_GET_H_ 12 | #define FL2_RADIX_GET_H_ 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | typedef struct 19 | { 20 | U32 length; 21 | U32 dist; 22 | } RMF_match; 23 | 24 | static size_t RMF_bitpackExtendMatch(const BYTE* const data, 25 | const U32* const table, 26 | ptrdiff_t const start_index, 27 | ptrdiff_t limit, 28 | U32 const link, 29 | size_t const length) 30 | { 31 | ptrdiff_t end_index = start_index + length; 32 | ptrdiff_t const dist = start_index - link; 33 | 34 | if (limit > start_index + (ptrdiff_t)kMatchLenMax) 35 | limit = start_index + kMatchLenMax; 36 | 37 | while (end_index < limit && end_index - (ptrdiff_t)(table[end_index] & RADIX_LINK_MASK) == dist) 38 | end_index += table[end_index] >> RADIX_LINK_BITS; 39 | 40 | if (end_index >= limit) { 41 | DEBUGLOG(7, "RMF_bitpackExtendMatch : pos %u, link %u, init length %u, full length %u", (U32)start_index, link, (U32)length, (U32)(limit - start_index)); 42 | return limit - start_index; 43 | } 44 | 45 | while (end_index < limit && data[end_index - dist] == data[end_index]) 46 | ++end_index; 47 | 48 | DEBUGLOG(7, "RMF_bitpackExtendMatch : pos %u, link %u, init length %u, full length %u", (U32)start_index, link, (U32)length, (U32)(end_index - start_index)); 49 | return end_index - start_index; 50 | } 51 | 52 | #define GetMatchLink(table, pos) ((const RMF_unit*)(table))[(pos) >> UNIT_BITS].links[(pos) & UNIT_MASK] 53 | 54 | #define GetMatchLength(table, pos) ((const RMF_unit*)(table))[(pos) >> UNIT_BITS].lengths[(pos) & UNIT_MASK] 55 | 56 | static size_t RMF_structuredExtendMatch(const BYTE* const data, 57 | const U32* const table, 58 | ptrdiff_t const start_index, 59 | ptrdiff_t limit, 60 | U32 const link, 61 | size_t const length) 62 | { 63 | ptrdiff_t end_index = start_index + length; 64 | ptrdiff_t const dist = start_index - link; 65 | 66 | if (limit > start_index + (ptrdiff_t)kMatchLenMax) 67 | limit = start_index + kMatchLenMax; 68 | 69 | while (end_index < limit && end_index - (ptrdiff_t)GetMatchLink(table, end_index) == dist) 70 | end_index += GetMatchLength(table, end_index); 71 | 72 | if (end_index >= limit) { 73 | DEBUGLOG(7, "RMF_structuredExtendMatch : pos %u, link %u, init length %u, full length %u", (U32)start_index, link, (U32)length, (U32)(limit - start_index)); 74 | return limit - start_index; 75 | } 76 | 77 | while (end_index < limit && data[end_index - dist] == data[end_index]) 78 | ++end_index; 79 | 80 | DEBUGLOG(7, "RMF_structuredExtendMatch : pos %u, link %u, init length %u, full length %u", (U32)start_index, link, (U32)length, (U32)(end_index - start_index)); 81 | return end_index - start_index; 82 | } 83 | 84 | FORCE_INLINE_TEMPLATE 85 | RMF_match RMF_getMatch(FL2_dataBlock block, 86 | FL2_matchTable* tbl, 87 | unsigned max_depth, 88 | int structTbl, 89 | size_t pos) 90 | { 91 | if (structTbl) 92 | { 93 | U32 const link = GetMatchLink(tbl->table, pos); 94 | 95 | RMF_match match; 96 | match.length = 0; 97 | 98 | if (link == RADIX_NULL_LINK) 99 | return match; 100 | 101 | size_t const length = GetMatchLength(tbl->table, pos); 102 | size_t const dist = pos - link - 1; 103 | 104 | if (length == max_depth || length == STRUCTURED_MAX_LENGTH /* from HandleRepeat */) 105 | match.length = (U32)RMF_structuredExtendMatch(block.data, tbl->table, pos, block.end, link, length); 106 | else 107 | match.length = (U32)length; 108 | 109 | match.dist = (U32)dist; 110 | 111 | return match; 112 | } 113 | else { 114 | U32 link = tbl->table[pos]; 115 | 116 | RMF_match match; 117 | match.length = 0; 118 | 119 | if (link == RADIX_NULL_LINK) 120 | return match; 121 | 122 | size_t const length = link >> RADIX_LINK_BITS; 123 | link &= RADIX_LINK_MASK; 124 | size_t const dist = pos - link - 1; 125 | 126 | if (length == max_depth || length == BITPACK_MAX_LENGTH /* from HandleRepeat */) 127 | match.length = (U32)RMF_bitpackExtendMatch(block.data, tbl->table, pos, block.end, link, length); 128 | else 129 | match.length = (U32)length; 130 | 131 | match.dist = (U32)dist; 132 | 133 | return match; 134 | } 135 | } 136 | 137 | FORCE_INLINE_TEMPLATE 138 | RMF_match RMF_getNextMatch(FL2_dataBlock block, 139 | FL2_matchTable* tbl, 140 | unsigned max_depth, 141 | int structTbl, 142 | size_t pos) 143 | { 144 | if (structTbl) 145 | { 146 | U32 const link = GetMatchLink(tbl->table, pos); 147 | 148 | RMF_match match; 149 | match.length = 0; 150 | 151 | if (link == RADIX_NULL_LINK) 152 | return match; 153 | 154 | size_t const length = GetMatchLength(tbl->table, pos); 155 | size_t const dist = pos - link - 1; 156 | 157 | /* same distance, one byte shorter */ 158 | if (link - 1 == GetMatchLink(tbl->table, pos - 1)) 159 | return match; 160 | 161 | if (length == max_depth || length == STRUCTURED_MAX_LENGTH /* from HandleRepeat */) 162 | match.length = (U32)RMF_structuredExtendMatch(block.data, tbl->table, pos, block.end, link, length); 163 | else 164 | match.length = (U32)length; 165 | 166 | match.dist = (U32)dist; 167 | 168 | return match; 169 | } 170 | else { 171 | U32 link = tbl->table[pos]; 172 | 173 | RMF_match match; 174 | match.length = 0; 175 | 176 | if (link == RADIX_NULL_LINK) 177 | return match; 178 | 179 | size_t const length = link >> RADIX_LINK_BITS; 180 | link &= RADIX_LINK_MASK; 181 | size_t const dist = pos - link - 1; 182 | 183 | /* same distance, one byte shorter */ 184 | if (link - 1 == (tbl->table[pos - 1] & RADIX_LINK_MASK)) 185 | return match; 186 | 187 | if (length == max_depth || length == BITPACK_MAX_LENGTH /* from HandleRepeat */) 188 | match.length = (U32)RMF_bitpackExtendMatch(block.data, tbl->table, pos, block.end, link, length); 189 | else 190 | match.length = (U32)length; 191 | 192 | match.dist = (U32)dist; 193 | 194 | return match; 195 | } 196 | } 197 | 198 | #if defined (__cplusplus) 199 | } 200 | #endif 201 | 202 | #endif /* FL2_RADIX_GET_H_ */ -------------------------------------------------------------------------------- /range_enc.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Bitwise range encoder by Igor Pavlov 3 | * Modified by Conor McCarthy 4 | * 5 | * Public domain 6 | */ 7 | 8 | #include "fl2_internal.h" 9 | #include "mem.h" 10 | #include "platform.h" 11 | #include "range_enc.h" 12 | 13 | /* The first and last elements of these tables are never used */ 14 | BYTE price_table[2][kPriceTableSize] = { { 15 | 0, 193, 182, 166, 154, 145, 137, 131, 16 | 125, 120, 115, 111, 107, 103, 100, 97, 17 | 94, 91, 89, 86, 84, 82, 80, 78, 18 | 76, 74, 72, 71, 69, 67, 66, 64, 19 | 63, 61, 60, 59, 57, 56, 55, 54, 20 | 53, 52, 50, 49, 48, 47, 46, 45, 21 | 44, 43, 42, 42, 41, 40, 39, 38, 22 | 37, 36, 36, 35, 34, 33, 33, 32, 23 | 31, 30, 30, 29, 28, 28, 27, 26, 24 | 26, 25, 25, 24, 23, 23, 22, 21, 25 | 21, 20, 20, 19, 19, 18, 18, 17, 26 | 17, 16, 16, 15, 15, 14, 14, 13, 27 | 13, 12, 12, 11, 11, 10, 10, 9, 28 | 9, 8, 8, 8, 7, 7, 6, 6, 29 | 5, 5, 5, 4, 4, 3, 3, 3, 30 | 2, 2, 2, 1, 1, 0, 0, 0 31 | }, { 32 | 0, 0, 0, 1, 1, 2, 2, 2, 33 | 3, 3, 3, 4, 4, 5, 5, 5, 34 | 6, 6, 7, 7, 8, 8, 8, 9, 35 | 9, 10, 10, 11, 11, 12, 12, 13, 36 | 13, 13, 14, 14, 15, 15, 16, 17, 37 | 17, 18, 18, 19, 19, 20, 20, 21, 38 | 21, 22, 23, 23, 24, 24, 25, 26, 39 | 26, 27, 28, 28, 29, 30, 30, 31, 40 | 32, 33, 33, 34, 35, 36, 36, 37, 41 | 38, 39, 40, 41, 41, 42, 43, 44, 42 | 45, 46, 47, 48, 49, 50, 51, 53, 43 | 54, 55, 56, 57, 59, 60, 61, 63, 44 | 64, 66, 67, 69, 70, 72, 74, 76, 45 | 78, 80, 82, 84, 86, 89, 91, 94, 46 | 97, 100, 103, 107, 111, 115, 119, 125, 47 | 130, 137, 145, 154, 165, 181, 192, 0 48 | } }; 49 | 50 | #if 0 51 | 52 | #include 53 | 54 | /* Generates price_table */ 55 | void RC_printPriceTable() 56 | { 57 | static const unsigned test_size = 0x4000; 58 | const unsigned test_div = test_size >> 8; 59 | BYTE buf[0x3062]; 60 | unsigned table0[kPriceTableSize]; 61 | unsigned table1[kPriceTableSize]; 62 | unsigned count[kPriceTableSize]; 63 | memset(table0, 0, sizeof(table0)); 64 | memset(table1, 0, sizeof(table1)); 65 | memset(count, 0, sizeof(count)); 66 | for (LZMA2_prob i = 31; i <= kBitModelTotal - 31; ++i) { 67 | RC_encoder rc; 68 | RC_reset(&rc); 69 | RC_setOutputBuffer(&rc, buf); 70 | for (unsigned j = 0; j < test_size; ++j) { 71 | LZMA2_prob prob = i; 72 | RC_encodeBit0(&rc, &prob); 73 | } 74 | RC_flush(&rc); 75 | table0[i >> kNumMoveReducingBits] += (unsigned)rc.out_index - 5; 76 | RC_reset(&rc); 77 | RC_setOutputBuffer(&rc, buf); 78 | for (unsigned j = 0; j < test_size; ++j) { 79 | LZMA2_prob prob = i; 80 | RC_encodeBit1(&rc, &prob); 81 | } 82 | RC_flush(&rc); 83 | table1[i >> kNumMoveReducingBits] += (unsigned)rc.out_index - 5; 84 | ++count[i >> kNumMoveReducingBits]; 85 | } 86 | for (int i = 0; i < kPriceTableSize; ++i) if (count[i]) { 87 | table0[i] = (table0[i] / count[i]) / test_div; 88 | table1[i] = (table1[i] / count[i]) / test_div; 89 | } 90 | fputs("const BYTE price_table[2][kPriceTableSize] = {\r\n", stdout); 91 | for (int i = 0; i < kPriceTableSize;) { 92 | for (int j = 0; j < 8; ++j, ++i) 93 | printf("%4d,", table0[i]); 94 | fputs("\r\n", stdout); 95 | } 96 | fputs("}, {\r\n", stdout); 97 | for (int i = 0; i < kPriceTableSize;) { 98 | for (int j = 0; j < 8; ++j, ++i) 99 | printf("%4d,", table1[i]); 100 | fputs("\r\n", stdout); 101 | } 102 | fputs("} };\r\n", stdout); 103 | } 104 | 105 | #endif 106 | 107 | void RC_setOutputBuffer(RC_encoder* const rc, BYTE *const out_buffer) 108 | { 109 | rc->out_buffer = out_buffer; 110 | rc->out_index = 0; 111 | } 112 | 113 | void RC_reset(RC_encoder* const rc) 114 | { 115 | rc->low = 0; 116 | rc->range = (U32)-1; 117 | rc->cache_size = 0; 118 | rc->cache = 0; 119 | } 120 | 121 | #ifdef __64BIT__ 122 | 123 | void FORCE_NOINLINE RC_shiftLow(RC_encoder* const rc) 124 | { 125 | U64 low = rc->low; 126 | rc->low = (U32)(low << 8); 127 | /* VC15 compiles 'if (low < 0xFF000000 || low > 0xFFFFFFFF)' to this single-branch conditional */ 128 | if (low + 0xFFFFFFFF01000000 > 0xFFFFFF) { 129 | BYTE high = (BYTE)(low >> 32); 130 | rc->out_buffer[rc->out_index++] = rc->cache + high; 131 | rc->cache = (BYTE)(low >> 24); 132 | if (rc->cache_size != 0) { 133 | high += 0xFF; 134 | do { 135 | rc->out_buffer[rc->out_index++] = high; 136 | } while (--rc->cache_size != 0); 137 | } 138 | } 139 | else { 140 | rc->cache_size++; 141 | } 142 | } 143 | 144 | #else 145 | 146 | void FORCE_NOINLINE RC_shiftLow(RC_encoder* const rc) 147 | { 148 | U32 low = (U32)rc->low; 149 | unsigned high = (unsigned)(rc->low >> 32); 150 | rc->low = low << 8; 151 | if (low < (U32)0xFF000000 || high != 0) { 152 | rc->out_buffer[rc->out_index++] = rc->cache + (BYTE)high; 153 | rc->cache = (BYTE)(low >> 24); 154 | if (rc->cache_size != 0) { 155 | high += 0xFF; 156 | do { 157 | rc->out_buffer[rc->out_index++] = (BYTE)high; 158 | } while (--rc->cache_size != 0); 159 | } 160 | } 161 | else { 162 | rc->cache_size++; 163 | } 164 | } 165 | 166 | #endif 167 | 168 | void RC_encodeBitTree(RC_encoder* const rc, LZMA2_prob *const probs, unsigned bit_count, unsigned symbol) 169 | { 170 | assert(bit_count > 1); 171 | --bit_count; 172 | unsigned bit = symbol >> bit_count; 173 | RC_encodeBit(rc, &probs[1], bit); 174 | size_t tree_index = 1; 175 | do { 176 | --bit_count; 177 | tree_index = (tree_index << 1) | bit; 178 | bit = (symbol >> bit_count) & 1; 179 | RC_encodeBit(rc, &probs[tree_index], bit); 180 | } while (bit_count != 0); 181 | } 182 | 183 | void RC_encodeBitTreeReverse(RC_encoder* const rc, LZMA2_prob *const probs, unsigned bit_count, unsigned symbol) 184 | { 185 | assert(bit_count != 0); 186 | unsigned bit = symbol & 1; 187 | RC_encodeBit(rc, &probs[1], bit); 188 | unsigned tree_index = 1; 189 | while (--bit_count != 0) { 190 | tree_index = (tree_index << 1) + bit; 191 | symbol >>= 1; 192 | bit = symbol & 1; 193 | RC_encodeBit(rc, &probs[tree_index], bit); 194 | } 195 | } 196 | 197 | void FORCE_NOINLINE RC_encodeDirect(RC_encoder* const rc, unsigned value, unsigned bit_count) 198 | { 199 | assert(bit_count > 0); 200 | do { 201 | rc->range >>= 1; 202 | --bit_count; 203 | rc->low += rc->range & -((int)(value >> bit_count) & 1); 204 | if (rc->range < kTopValue) { 205 | rc->range <<= 8; 206 | RC_shiftLow(rc); 207 | } 208 | } while (bit_count != 0); 209 | } 210 | 211 | 212 | -------------------------------------------------------------------------------- /fl2_pool.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * Modified for FL2 by Conor McCarthy 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | 13 | /* ====== Dependencies ======= */ 14 | #include /* size_t */ 15 | #include /* malloc, calloc */ 16 | #include "fl2_pool.h" 17 | #include "fl2_internal.h" 18 | 19 | 20 | #ifndef FL2_SINGLETHREAD 21 | 22 | #include "fl2_threading.h" /* pthread adaptation */ 23 | 24 | struct FL2POOL_ctx_s { 25 | /* Keep track of the threads */ 26 | size_t numThreads; 27 | 28 | /* All threads work on the same function and object during a job */ 29 | FL2POOL_function function; 30 | void *opaque; 31 | 32 | /* The number of threads working on jobs */ 33 | size_t numThreadsBusy; 34 | /* Indicates the number of threads requested and the values to pass */ 35 | ptrdiff_t queueIndex; 36 | ptrdiff_t queueEnd; 37 | 38 | /* The mutex protects the queue */ 39 | FL2_pthread_mutex_t queueMutex; 40 | /* Condition variable for pushers to wait on when the queue is full */ 41 | FL2_pthread_cond_t busyCond; 42 | /* Condition variable for poppers to wait on when the queue is empty */ 43 | FL2_pthread_cond_t newJobsCond; 44 | /* Indicates if the queue is shutting down */ 45 | int shutdown; 46 | 47 | /* The threads. Extras to be calloc'd */ 48 | FL2_pthread_t threads[1]; 49 | }; 50 | 51 | /* FL2POOL_thread() : 52 | Work thread for the thread pool. 53 | Waits for jobs and executes them. 54 | @returns : NULL on failure else non-null. 55 | */ 56 | static void* FL2POOL_thread(void* opaque) 57 | { 58 | FL2POOL_ctx* const ctx = (FL2POOL_ctx*)opaque; 59 | if (!ctx) { return NULL; } 60 | FL2_pthread_mutex_lock(&ctx->queueMutex); 61 | for (;;) { 62 | 63 | /* While the mutex is locked, wait for a non-empty queue or until shutdown */ 64 | while (ctx->queueIndex >= ctx->queueEnd && !ctx->shutdown) { 65 | FL2_pthread_cond_wait(&ctx->newJobsCond, &ctx->queueMutex); 66 | } 67 | /* empty => shutting down: so stop */ 68 | if (ctx->shutdown) { 69 | FL2_pthread_mutex_unlock(&ctx->queueMutex); 70 | return opaque; 71 | } 72 | /* Pop a job off the queue */ 73 | size_t n = ctx->queueIndex; 74 | ++ctx->queueIndex; 75 | ++ctx->numThreadsBusy; 76 | /* Unlock the mutex and run the job */ 77 | FL2_pthread_mutex_unlock(&ctx->queueMutex); 78 | 79 | ctx->function(ctx->opaque, n); 80 | 81 | FL2_pthread_mutex_lock(&ctx->queueMutex); 82 | --ctx->numThreadsBusy; 83 | /* Signal the master thread waiting for jobs to complete */ 84 | FL2_pthread_cond_signal(&ctx->busyCond); 85 | } /* for (;;) */ 86 | /* Unreachable */ 87 | } 88 | 89 | FL2POOL_ctx* FL2POOL_create(size_t numThreads) 90 | { 91 | FL2POOL_ctx* ctx; 92 | /* Check the parameters */ 93 | if (!numThreads) { return NULL; } 94 | /* Allocate the context and zero initialize */ 95 | ctx = calloc(1, sizeof(FL2POOL_ctx) + (numThreads - 1) * sizeof(FL2_pthread_t)); 96 | if (!ctx) { return NULL; } 97 | /* Initialize the busy count and jobs range */ 98 | ctx->numThreadsBusy = 0; 99 | ctx->queueIndex = 0; 100 | ctx->queueEnd = 0; 101 | (void)FL2_pthread_mutex_init(&ctx->queueMutex, NULL); 102 | (void)FL2_pthread_cond_init(&ctx->busyCond, NULL); 103 | (void)FL2_pthread_cond_init(&ctx->newJobsCond, NULL); 104 | ctx->shutdown = 0; 105 | ctx->numThreads = 0; 106 | /* Initialize the threads */ 107 | { size_t i; 108 | for (i = 0; i < numThreads; ++i) { 109 | if (FL2_pthread_create(&ctx->threads[i], NULL, &FL2POOL_thread, ctx)) { 110 | ctx->numThreads = i; 111 | FL2POOL_free(ctx); 112 | return NULL; 113 | } } 114 | ctx->numThreads = numThreads; 115 | } 116 | return ctx; 117 | } 118 | 119 | /*! FL2POOL_join() : 120 | Shutdown the queue, wake any sleeping threads, and join all of the threads. 121 | */ 122 | static void FL2POOL_join(FL2POOL_ctx* ctx) 123 | { 124 | /* Shut down the queue */ 125 | FL2_pthread_mutex_lock(&ctx->queueMutex); 126 | ctx->shutdown = 1; 127 | /* Wake up sleeping threads */ 128 | FL2_pthread_cond_broadcast(&ctx->newJobsCond); 129 | FL2_pthread_mutex_unlock(&ctx->queueMutex); 130 | /* Join all of the threads */ 131 | for (size_t i = 0; i < ctx->numThreads; ++i) 132 | FL2_pthread_join(ctx->threads[i], NULL); 133 | } 134 | 135 | void FL2POOL_free(FL2POOL_ctx *ctx) 136 | { 137 | if (!ctx) { return; } 138 | FL2POOL_join(ctx); 139 | FL2_pthread_mutex_destroy(&ctx->queueMutex); 140 | FL2_pthread_cond_destroy(&ctx->busyCond); 141 | FL2_pthread_cond_destroy(&ctx->newJobsCond); 142 | free(ctx); 143 | } 144 | 145 | size_t FL2POOL_sizeof(FL2POOL_ctx *ctx) 146 | { 147 | if (ctx==NULL) return 0; /* supports sizeof NULL */ 148 | return sizeof(*ctx) + ctx->numThreads * sizeof(FL2_pthread_t); 149 | } 150 | 151 | void FL2POOL_addRange(void* ctxVoid, FL2POOL_function function, void *opaque, ptrdiff_t first, ptrdiff_t end) 152 | { 153 | FL2POOL_ctx* const ctx = (FL2POOL_ctx*)ctxVoid; 154 | if (!ctx) 155 | return; 156 | 157 | /* Callers always wait for jobs to complete before adding a new set */ 158 | assert(!ctx->numThreadsBusy); 159 | 160 | FL2_pthread_mutex_lock(&ctx->queueMutex); 161 | ctx->function = function; 162 | ctx->opaque = opaque; 163 | ctx->queueIndex = first; 164 | ctx->queueEnd = end; 165 | FL2_pthread_cond_broadcast(&ctx->newJobsCond); 166 | FL2_pthread_mutex_unlock(&ctx->queueMutex); 167 | } 168 | 169 | void FL2POOL_add(void* ctxVoid, FL2POOL_function function, void *opaque, ptrdiff_t n) 170 | { 171 | FL2POOL_addRange(ctxVoid, function, opaque, n, n + 1); 172 | } 173 | 174 | int FL2POOL_waitAll(void *ctxVoid, unsigned timeout) 175 | { 176 | FL2POOL_ctx* const ctx = (FL2POOL_ctx*)ctxVoid; 177 | if (!ctx || (!ctx->numThreadsBusy && ctx->queueIndex >= ctx->queueEnd) || ctx->shutdown) { return 0; } 178 | 179 | FL2_pthread_mutex_lock(&ctx->queueMutex); 180 | /* Need to test for ctx->queueIndex < ctx->queueEnd in case not all jobs have started */ 181 | if (timeout != 0) { 182 | if ((ctx->numThreadsBusy || ctx->queueIndex < ctx->queueEnd) && !ctx->shutdown) 183 | FL2_pthread_cond_timedwait(&ctx->busyCond, &ctx->queueMutex, timeout); 184 | } 185 | else { 186 | while ((ctx->numThreadsBusy || ctx->queueIndex < ctx->queueEnd) && !ctx->shutdown) 187 | FL2_pthread_cond_wait(&ctx->busyCond, &ctx->queueMutex); 188 | } 189 | FL2_pthread_mutex_unlock(&ctx->queueMutex); 190 | return ctx->numThreadsBusy && !ctx->shutdown; 191 | } 192 | 193 | size_t FL2POOL_threadsBusy(void * ctx) 194 | { 195 | return ((FL2POOL_ctx*)ctx)->numThreadsBusy; 196 | } 197 | 198 | #endif /* FL2_SINGLETHREAD */ 199 | -------------------------------------------------------------------------------- /dict_buffer.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2019, Conor McCarthy 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #include 12 | #include "dict_buffer.h" 13 | #include "fl2_internal.h" 14 | 15 | #define ALIGNMENT_SIZE 16U 16 | #define ALIGNMENT_MASK (~(size_t)(ALIGNMENT_SIZE-1)) 17 | 18 | /* DICT_buffer functions */ 19 | 20 | int DICT_construct(DICT_buffer * const buf, int const async) 21 | { 22 | buf->data[0] = NULL; 23 | buf->data[1] = NULL; 24 | buf->size = 0; 25 | 26 | buf->async = (async != 0); 27 | 28 | #ifndef NO_XXHASH 29 | buf->xxh = NULL; 30 | #endif 31 | 32 | return 0; 33 | } 34 | 35 | int DICT_init(DICT_buffer * const buf, size_t const dict_size, size_t const overlap, unsigned const reset_multiplier, int const do_hash) 36 | { 37 | /* Allocate if not yet allocated or existing dict too small */ 38 | if (buf->data[0] == NULL || dict_size > buf->size) { 39 | /* Free any existing buffers */ 40 | DICT_destruct(buf); 41 | 42 | buf->data[0] = malloc(dict_size); 43 | 44 | buf->data[1] = NULL; 45 | if (buf->async) 46 | buf->data[1] = malloc(dict_size); 47 | 48 | if (buf->data[0] == NULL || (buf->async && buf->data[1] == NULL)) { 49 | DICT_destruct(buf); 50 | return 1; 51 | } 52 | } 53 | buf->index = 0; 54 | buf->overlap = overlap; 55 | buf->start = 0; 56 | buf->end = 0; 57 | buf->size = dict_size; 58 | buf->total = 0; 59 | buf->reset_interval = (reset_multiplier != 0) ? dict_size * reset_multiplier : ((size_t)1 << 31); 60 | 61 | #ifndef NO_XXHASH 62 | if (do_hash) { 63 | if (buf->xxh == NULL) { 64 | buf->xxh = XXH32_createState(); 65 | if (buf->xxh == NULL) { 66 | DICT_destruct(buf); 67 | return 1; 68 | } 69 | } 70 | XXH32_reset(buf->xxh, 0); 71 | } 72 | else { 73 | XXH32_freeState(buf->xxh); 74 | buf->xxh = NULL; 75 | } 76 | #else 77 | (void)do_hash; 78 | #endif 79 | 80 | return 0; 81 | } 82 | 83 | void DICT_destruct(DICT_buffer * const buf) 84 | { 85 | free(buf->data[0]); 86 | free(buf->data[1]); 87 | buf->data[0] = NULL; 88 | buf->data[1] = NULL; 89 | buf->size = 0; 90 | #ifndef NO_XXHASH 91 | XXH32_freeState(buf->xxh); 92 | buf->xxh = NULL; 93 | #endif 94 | } 95 | 96 | size_t DICT_size(const DICT_buffer * const buf) 97 | { 98 | return buf->size; 99 | } 100 | 101 | /* Get the dictionary buffer for adding input */ 102 | size_t DICT_get(DICT_buffer * const buf, void **const dict) 103 | { 104 | DICT_shift(buf); 105 | 106 | DEBUGLOG(5, "Getting dict buffer %u, pos %u, avail %u", (unsigned)buf->index, (unsigned)buf->end, (unsigned)(buf->size - buf->end)); 107 | *dict = buf->data[buf->index] + buf->end; 108 | return buf->size - buf->end; 109 | } 110 | 111 | /* Update with the amount added */ 112 | int DICT_update(DICT_buffer * const buf, size_t const added_size) 113 | { 114 | DEBUGLOG(5, "Added %u bytes to dict buffer %u", (unsigned)added_size, (unsigned)buf->index); 115 | buf->end += added_size; 116 | assert(buf->end <= buf->size); 117 | return !DICT_availSpace(buf); 118 | } 119 | 120 | /* Read from input and write to the dict */ 121 | void DICT_put(DICT_buffer * const buf, FL2_inBuffer * const input) 122 | { 123 | size_t const to_read = MIN(buf->size - buf->end, input->size - input->pos); 124 | 125 | DEBUGLOG(5, "CStream : reading %u bytes", (U32)to_read); 126 | 127 | memcpy(buf->data[buf->index] + buf->end, (BYTE*)input->src + input->pos, to_read); 128 | 129 | input->pos += to_read; 130 | buf->end += to_read; 131 | } 132 | 133 | size_t DICT_availSpace(const DICT_buffer * const buf) 134 | { 135 | return buf->size - buf->end; 136 | } 137 | 138 | /* Get the size of uncompressed data. start is set to end after compression */ 139 | int DICT_hasUnprocessed(const DICT_buffer * const buf) 140 | { 141 | return buf->start < buf->end; 142 | } 143 | 144 | /* Get the buffer, overlap and end for compression */ 145 | void DICT_getBlock(DICT_buffer * const buf, FL2_dataBlock * const block) 146 | { 147 | block->data = buf->data[buf->index]; 148 | block->start = buf->start; 149 | block->end = buf->end; 150 | 151 | #ifndef NO_XXHASH 152 | if (buf->xxh != NULL) 153 | XXH32_update(buf->xxh, buf->data[buf->index] + buf->start, buf->end - buf->start); 154 | #endif 155 | 156 | buf->total += buf->end - buf->start; 157 | buf->start = buf->end; 158 | } 159 | 160 | /* Shift occurs when all is processed and end is beyond the overlap size */ 161 | int DICT_needShift(DICT_buffer * const buf) 162 | { 163 | if (buf->start < buf->end) 164 | return 0; 165 | /* Reset the dict if the next compression cycle would exceed the reset interval */ 166 | size_t overlap = (buf->total + buf->size - buf->overlap > buf->reset_interval) ? 0 : buf->overlap; 167 | return buf->start == buf->end && (overlap == 0 || buf->end >= overlap + ALIGNMENT_SIZE); 168 | } 169 | 170 | int DICT_async(const DICT_buffer * const buf) 171 | { 172 | return (int)buf->async; 173 | } 174 | 175 | /* Shift the overlap amount to the start of either the only dict buffer or the alternate one 176 | * if it exists */ 177 | void DICT_shift(DICT_buffer * const buf) 178 | { 179 | if (buf->start < buf->end) 180 | return; 181 | 182 | size_t overlap = buf->overlap; 183 | /* Reset the dict if the next compression cycle would exceed the reset interval */ 184 | if (buf->total + buf->size - buf->overlap > buf->reset_interval) { 185 | DEBUGLOG(4, "Resetting dictionary after %u bytes", (unsigned)buf->total); 186 | overlap = 0; 187 | } 188 | 189 | if (overlap == 0) { 190 | /* No overlap means a simple buffer switch */ 191 | buf->start = 0; 192 | buf->end = 0; 193 | buf->index ^= buf->async; 194 | buf->total = 0; 195 | } 196 | else if (buf->end >= overlap + ALIGNMENT_SIZE) { 197 | size_t const from = (buf->end - overlap) & ALIGNMENT_MASK; 198 | const BYTE *const src = buf->data[buf->index]; 199 | /* Copy to the alternate if one exists */ 200 | BYTE *const dst = buf->data[buf->index ^ buf->async]; 201 | 202 | overlap = buf->end - from; 203 | 204 | if (overlap <= from || dst != src) { 205 | DEBUGLOG(5, "Copy overlap data : %u bytes from %u", (unsigned)overlap, (unsigned)from); 206 | memcpy(dst, src + from, overlap); 207 | } 208 | else if (from != 0) { 209 | DEBUGLOG(5, "Move overlap data : %u bytes from %u", (unsigned)overlap, (unsigned)from); 210 | memmove(dst, src + from, overlap); 211 | } 212 | /* New data will be written after the overlap */ 213 | buf->start = overlap; 214 | buf->end = overlap; 215 | /* Switch buffers */ 216 | buf->index ^= buf->async; 217 | } 218 | } 219 | 220 | #ifndef NO_XXHASH 221 | XXH32_hash_t DICT_getDigest(const DICT_buffer * const buf) 222 | { 223 | return XXH32_digest(buf->xxh); 224 | } 225 | #endif 226 | 227 | size_t DICT_memUsage(const DICT_buffer * const buf) 228 | { 229 | return (1 + buf->async) * buf->size; 230 | } 231 | -------------------------------------------------------------------------------- /util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef UTIL_H_MODULE 12 | #define UTIL_H_MODULE 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | 19 | /*-**************************************** 20 | * Dependencies 21 | ******************************************/ 22 | #include "platform.h" /* PLATFORM_POSIX_VERSION, ZSTD_NANOSLEEP_SUPPORT, ZSTD_SETPRIORITY_SUPPORT */ 23 | #include /* malloc, realloc, free */ 24 | #include /* size_t, ptrdiff_t */ 25 | #include /* fprintf */ 26 | #include /* stat, utime */ 27 | #include /* stat, chmod */ 28 | #if defined(_MSC_VER) 29 | # include /* utime */ 30 | # include /* _chmod */ 31 | #else 32 | # include /* chown, stat */ 33 | # include /* utime */ 34 | #endif 35 | #include /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */ 36 | #include "mem.h" /* U32, U64 */ 37 | 38 | 39 | /*-************************************************************ 40 | * Avoid fseek()'s 2GiB barrier with MSVC, macOS, *BSD, MinGW 41 | ***************************************************************/ 42 | #if defined(_MSC_VER) && (_MSC_VER >= 1400) 43 | # define UTIL_fseek _fseeki64 44 | #elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */ 45 | # define UTIL_fseek fseeko 46 | #elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS) 47 | # define UTIL_fseek fseeko64 48 | #else 49 | # define UTIL_fseek fseek 50 | #endif 51 | 52 | 53 | /*-************************************************* 54 | * Sleep & priority functions: Windows - Posix - others 55 | ***************************************************/ 56 | #if defined(_WIN32) 57 | # include 58 | # define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS) 59 | # define UTIL_sleep(s) Sleep(1000*s) 60 | # define UTIL_sleepMilli(milli) Sleep(milli) 61 | 62 | #elif PLATFORM_POSIX_VERSION > 0 /* Unix-like operating system */ 63 | # include /* sleep */ 64 | # define UTIL_sleep(s) sleep(s) 65 | # if ZSTD_NANOSLEEP_SUPPORT /* necessarily defined in platform.h */ 66 | # define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); } 67 | # else 68 | # define UTIL_sleepMilli(milli) /* disabled */ 69 | # endif 70 | # if ZSTD_SETPRIORITY_SUPPORT 71 | # include /* setpriority */ 72 | # define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20) 73 | # else 74 | # define SET_REALTIME_PRIORITY /* disabled */ 75 | # endif 76 | 77 | #else /* unknown non-unix operating systen */ 78 | # define UTIL_sleep(s) /* disabled */ 79 | # define UTIL_sleepMilli(milli) /* disabled */ 80 | # define SET_REALTIME_PRIORITY /* disabled */ 81 | #endif 82 | 83 | 84 | /*-************************************* 85 | * Constants 86 | ***************************************/ 87 | #define LIST_SIZE_INCREASE (8*1024) 88 | 89 | 90 | /*-**************************************** 91 | * Compiler specifics 92 | ******************************************/ 93 | #if defined(__INTEL_COMPILER) 94 | # pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */ 95 | #endif 96 | #if defined(__GNUC__) 97 | # define UTIL_STATIC static __attribute__((unused)) 98 | #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 99 | # define UTIL_STATIC static inline 100 | #elif defined(_MSC_VER) 101 | # define UTIL_STATIC static __inline 102 | #else 103 | # define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ 104 | #endif 105 | 106 | 107 | /*-**************************************** 108 | * Console log 109 | ******************************************/ 110 | extern int g_utilDisplayLevel; 111 | #define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__) 112 | #define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } } 113 | 114 | 115 | /*-**************************************** 116 | * Time functions 117 | ******************************************/ 118 | #if defined(_WIN32) /* Windows */ 119 | 120 | #define UTIL_TIME_INITIALIZER { { 0, 0 } } 121 | typedef LARGE_INTEGER UTIL_time_t; 122 | 123 | #elif defined(__APPLE__) && defined(__MACH__) 124 | 125 | #include 126 | #define UTIL_TIME_INITIALIZER 0 127 | typedef U64 UTIL_time_t; 128 | 129 | #elif (PLATFORM_POSIX_VERSION >= 200112L) \ 130 | && (defined(__UCLIBC__) \ 131 | || (defined(__GLIBC__) \ 132 | && ((__GLIBC__ == 2 && __GLIBC_MINOR__ >= 17) \ 133 | || (__GLIBC__ > 2)))) 134 | 135 | #define UTIL_TIME_INITIALIZER { 0, 0 } 136 | typedef struct timespec UTIL_freq_t; 137 | typedef struct timespec UTIL_time_t; 138 | 139 | UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end); 140 | 141 | #else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */ 142 | 143 | typedef clock_t UTIL_time_t; 144 | #define UTIL_TIME_INITIALIZER 0 145 | 146 | #endif 147 | 148 | UTIL_time_t UTIL_getTime(void); 149 | U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd); 150 | U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd); 151 | 152 | #define SEC_TO_MICRO 1000000 153 | 154 | /* returns time span in microseconds */ 155 | U64 UTIL_clockSpanMicro(UTIL_time_t clockStart); 156 | 157 | /* returns time span in microseconds */ 158 | U64 UTIL_clockSpanNano(UTIL_time_t clockStart); 159 | void UTIL_waitForNextTick(void); 160 | 161 | /*-**************************************** 162 | * File functions 163 | ******************************************/ 164 | #if defined(_MSC_VER) 165 | #define chmod _chmod 166 | typedef struct __stat64 stat_t; 167 | #else 168 | typedef struct stat stat_t; 169 | #endif 170 | 171 | 172 | int UTIL_fileExist(const char* filename); 173 | int UTIL_isRegularFile(const char* infilename); 174 | int UTIL_setFileStat(const char* filename, stat_t* statbuf); 175 | U32 UTIL_isDirectory(const char* infilename); 176 | int UTIL_getFileStat(const char* infilename, stat_t* statbuf); 177 | 178 | U32 UTIL_isLink(const char* infilename); 179 | #define UTIL_FILESIZE_UNKNOWN ((U64)(-1)) 180 | U64 UTIL_getFileSize(const char* infilename); 181 | 182 | U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles); 183 | 184 | /* 185 | * A modified version of realloc(). 186 | * If UTIL_realloc() fails the original block is freed. 187 | */ 188 | UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size) 189 | { 190 | void *newptr = realloc(ptr, size); 191 | if (newptr) return newptr; 192 | free(ptr); 193 | return NULL; 194 | } 195 | 196 | int UTIL_prepareFileList(const char* dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks); 197 | #ifdef _WIN32 198 | # define UTIL_HAS_CREATEFILELIST 199 | #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ 200 | # define UTIL_HAS_CREATEFILELIST 201 | # include /* opendir, readdir */ 202 | # include /* strerror, memcpy */ 203 | #else 204 | #endif /* #ifdef _WIN32 */ 205 | 206 | /* 207 | * UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories, 208 | * and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb). 209 | * After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer) 210 | * In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called. 211 | */ 212 | const char** 213 | UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, 214 | char** allocatedBuffer, unsigned* allocatedNamesNb, 215 | int followLinks); 216 | 217 | UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer) 218 | { 219 | if (allocatedBuffer) free(allocatedBuffer); 220 | if (filenameTable) free((void*)filenameTable); 221 | } 222 | 223 | int UTIL_countPhysicalCores(void); 224 | 225 | #if defined (__cplusplus) 226 | } 227 | #endif 228 | 229 | #endif /* UTIL_H_MODULE */ 230 | -------------------------------------------------------------------------------- /platform.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef PLATFORM_H_MODULE 12 | #define PLATFORM_H_MODULE 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | 19 | 20 | /* ************************************** 21 | * Compiler Options 22 | ****************************************/ 23 | #if defined(_MSC_VER) 24 | # define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */ 25 | # if (_MSC_VER <= 1800) /* 1800 == Visual Studio 2013 */ 26 | # define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before and */ 27 | # define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */ 28 | # endif 29 | #endif 30 | 31 | 32 | /* ************************************** 33 | * Detect 64-bit OS 34 | * http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros 35 | ****************************************/ 36 | #if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \ 37 | || defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \ 38 | || (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \ 39 | || defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \ 40 | || defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \ 41 | || (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \ 42 | || defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \ 43 | || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */ 44 | # if !defined(__64BIT__) 45 | # define __64BIT__ 1 46 | # endif 47 | #endif 48 | 49 | 50 | /* ********************************************************* 51 | * Turn on Large Files support (>4GB) for 32-bit Linux/Unix 52 | ***********************************************************/ 53 | #if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */ 54 | # if !defined(_FILE_OFFSET_BITS) 55 | # define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */ 56 | # endif 57 | # if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */ 58 | # define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */ 59 | # endif 60 | # if defined(_AIX) || defined(__hpux) 61 | # define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */ 62 | # endif 63 | #endif 64 | 65 | 66 | /* ************************************************************ 67 | * Detect POSIX version 68 | * PLATFORM_POSIX_VERSION = 0 for non-Unix e.g. Windows 69 | * PLATFORM_POSIX_VERSION = 1 for Unix-like but non-POSIX 70 | * PLATFORM_POSIX_VERSION > 1 is equal to found _POSIX_VERSION 71 | * Value of PLATFORM_POSIX_VERSION can be forced on command line 72 | ***************************************************************/ 73 | #ifndef PLATFORM_POSIX_VERSION 74 | 75 | # if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1-2001 (SUSv3) conformant */ \ 76 | || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ 77 | /* exception rule : force posix version to 200112L, 78 | * note: it's better to use unistd.h's _POSIX_VERSION whenever possible */ 79 | # define PLATFORM_POSIX_VERSION 200112L 80 | 81 | /* try to determine posix version through official unistd.h's _POSIX_VERSION (http://pubs.opengroup.org/onlinepubs/7908799/xsh/unistd.h.html). 82 | * note : there is no simple way to know in advance if is present or not on target system, 83 | * Posix specification mandates its presence and its content, but target system must respect this spec. 84 | * It's necessary to _not_ #include whenever target OS is not unix-like 85 | * otherwise it will block preprocessing stage. 86 | * The following list of build macros tries to "guess" if target OS is likely unix-like, and therefore can #include 87 | */ 88 | # elif !defined(_WIN32) \ 89 | && (defined(__unix__) || defined(__unix) \ 90 | || defined(__midipix__) || defined(__VMS) || defined(__HAIKU__)) 91 | 92 | # if defined(__linux__) || defined(__linux) 93 | # ifndef _POSIX_C_SOURCE 94 | # define _POSIX_C_SOURCE 200112L /* feature test macro : https://www.gnu.org/software/libc/manual/html_node/Feature-Test-Macros.html */ 95 | # endif 96 | # endif 97 | # include /* declares _POSIX_VERSION */ 98 | # if defined(_POSIX_VERSION) /* POSIX compliant */ 99 | # define PLATFORM_POSIX_VERSION _POSIX_VERSION 100 | # else 101 | # define PLATFORM_POSIX_VERSION 1 102 | # endif 103 | 104 | # else /* non-unix target platform (like Windows) */ 105 | # define PLATFORM_POSIX_VERSION 0 106 | # endif 107 | 108 | #endif /* PLATFORM_POSIX_VERSION */ 109 | 110 | /*-********************************************* 111 | * Detect if isatty() and fileno() are available 112 | ************************************************/ 113 | #if (defined(__linux__) && (PLATFORM_POSIX_VERSION > 1)) \ 114 | || (PLATFORM_POSIX_VERSION >= 200112L) \ 115 | || defined(__DJGPP__) \ 116 | || defined(__MSYS__) 117 | # include /* isatty */ 118 | # define IS_CONSOLE(stdStream) isatty(fileno(stdStream)) 119 | #elif defined(MSDOS) || defined(OS2) || defined(__CYGWIN__) 120 | # include /* _isatty */ 121 | # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) 122 | #elif defined(WIN32) || defined(_WIN32) 123 | # include /* _isatty */ 124 | # include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ 125 | # include /* FILE */ 126 | static __inline int IS_CONSOLE(FILE* stdStream) { 127 | DWORD dummy; 128 | return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy); 129 | } 130 | #else 131 | # define IS_CONSOLE(stdStream) 0 132 | #endif 133 | 134 | 135 | /****************************** 136 | * OS-specific IO behaviors 137 | ******************************/ 138 | #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) 139 | # include /* _O_BINARY */ 140 | # include /* _setmode, _fileno, _get_osfhandle */ 141 | # if !defined(__DJGPP__) 142 | # include /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ 143 | # include /* FSCTL_SET_SPARSE */ 144 | # define SET_BINARY_MODE(file) { int const unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } 145 | # define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); } 146 | # else 147 | # define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) 148 | # define SET_SPARSE_FILE_MODE(file) 149 | # endif 150 | #else 151 | # define SET_BINARY_MODE(file) 152 | # define SET_SPARSE_FILE_MODE(file) 153 | #endif 154 | 155 | 156 | #ifndef ZSTD_SPARSE_DEFAULT 157 | # if (defined(__APPLE__) && defined(__MACH__)) 158 | # define ZSTD_SPARSE_DEFAULT 0 159 | # else 160 | # define ZSTD_SPARSE_DEFAULT 1 161 | # endif 162 | #endif 163 | 164 | 165 | #ifndef ZSTD_START_SYMBOLLIST_FRAME 166 | # ifdef __linux__ 167 | # define ZSTD_START_SYMBOLLIST_FRAME 2 168 | # elif defined __APPLE__ 169 | # define ZSTD_START_SYMBOLLIST_FRAME 4 170 | # else 171 | # define ZSTD_START_SYMBOLLIST_FRAME 0 172 | # endif 173 | #endif 174 | 175 | 176 | #ifndef ZSTD_SETPRIORITY_SUPPORT 177 | /* mandates presence of and support for setpriority() : http://man7.org/linux/man-pages/man2/setpriority.2.html */ 178 | # define ZSTD_SETPRIORITY_SUPPORT (PLATFORM_POSIX_VERSION >= 200112L) 179 | #endif 180 | 181 | 182 | #ifndef ZSTD_NANOSLEEP_SUPPORT 183 | /* mandates support of nanosleep() within : http://man7.org/linux/man-pages/man2/nanosleep.2.html */ 184 | # if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) \ 185 | || (PLATFORM_POSIX_VERSION >= 200112L) 186 | # define ZSTD_NANOSLEEP_SUPPORT 1 187 | # else 188 | # define ZSTD_NANOSLEEP_SUPPORT 0 189 | # endif 190 | #endif 191 | 192 | 193 | #if defined (__cplusplus) 194 | } 195 | #endif 196 | 197 | #endif /* PLATFORM_H_MODULE */ 198 | -------------------------------------------------------------------------------- /fuzzer/fuzzer.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | Debug 14 | x64 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | {A236F110-80EE-4502-A67C-C1BD0CCFAB15} 32 | Win32Proj 33 | bench 34 | 8.1 35 | 36 | 37 | 38 | Application 39 | true 40 | v140 41 | Unicode 42 | 43 | 44 | Application 45 | false 46 | v140 47 | true 48 | Unicode 49 | 50 | 51 | Application 52 | true 53 | v140 54 | Unicode 55 | 56 | 57 | Application 58 | false 59 | v140 60 | true 61 | Unicode 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | true 83 | 84 | 85 | true 86 | 87 | 88 | false 89 | 90 | 91 | false 92 | 93 | 94 | 95 | 96 | 97 | Level3 98 | Disabled 99 | WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) 100 | ..\..\ 101 | 102 | 103 | Console 104 | true 105 | $(SolutionDir)$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 106 | 107 | 108 | 109 | 110 | 111 | 112 | Level3 113 | Disabled 114 | _DEBUG;_CONSOLE;%(PreprocessorDefinitions) 115 | ..\..;..\..\tests 116 | false 117 | 118 | 119 | Console 120 | true 121 | $(SolutionDir)$(Platform)\$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 122 | 123 | 124 | 125 | 126 | Level3 127 | 128 | 129 | MaxSpeed 130 | true 131 | true 132 | WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 133 | true 134 | ..\..\ 135 | 136 | 137 | Console 138 | true 139 | true 140 | true 141 | $(SolutionDir)$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 142 | 143 | 144 | 145 | 146 | Level3 147 | 148 | 149 | MaxSpeed 150 | true 151 | true 152 | NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 153 | ..\..;..\..\tests 154 | 155 | 156 | Console 157 | true 158 | true 159 | true 160 | $(SolutionDir)$(Platform)\$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 161 | 162 | 163 | 164 | 165 | 166 | -------------------------------------------------------------------------------- /test/file_test.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | Debug 14 | x64 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | 23 | 24 | 25 | 15.0 26 | {0809DF55-8AA5-421A-834D-656B3D053D23} 27 | Win32Proj 28 | filetest 29 | 8.1 30 | 31 | 32 | 33 | Application 34 | true 35 | v141 36 | Unicode 37 | 38 | 39 | Application 40 | false 41 | v141 42 | true 43 | Unicode 44 | 45 | 46 | Application 47 | true 48 | v141 49 | Unicode 50 | 51 | 52 | Application 53 | false 54 | v140 55 | true 56 | Unicode 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | false 78 | 79 | 80 | true 81 | 82 | 83 | true 84 | 85 | 86 | false 87 | 88 | 89 | 90 | NotUsing 91 | Level3 92 | MaxSpeed 93 | true 94 | true 95 | true 96 | _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 97 | true 98 | pch.h 99 | ..\ 100 | 101 | 102 | Console 103 | true 104 | true 105 | true 106 | $(SolutionDir)$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 107 | 108 | 109 | 110 | 111 | NotUsing 112 | Level3 113 | Disabled 114 | true 115 | _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) 116 | true 117 | pch.h 118 | ..\ 119 | 120 | 121 | Console 122 | true 123 | $(SolutionDir)$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 124 | 125 | 126 | 127 | 128 | NotUsing 129 | Level3 130 | Disabled 131 | true 132 | _CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) 133 | true 134 | pch.h 135 | ..\ 136 | 137 | 138 | Console 139 | true 140 | $(SolutionDir)$(Platform)\$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 141 | 142 | 143 | 144 | 145 | NotUsing 146 | Level3 147 | MaxSpeed 148 | true 149 | true 150 | true 151 | _CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 152 | true 153 | pch.h 154 | ..\ 155 | 156 | 157 | Console 158 | true 159 | true 160 | true 161 | $(SolutionDir)$(Platform)\$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 162 | 163 | 164 | 165 | 166 | 167 | -------------------------------------------------------------------------------- /bench/bench.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | Debug 14 | x64 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | {E49A84FD-DE73-4A88-8968-F3B1C328B238} 33 | Win32Proj 34 | bench 35 | 8.1 36 | 37 | 38 | 39 | Application 40 | true 41 | v140 42 | Unicode 43 | 44 | 45 | Application 46 | false 47 | v140 48 | true 49 | Unicode 50 | 51 | 52 | Application 53 | true 54 | v140 55 | Unicode 56 | 57 | 58 | Application 59 | false 60 | v140 61 | true 62 | Unicode 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | true 84 | 85 | 86 | true 87 | 88 | 89 | false 90 | 91 | 92 | false 93 | 94 | 95 | 96 | 97 | 98 | Level3 99 | Disabled 100 | WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) 101 | 102 | 103 | Console 104 | true 105 | $(SolutionDir)$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 106 | 107 | 108 | 109 | 110 | 111 | 112 | Level3 113 | Disabled 114 | _DEBUG;_CONSOLE;%(PreprocessorDefinitions) 115 | 116 | 117 | 118 | 119 | Console 120 | true 121 | $(SolutionDir)$(Platform)\$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 122 | 123 | 124 | 125 | 126 | Level3 127 | 128 | 129 | MaxSpeed 130 | true 131 | true 132 | WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 133 | 134 | 135 | true 136 | AnySuitable 137 | Cdecl 138 | MultiThreaded 139 | 140 | 141 | Console 142 | true 143 | true 144 | true 145 | UseLinkTimeCodeGeneration 146 | $(SolutionDir)$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 147 | 148 | 149 | 150 | 151 | Level3 152 | 153 | 154 | MaxSpeed 155 | true 156 | true 157 | RMF_REFERENCE;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 158 | 159 | 160 | true 161 | AnySuitable 162 | FastCall 163 | Neither 164 | MultiThreaded 165 | 166 | 167 | Console 168 | true 169 | true 170 | true 171 | UseLinkTimeCodeGeneration 172 | $(SolutionDir)$(Platform)\$(Configuration)\fast-lzma2.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) 173 | 174 | 175 | 176 | 177 | 178 | -------------------------------------------------------------------------------- /bench/bench.c: -------------------------------------------------------------------------------- 1 | // bench.cpp : Defines the entry point for the console application. 2 | // 3 | 4 | #include 5 | #include 6 | #include "../fast-lzma2.h" 7 | #include "../fuzzer/datagen.h" 8 | #include "../mem.h" 9 | #include "../util.h" 10 | 11 | #define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ 12 | #define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ 13 | #define COOLPERIOD_SEC 5 14 | 15 | #define KB *(1 <<10) 16 | #define MB *(1 <<20) 17 | #define GB *(1U<<30) 18 | 19 | static U32 g_nbSeconds = 0; 20 | static unsigned g_iterations = 2; 21 | 22 | static void benchmark(FL2_CCtx* fcs, FL2_DCtx* dctx, char* srcBuffer, size_t srcSize, char* compressedBuffer, size_t maxCompressedSize, 23 | char* resultBuffer) 24 | { 25 | 26 | // RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); 27 | 28 | U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); 29 | UTIL_time_t coolTime; 30 | U64 const maxTime = (g_nbSeconds * TIMELOOP_MICROSEC) + 1; 31 | U64 totalCTime = 0, totalDTime = 0; 32 | U32 cCompleted = 0, dCompleted = 0; 33 | unsigned totalCLoops = 0, totalDLoops = 0; 34 | # define NB_MARKS 4 35 | const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" }; 36 | U32 markNb = 0; 37 | 38 | coolTime = UTIL_getTime(); 39 | printf( "\r%79s\r", ""); 40 | size_t cSize = 0; 41 | while (!cCompleted || !dCompleted) { 42 | 43 | /* overheat protection */ 44 | if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { 45 | printf( "\rcooling down ... \r"); 46 | UTIL_sleep(COOLPERIOD_SEC); 47 | printf("\r \r"); 48 | coolTime = UTIL_getTime(); 49 | } 50 | 51 | UTIL_time_t clockStart; 52 | /* Compression */ 53 | if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ 54 | 55 | UTIL_waitForNextTick(); 56 | clockStart = UTIL_getTime(); 57 | 58 | if (!cCompleted) { /* still some time to do compression tests */ 59 | U64 const clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; 60 | U32 nbLoops = 0; 61 | do { 62 | cSize = FL2_compressCCtx(fcs, compressedBuffer, maxCompressedSize, srcBuffer, srcSize, 0); 63 | if (FL2_isError(cSize)) { 64 | printf("FL2_compressCCtx() error : %s \r\n", FL2_getErrorName(cSize)); 65 | return; 66 | } 67 | nbLoops++; 68 | } while (UTIL_clockSpanMicro(clockStart) < clockLoop); 69 | U64 const loopDuration = UTIL_clockSpanMicro(clockStart); 70 | if (loopDuration < fastestC*nbLoops) 71 | fastestC = loopDuration / nbLoops; 72 | totalCTime += loopDuration; 73 | totalCLoops += nbLoops; 74 | cCompleted = (totalCTime >= maxTime && totalCLoops >= g_iterations); /* end compression tests */ 75 | } 76 | 77 | #if 0 /* disable decompression test */ 78 | dCompleted = 1; 79 | (void)totalDTime; (void)fastestD; /* unused when decompression disabled */ 80 | #else 81 | /* Decompression */ 82 | if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ 83 | 84 | UTIL_waitForNextTick(); 85 | 86 | if (!dCompleted) { 87 | U64 clockLoop = g_nbSeconds ? TIMELOOP_MICROSEC : 1; 88 | U32 nbLoops = 0; 89 | UTIL_time_t const clockStart = UTIL_getTime(); 90 | do { 91 | size_t const regenSize = FL2_decompressDCtx(dctx, 92 | resultBuffer, srcSize, 93 | compressedBuffer, cSize); 94 | if (FL2_isError(regenSize)) { 95 | printf("FL2_decompressDCtx() failed on size %u : %s \r\n", 96 | (unsigned)cSize, FL2_getErrorName(regenSize)); 97 | return; 98 | } 99 | nbLoops++; 100 | } while (UTIL_clockSpanMicro(clockStart) < clockLoop); 101 | { U64 const loopDuration = UTIL_clockSpanMicro(clockStart); 102 | if (loopDuration < fastestD*nbLoops) 103 | fastestD = loopDuration / nbLoops; 104 | totalDTime += loopDuration; 105 | totalDLoops += nbLoops; 106 | dCompleted = (totalDTime >= maxTime && totalDLoops >= g_iterations); 107 | } 108 | if (memcmp(resultBuffer, srcBuffer, srcSize) != 0) 109 | printf("Corruption on dSize %u cSize %u\r\n", (unsigned)srcSize, (unsigned)cSize); 110 | } 111 | 112 | #endif 113 | double ratio = (double)srcSize / (double)cSize; 114 | markNb = (markNb + 1) % NB_MARKS; 115 | { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; 116 | double const compressionSpeed = (double)srcSize / fastestC; 117 | int const cSpeedAccuracy = 2;// (compressionSpeed < 10.) ? 2 : 1; 118 | double const decompressionSpeed = (double)srcSize / fastestD; 119 | printf("%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s\r", 120 | marks[markNb], "", (U32)srcSize, (U32)cSize, 121 | ratioAccuracy, ratio, 122 | cSpeedAccuracy, compressionSpeed, 123 | decompressionSpeed); 124 | } 125 | } 126 | } 127 | 128 | static int parse_params(FL2_CCtx* fcs, int argc, char** argv) 129 | { 130 | for (int i = 2; i < argc; ++i) { 131 | if (argv[i][0] == '-' && argv[i][1] >= '0' && argv[i][1] <= '9') 132 | FL2_CCtx_setParameter(fcs, FL2_p_compressionLevel, atoi(argv[i] + 1)); 133 | } 134 | int end_level = 0; 135 | for (int i = 2; i < argc; ++i) { 136 | if (argv[i][0] != '-') 137 | continue; 138 | char param[4]; 139 | int j = 1; 140 | for (; j < 4 && argv[i][j] && (argv[i][j] < '0' || argv[i][j] > '9'); ++j) { 141 | param[j-1] = argv[i][j]; 142 | } 143 | param[j-1] = 0; 144 | unsigned value = atoi(argv[i] + j); 145 | if (strcmp(param, "t") == 0) { 146 | g_nbSeconds = value; 147 | } 148 | else if (strcmp(param, "i") == 0) { 149 | g_iterations = value; 150 | } 151 | else if(strcmp(param, "d") == 0) { 152 | FL2_CCtx_setParameter(fcs, FL2_p_dictionaryLog, value); 153 | } 154 | else if (strcmp(param, "o") == 0) { 155 | FL2_CCtx_setParameter(fcs, FL2_p_overlapFraction, value); 156 | } 157 | else if (strcmp(param, "ds") == 0) { 158 | FL2_CCtx_setParameter(fcs, FL2_p_hybridChainLog, value); 159 | } 160 | else if (strcmp(param, "mc") == 0) { 161 | FL2_CCtx_setParameter(fcs, FL2_p_hybridCycles, value); 162 | } 163 | else if (strcmp(param, "sd") == 0) { 164 | FL2_CCtx_setParameter(fcs, FL2_p_searchDepth, value); 165 | } 166 | else if (strcmp(param, "fb") == 0) { 167 | FL2_CCtx_setParameter(fcs, FL2_p_fastLength, value); 168 | } 169 | else if (strcmp(param, "q") == 0) { 170 | FL2_CCtx_setParameter(fcs, FL2_p_divideAndConquer, value); 171 | } 172 | else if (strcmp(param, "bm") == 0) { 173 | FL2_CCtx_setParameter(fcs, FL2_p_resetInterval, value); 174 | } 175 | else if (strcmp(param, "b") == 0) { 176 | FL2_CCtx_setParameter(fcs, FL2_p_bufferResize, value); 177 | } 178 | else if (strcmp(param, "a") == 0) { 179 | FL2_CCtx_setParameter(fcs, FL2_p_strategy, value); 180 | } 181 | else if (strcmp(param, "h") == 0) { 182 | FL2_CCtx_setParameter(fcs, FL2_p_doXXHash, value); 183 | } 184 | else if (strcmp(param, "x") == 0) { 185 | FL2_CCtx_setParameter(fcs, FL2_p_highCompression, value); 186 | } 187 | else if (strcmp(param, "e") == 0) { 188 | end_level = value; 189 | } 190 | #ifdef RMF_REFERENCE 191 | else if (strcmp(param, "r") == 0) { 192 | FL2_CCtx_setParameter(fcs, FL2_p_useReferenceMF, value); 193 | } 194 | #endif 195 | } 196 | return end_level; 197 | } 198 | 199 | int FL2LIB_CALL main(int argc, char** argv) 200 | { 201 | if (argc < 2) 202 | return 1; 203 | size_t size; 204 | char* src; 205 | if (1) { 206 | FILE* f = fopen(argv[1], "rb"); 207 | if (f == NULL) 208 | return 1; 209 | fseek(f, 0, 2); 210 | size = ftell(f); 211 | if (size > 3UL << 29) size = 3UL << 29; 212 | fseek(f, 0, 0); 213 | src = malloc(size); 214 | size = fread(src, 1, size, f); 215 | fclose(f); 216 | } 217 | else { 218 | size = 1UL << 30; 219 | src = malloc(size); 220 | RDG_genBuffer(src, size, 0.6, 0.02, 1); 221 | } 222 | unsigned threads = 1; 223 | unsigned dthreads = ~0U; 224 | for (int i = 2; i < argc; ++i) { 225 | if (argv[i][0] == '-' && argv[i][1] == 'T') 226 | threads = atoi(argv[i] + 2); 227 | if (argv[i][0] == '-' && argv[i][1] == 'D') 228 | dthreads = atoi(argv[i] + 2); 229 | } 230 | if (dthreads == ~0U) 231 | dthreads = threads; 232 | FL2_CCtx* fcs = FL2_createCCtxMt(threads); 233 | FL2_DCtx* dctx = FL2_createDCtxMt(dthreads); 234 | if (fcs == NULL) 235 | return 1; 236 | int end_level = parse_params(fcs, argc, argv); 237 | int level = (int)FL2_CCtx_getParameter(fcs, FL2_p_compressionLevel); 238 | size_t maxCompressedSize = FL2_compressBound(size); 239 | char* compressedBuffer = malloc(maxCompressedSize); 240 | char* resultBuffer = malloc(size); 241 | if (!end_level) 242 | end_level = level; 243 | else if (end_level > FL2_maxCLevel()) 244 | end_level = FL2_maxCLevel(); 245 | for (; level <= end_level; ++level) { 246 | benchmark(fcs, dctx, src, size, compressedBuffer, maxCompressedSize, resultBuffer); 247 | FL2_CCtx_setParameter(fcs, FL2_p_compressionLevel, level + 1); 248 | printf("%u\r\n", level); 249 | } 250 | FL2_freeDCtx(dctx); 251 | FL2_freeCCtx(fcs); 252 | free(resultBuffer); 253 | free(compressedBuffer); 254 | free(src); 255 | return 0; 256 | } 257 | 258 | -------------------------------------------------------------------------------- /dll/fast-lzma2.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | Debug 14 | x64 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | {3B412B92-EC84-49A6-9C20-9BCFD9E34851} 23 | Win32Proj 24 | 8.1 25 | fast-lzma2 26 | 27 | 28 | 29 | DynamicLibrary 30 | true 31 | v140 32 | 33 | 34 | DynamicLibrary 35 | false 36 | v140 37 | true 38 | 39 | 40 | DynamicLibrary 41 | true 42 | v140 43 | Unicode 44 | 45 | 46 | DynamicLibrary 47 | false 48 | v140 49 | Unicode 50 | true 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | true 73 | fast-lzma2 74 | 75 | 76 | false 77 | fast-lzma2 78 | 79 | 80 | .dll 81 | fast-lzma2 82 | 83 | 84 | fast-lzma2 85 | .dll 86 | 87 | 88 | 89 | WIN32;_DEBUG;_WINDOWS;_USRDLL;FL2_DLL_EXPORT=1;%(PreprocessorDefinitions) 90 | 91 | 92 | MultiThreadedDebugDLL 93 | Level3 94 | ProgramDatabase 95 | Disabled 96 | 97 | 98 | MachineX86 99 | true 100 | Windows 101 | 102 | 103 | 104 | 105 | WIN32;NDEBUG;_WINDOWS;_USRDLL;FL2_DLL_EXPORT=1;%(PreprocessorDefinitions) 106 | 107 | 108 | MultiThreaded 109 | Level3 110 | None 111 | AnySuitable 112 | true 113 | true 114 | true 115 | FastCall 116 | false 117 | 118 | 119 | MachineX86 120 | false 121 | Windows 122 | true 123 | true 124 | UseLinkTimeCodeGeneration 125 | 126 | 127 | 128 | 129 | AnySuitable 130 | 131 | 132 | 133 | 134 | true 135 | 136 | 137 | 138 | 139 | true 140 | 141 | 142 | 143 | 144 | true 145 | LZMA2_DEC_OPT;FL2_DLL_EXPORT=1; 146 | false 147 | MultiThreaded 148 | true 149 | Level4 150 | MaxSpeed 151 | None 152 | false 153 | Neither 154 | 155 | 156 | UseLinkTimeCodeGeneration 157 | false 158 | false 159 | 160 | 161 | 162 | 163 | FL2_DEBUG=1;LZMA2_DEC_OPT;FL2_DLL_EXPORT=1; 164 | 165 | 166 | true 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | /Dx64 %(AdditionalOptions) 213 | -Dx64 %(AdditionalOptions) 214 | true 215 | true 216 | 217 | 218 | 219 | 220 | 221 | 222 | -------------------------------------------------------------------------------- /mem.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef MEM_H_MODULE 12 | #define MEM_H_MODULE 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | /*-**************************************** 19 | * Dependencies 20 | ******************************************/ 21 | #include /* size_t, ptrdiff_t */ 22 | #include /* memcpy */ 23 | 24 | 25 | /*-**************************************** 26 | * Compiler specifics 27 | ******************************************/ 28 | #if defined(_MSC_VER) /* Visual Studio */ 29 | # include /* _byteswap_ulong */ 30 | # include /* _byteswap_* */ 31 | #endif 32 | #if defined(__GNUC__) 33 | # define MEM_STATIC static __inline __attribute__((unused)) 34 | #elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 35 | # define MEM_STATIC static inline 36 | #elif defined(_MSC_VER) 37 | # define MEM_STATIC static __inline 38 | #else 39 | # define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ 40 | #endif 41 | 42 | #ifndef __has_builtin 43 | # define __has_builtin(x) 0 /* compat. with non-clang compilers */ 44 | #endif 45 | 46 | /* code only tested on 32 and 64 bits systems */ 47 | #define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; } 48 | MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } 49 | 50 | 51 | /*-************************************************************** 52 | * Basic Types 53 | *****************************************************************/ 54 | #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) 55 | # include 56 | typedef uint8_t BYTE; 57 | typedef uint16_t U16; 58 | typedef int16_t S16; 59 | typedef uint32_t U32; 60 | typedef int32_t S32; 61 | typedef uint64_t U64; 62 | typedef int64_t S64; 63 | #else 64 | # include 65 | #if CHAR_BIT != 8 66 | # error "this implementation requires char to be exactly 8-bit type" 67 | #endif 68 | typedef unsigned char BYTE; 69 | #if USHRT_MAX != 65535 70 | # error "this implementation requires short to be exactly 16-bit type" 71 | #endif 72 | typedef unsigned short U16; 73 | typedef signed short S16; 74 | #if UINT_MAX != 4294967295 75 | # error "this implementation requires int to be exactly 32-bit type" 76 | #endif 77 | typedef unsigned int U32; 78 | typedef signed int S32; 79 | /* note : there are no limits defined for long long type in C90. 80 | * limits exist in C99, however, in such case, is preferred */ 81 | typedef unsigned long long U64; 82 | typedef signed long long S64; 83 | #endif 84 | 85 | 86 | /*-************************************************************** 87 | * Memory I/O 88 | *****************************************************************/ 89 | /* MEM_FORCE_MEMORY_ACCESS : 90 | * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. 91 | * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. 92 | * The below switch allow to select different access method for improved performance. 93 | * Method 0 (default) : use `memcpy()`. Safe and portable. 94 | * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). 95 | * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. 96 | * Method 2 : direct access. This method is portable but violate C standard. 97 | * It can generate buggy code on targets depending on alignment. 98 | * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) 99 | * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. 100 | * Prefer these methods in priority order (0 > 1 > 2) 101 | */ 102 | #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ 103 | # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) 104 | # define MEM_FORCE_MEMORY_ACCESS 2 105 | # elif defined(__INTEL_COMPILER) || defined(__GNUC__) 106 | # define MEM_FORCE_MEMORY_ACCESS 1 107 | # endif 108 | #endif 109 | 110 | MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } 111 | MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } 112 | 113 | MEM_STATIC unsigned MEM_isLittleEndian(void) 114 | { 115 | const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ 116 | return one.c[0]; 117 | } 118 | 119 | #if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) 120 | 121 | /* violates C standard, by lying on structure alignment. 122 | Only use if no other choice to achieve best performance on target platform */ 123 | MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } 124 | MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } 125 | MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } 126 | MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } 127 | 128 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } 129 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } 130 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } 131 | 132 | #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) 133 | 134 | /* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ 135 | /* currently only defined for gcc and icc */ 136 | #if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) 137 | __pragma( pack(push, 1) ) 138 | typedef struct { U16 v; } unalign16; 139 | typedef struct { U32 v; } unalign32; 140 | typedef struct { U64 v; } unalign64; 141 | typedef struct { size_t v; } unalignArch; 142 | __pragma( pack(pop) ) 143 | #else 144 | typedef struct { U16 v; } __attribute__((packed)) unalign16; 145 | typedef struct { U32 v; } __attribute__((packed)) unalign32; 146 | typedef struct { U64 v; } __attribute__((packed)) unalign64; 147 | typedef struct { size_t v; } __attribute__((packed)) unalignArch; 148 | #endif 149 | 150 | MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; } 151 | MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; } 152 | MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; } 153 | MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; } 154 | 155 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; } 156 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; } 157 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; } 158 | 159 | #else 160 | 161 | /* default method, safe and standard. 162 | can sometimes prove slower */ 163 | 164 | MEM_STATIC U16 MEM_read16(const void* memPtr) 165 | { 166 | U16 val; memcpy(&val, memPtr, sizeof(val)); return val; 167 | } 168 | 169 | MEM_STATIC U32 MEM_read32(const void* memPtr) 170 | { 171 | U32 val; memcpy(&val, memPtr, sizeof(val)); return val; 172 | } 173 | 174 | MEM_STATIC U64 MEM_read64(const void* memPtr) 175 | { 176 | U64 val; memcpy(&val, memPtr, sizeof(val)); return val; 177 | } 178 | 179 | MEM_STATIC size_t MEM_readST(const void* memPtr) 180 | { 181 | size_t val; memcpy(&val, memPtr, sizeof(val)); return val; 182 | } 183 | 184 | MEM_STATIC void MEM_write16(void* memPtr, U16 value) 185 | { 186 | memcpy(memPtr, &value, sizeof(value)); 187 | } 188 | 189 | MEM_STATIC void MEM_write32(void* memPtr, U32 value) 190 | { 191 | memcpy(memPtr, &value, sizeof(value)); 192 | } 193 | 194 | MEM_STATIC void MEM_write64(void* memPtr, U64 value) 195 | { 196 | memcpy(memPtr, &value, sizeof(value)); 197 | } 198 | 199 | #endif /* MEM_FORCE_MEMORY_ACCESS */ 200 | 201 | MEM_STATIC U32 MEM_swap32(U32 in) 202 | { 203 | #if defined(_MSC_VER) /* Visual Studio */ 204 | return _byteswap_ulong(in); 205 | #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ 206 | || (defined(__clang__) && __has_builtin(__builtin_bswap32)) 207 | return __builtin_bswap32(in); 208 | #else 209 | return ((in << 24) & 0xff000000 ) | 210 | ((in << 8) & 0x00ff0000 ) | 211 | ((in >> 8) & 0x0000ff00 ) | 212 | ((in >> 24) & 0x000000ff ); 213 | #endif 214 | } 215 | 216 | MEM_STATIC U64 MEM_swap64(U64 in) 217 | { 218 | #if defined(_MSC_VER) /* Visual Studio */ 219 | return _byteswap_uint64(in); 220 | #elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ 221 | || (defined(__clang__) && __has_builtin(__builtin_bswap64)) 222 | return __builtin_bswap64(in); 223 | #else 224 | return ((in << 56) & 0xff00000000000000ULL) | 225 | ((in << 40) & 0x00ff000000000000ULL) | 226 | ((in << 24) & 0x0000ff0000000000ULL) | 227 | ((in << 8) & 0x000000ff00000000ULL) | 228 | ((in >> 8) & 0x00000000ff000000ULL) | 229 | ((in >> 24) & 0x0000000000ff0000ULL) | 230 | ((in >> 40) & 0x000000000000ff00ULL) | 231 | ((in >> 56) & 0x00000000000000ffULL); 232 | #endif 233 | } 234 | 235 | MEM_STATIC size_t MEM_swapST(size_t in) 236 | { 237 | if (MEM_32bits()) 238 | return (size_t)MEM_swap32((U32)in); 239 | else 240 | return (size_t)MEM_swap64((U64)in); 241 | } 242 | 243 | /*=== Little endian r/w ===*/ 244 | 245 | MEM_STATIC U16 MEM_readLE16(const void* memPtr) 246 | { 247 | if (MEM_isLittleEndian()) 248 | return MEM_read16(memPtr); 249 | else { 250 | const BYTE* p = (const BYTE*)memPtr; 251 | return (U16)(p[0] + (p[1]<<8)); 252 | } 253 | } 254 | 255 | MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) 256 | { 257 | if (MEM_isLittleEndian()) { 258 | MEM_write16(memPtr, val); 259 | } else { 260 | BYTE* p = (BYTE*)memPtr; 261 | p[0] = (BYTE)val; 262 | p[1] = (BYTE)(val>>8); 263 | } 264 | } 265 | 266 | MEM_STATIC U32 MEM_readLE24(const void* memPtr) 267 | { 268 | return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); 269 | } 270 | 271 | MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) 272 | { 273 | MEM_writeLE16(memPtr, (U16)val); 274 | ((BYTE*)memPtr)[2] = (BYTE)(val>>16); 275 | } 276 | 277 | MEM_STATIC U32 MEM_readLE32(const void* memPtr) 278 | { 279 | if (MEM_isLittleEndian()) 280 | return MEM_read32(memPtr); 281 | else 282 | return MEM_swap32(MEM_read32(memPtr)); 283 | } 284 | 285 | MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) 286 | { 287 | if (MEM_isLittleEndian()) 288 | MEM_write32(memPtr, val32); 289 | else 290 | MEM_write32(memPtr, MEM_swap32(val32)); 291 | } 292 | 293 | MEM_STATIC U64 MEM_readLE64(const void* memPtr) 294 | { 295 | if (MEM_isLittleEndian()) 296 | return MEM_read64(memPtr); 297 | else 298 | return MEM_swap64(MEM_read64(memPtr)); 299 | } 300 | 301 | MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) 302 | { 303 | if (MEM_isLittleEndian()) 304 | MEM_write64(memPtr, val64); 305 | else 306 | MEM_write64(memPtr, MEM_swap64(val64)); 307 | } 308 | 309 | MEM_STATIC size_t MEM_readLEST(const void* memPtr) 310 | { 311 | if (MEM_32bits()) 312 | return (size_t)MEM_readLE32(memPtr); 313 | else 314 | return (size_t)MEM_readLE64(memPtr); 315 | } 316 | 317 | MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) 318 | { 319 | if (MEM_32bits()) 320 | MEM_writeLE32(memPtr, (U32)val); 321 | else 322 | MEM_writeLE64(memPtr, (U64)val); 323 | } 324 | 325 | /*=== Big endian r/w ===*/ 326 | 327 | MEM_STATIC U32 MEM_readBE32(const void* memPtr) 328 | { 329 | if (MEM_isLittleEndian()) 330 | return MEM_swap32(MEM_read32(memPtr)); 331 | else 332 | return MEM_read32(memPtr); 333 | } 334 | 335 | MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) 336 | { 337 | if (MEM_isLittleEndian()) 338 | MEM_write32(memPtr, MEM_swap32(val32)); 339 | else 340 | MEM_write32(memPtr, val32); 341 | } 342 | 343 | MEM_STATIC U64 MEM_readBE64(const void* memPtr) 344 | { 345 | if (MEM_isLittleEndian()) 346 | return MEM_swap64(MEM_read64(memPtr)); 347 | else 348 | return MEM_read64(memPtr); 349 | } 350 | 351 | MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) 352 | { 353 | if (MEM_isLittleEndian()) 354 | MEM_write64(memPtr, MEM_swap64(val64)); 355 | else 356 | MEM_write64(memPtr, val64); 357 | } 358 | 359 | MEM_STATIC size_t MEM_readBEST(const void* memPtr) 360 | { 361 | if (MEM_32bits()) 362 | return (size_t)MEM_readBE32(memPtr); 363 | else 364 | return (size_t)MEM_readBE64(memPtr); 365 | } 366 | 367 | MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) 368 | { 369 | if (MEM_32bits()) 370 | MEM_writeBE32(memPtr, (U32)val); 371 | else 372 | MEM_writeBE64(memPtr, (U64)val); 373 | } 374 | 375 | 376 | #if defined (__cplusplus) 377 | } 378 | #endif 379 | 380 | #endif /* MEM_H_MODULE */ 381 | -------------------------------------------------------------------------------- /xxhash.h: -------------------------------------------------------------------------------- 1 | /* 2 | xxHash - Extremely Fast Hash algorithm 3 | Header File 4 | Copyright (C) 2012-2016, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - xxHash source repository : https://github.com/Cyan4973/xxHash 33 | */ 34 | 35 | /* Notice extracted from xxHash homepage : 36 | 37 | xxHash is an extremely fast Hash algorithm, running at RAM speed limits. 38 | It also successfully passes all tests from the SMHasher suite. 39 | 40 | Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) 41 | 42 | Name Speed Q.Score Author 43 | xxHash 5.4 GB/s 10 44 | CrapWow 3.2 GB/s 2 Andrew 45 | MumurHash 3a 2.7 GB/s 10 Austin Appleby 46 | SpookyHash 2.0 GB/s 10 Bob Jenkins 47 | SBox 1.4 GB/s 9 Bret Mulvey 48 | Lookup3 1.2 GB/s 9 Bob Jenkins 49 | SuperFastHash 1.2 GB/s 1 Paul Hsieh 50 | CityHash64 1.05 GB/s 10 Pike & Alakuijala 51 | FNV 0.55 GB/s 5 Fowler, Noll, Vo 52 | CRC32 0.43 GB/s 9 53 | MD5-32 0.33 GB/s 10 Ronald L. Rivest 54 | SHA1-32 0.28 GB/s 10 55 | 56 | Q.Score is a measure of quality of the hash function. 57 | It depends on successfully passing SMHasher test set. 58 | 10 is a perfect score. 59 | 60 | A 64-bits version, named XXH64, is available since r35. 61 | It offers much better speed, but for 64-bits applications only. 62 | Name Speed on 64 bits Speed on 32 bits 63 | XXH64 13.8 GB/s 1.9 GB/s 64 | XXH32 6.8 GB/s 6.0 GB/s 65 | */ 66 | 67 | #if defined (__cplusplus) 68 | extern "C" { 69 | #endif 70 | 71 | #ifndef XXHASH_H_5627135585666179 72 | #define XXHASH_H_5627135585666179 1 73 | 74 | 75 | /* **************************** 76 | * Definitions 77 | ******************************/ 78 | #include /* size_t */ 79 | typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; 80 | 81 | 82 | /* **************************** 83 | * API modifier 84 | ******************************/ 85 | /** XXH_PRIVATE_API 86 | * This is useful if you want to include xxhash functions in `static` mode 87 | * in order to inline them, and remove their symbol from the public list. 88 | * Methodology : 89 | * #define XXH_PRIVATE_API 90 | * #include "xxhash.h" 91 | * `xxhash.c` is automatically included. 92 | * It's not useful to compile and link it as a separate module anymore. 93 | */ 94 | #ifdef XXH_PRIVATE_API 95 | # ifndef XXH_STATIC_LINKING_ONLY 96 | # define XXH_STATIC_LINKING_ONLY 97 | # endif 98 | # if defined(__GNUC__) 99 | # define XXH_PUBLIC_API static __inline __attribute__((unused)) 100 | # elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 101 | # define XXH_PUBLIC_API static inline 102 | # elif defined(_MSC_VER) 103 | # define XXH_PUBLIC_API static __inline 104 | # else 105 | # define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ 106 | # endif 107 | #else 108 | # define XXH_PUBLIC_API /* do nothing */ 109 | #endif /* XXH_PRIVATE_API */ 110 | 111 | /*!XXH_NAMESPACE, aka Namespace Emulation : 112 | 113 | If you want to include _and expose_ xxHash functions from within your own library, 114 | but also want to avoid symbol collisions with another library which also includes xxHash, 115 | 116 | you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library 117 | with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). 118 | 119 | Note that no change is required within the calling program as long as it includes `xxhash.h` : 120 | regular symbol name will be automatically translated by this header. 121 | */ 122 | #ifdef XXH_NAMESPACE 123 | # define XXH_CAT(A,B) A##B 124 | # define XXH_NAME2(A,B) XXH_CAT(A,B) 125 | # define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) 126 | # define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) 127 | # define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) 128 | # define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) 129 | # define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) 130 | # define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) 131 | # define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) 132 | # define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) 133 | # define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) 134 | # define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) 135 | # define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) 136 | # define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) 137 | # define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) 138 | # define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) 139 | # define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) 140 | # define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) 141 | # define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) 142 | # define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) 143 | # define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) 144 | #endif 145 | 146 | 147 | /* ************************************* 148 | * Version 149 | ***************************************/ 150 | #define XXH_VERSION_MAJOR 0 151 | #define XXH_VERSION_MINOR 6 152 | #define XXH_VERSION_RELEASE 2 153 | #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) 154 | XXH_PUBLIC_API unsigned XXH_versionNumber (void); 155 | 156 | 157 | /* **************************** 158 | * Simple Hash Functions 159 | ******************************/ 160 | typedef unsigned int XXH32_hash_t; 161 | typedef unsigned long long XXH64_hash_t; 162 | 163 | XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); 164 | XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); 165 | 166 | /*! 167 | XXH32() : 168 | Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". 169 | The memory between input & input+length must be valid (allocated and read-accessible). 170 | "seed" can be used to alter the result predictably. 171 | Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s 172 | XXH64() : 173 | Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". 174 | "seed" can be used to alter the result predictably. 175 | This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). 176 | */ 177 | 178 | 179 | /* **************************** 180 | * Streaming Hash Functions 181 | ******************************/ 182 | typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ 183 | typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ 184 | 185 | /*! State allocation, compatible with dynamic libraries */ 186 | 187 | XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); 188 | XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); 189 | 190 | XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); 191 | XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); 192 | 193 | 194 | /* hash streaming */ 195 | 196 | XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); 197 | XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); 198 | XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); 199 | 200 | XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); 201 | XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); 202 | XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); 203 | 204 | /* 205 | These functions generate the xxHash of an input provided in multiple segments. 206 | Note that, for small input, they are slower than single-call functions, due to state management. 207 | For small input, prefer `XXH32()` and `XXH64()` . 208 | 209 | XXH state must first be allocated, using XXH*_createState() . 210 | 211 | Start a new hash by initializing state with a seed, using XXH*_reset(). 212 | 213 | Then, feed the hash state by calling XXH*_update() as many times as necessary. 214 | Obviously, input must be allocated and read accessible. 215 | The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. 216 | 217 | Finally, a hash value can be produced anytime, by using XXH*_digest(). 218 | This function returns the nn-bits hash as an int or long long. 219 | 220 | It's still possible to continue inserting input into the hash state after a digest, 221 | and generate some new hashes later on, by calling again XXH*_digest(). 222 | 223 | When done, free XXH state space if it was allocated dynamically. 224 | */ 225 | 226 | 227 | /* ************************** 228 | * Utils 229 | ****************************/ 230 | #if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ 231 | # define restrict /* disable restrict */ 232 | #endif 233 | 234 | XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); 235 | XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); 236 | 237 | 238 | /* ************************** 239 | * Canonical representation 240 | ****************************/ 241 | /* Default result type for XXH functions are primitive unsigned 32 and 64 bits. 242 | * The canonical representation uses human-readable write convention, aka big-endian (large digits first). 243 | * These functions allow transformation of hash result into and from its canonical format. 244 | * This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. 245 | */ 246 | typedef struct { unsigned char digest[4]; } XXH32_canonical_t; 247 | typedef struct { unsigned char digest[8]; } XXH64_canonical_t; 248 | 249 | XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); 250 | XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); 251 | 252 | XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); 253 | XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); 254 | 255 | #endif /* XXHASH_H_5627135585666179 */ 256 | 257 | 258 | 259 | /* ================================================================================================ 260 | This section contains definitions which are not guaranteed to remain stable. 261 | They may change in future versions, becoming incompatible with a different version of the library. 262 | They shall only be used with static linking. 263 | Never use these definitions in association with dynamic linking ! 264 | =================================================================================================== */ 265 | #if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXH_STATIC_H_3543687687345) 266 | #define XXH_STATIC_H_3543687687345 267 | 268 | /* These definitions are only meant to allow allocation of XXH state 269 | statically, on stack, or in a struct for example. 270 | Do not use members directly. */ 271 | 272 | struct XXH32_state_s { 273 | unsigned total_len_32; 274 | unsigned large_len; 275 | unsigned v1; 276 | unsigned v2; 277 | unsigned v3; 278 | unsigned v4; 279 | unsigned mem32[4]; /* buffer defined as U32 for alignment */ 280 | unsigned memsize; 281 | unsigned reserved; /* never read nor write, will be removed in a future version */ 282 | }; /* typedef'd to XXH32_state_t */ 283 | 284 | struct XXH64_state_s { 285 | unsigned long long total_len; 286 | unsigned long long v1; 287 | unsigned long long v2; 288 | unsigned long long v3; 289 | unsigned long long v4; 290 | unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ 291 | unsigned memsize; 292 | unsigned reserved[2]; /* never read nor write, will be removed in a future version */ 293 | }; /* typedef'd to XXH64_state_t */ 294 | 295 | 296 | # ifdef XXH_PRIVATE_API 297 | # include "xxhash.c" /* include xxhash functions as `static`, for inlining */ 298 | # endif 299 | 300 | #endif /* XXH_STATIC_LINKING_ONLY && XXH_STATIC_H_3543687687345 */ 301 | 302 | 303 | #if defined (__cplusplus) 304 | } 305 | #endif 306 | --------------------------------------------------------------------------------