├── contrib ├── debian │ ├── docs │ ├── compat │ ├── dirs │ ├── liblz4.install │ ├── liblz4-dev.install │ ├── rules │ ├── copyright │ ├── changelog │ └── control ├── gen_manual │ ├── .gitignore │ ├── gen-lz4-manual.sh │ ├── README.md │ └── Makefile ├── meson │ ├── meson │ │ ├── contrib │ │ │ ├── meson.build │ │ │ └── gen_manual │ │ │ │ └── meson.build │ │ ├── examples │ │ │ └── meson.build │ │ ├── GetLz4LibraryVersion.py │ │ ├── programs │ │ │ └── meson.build │ │ ├── meson.build │ │ ├── tests │ │ │ └── meson.build │ │ └── lib │ │ │ └── meson.build │ ├── README.md │ ├── meson.build │ └── meson_options.txt ├── djgpp │ ├── README.MD │ ├── LICENSE │ └── Makefile └── snap │ ├── snapcraft.yaml │ └── README.md ├── lib ├── .gitignore ├── liblz4.pc.in ├── liblz4-dll.rc.in ├── dll │ └── example │ │ ├── fullbench-dll.sln │ │ ├── Makefile │ │ └── README.md ├── LICENSE ├── lz4frame_static.h └── lz4file.h ├── tests ├── goldenSamples │ └── skip.bin ├── check_liblz4_version.sh ├── .gitignore ├── test_install.sh ├── unicode_lint.sh ├── test_custom_block_sizes.sh ├── decompress-partial.c ├── checkTag.c ├── README.md ├── decompress-partial-usingDict.c ├── datagencli.c └── test-lz4-versions.py ├── .cirrus.yml ├── ossfuzz ├── .gitignore ├── lz4_helpers.h ├── ossfuzz.sh ├── travisoss.sh ├── fuzz_data_producer.h ├── round_trip_hc_fuzzer.c ├── round_trip_frame_fuzzer.c ├── compress_frame_fuzzer.c ├── fuzz.h ├── lz4_helpers.c ├── standaloneengine.c ├── compress_fuzzer.c ├── compress_hc_fuzzer.c ├── decompress_frame_fuzzer.c ├── fuzz_data_producer.c ├── Makefile ├── fuzz_helpers.h ├── decompress_fuzzer.c ├── round_trip_fuzzer.c └── round_trip_frame_uncompressed_fuzzer.c ├── examples ├── .gitignore ├── printVersion.c ├── README.md ├── dictionaryRandomAccess.md ├── blockStreaming_doubleBuffer.md ├── streaming_api_basics.md ├── blockStreaming_lineByLine.md ├── Makefile ├── blockStreaming_doubleBuffer.c ├── blockStreaming_ringBuffer.c ├── simple_buffer.c ├── blockStreaming_lineByLine.c └── fileCompress.c ├── programs ├── .gitignore ├── lz4-exe.rc.in ├── datagen.h ├── bench.h ├── README.md └── lz4io.h ├── .gitattributes ├── .gitignore ├── LICENSE ├── INSTALL ├── .github ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md └── workflows │ └── README.md ├── .circleci ├── images │ └── primary │ │ └── Dockerfile └── config.yml ├── .travis.yml ├── Makefile.inc ├── README.md └── appveyor.yml /contrib/debian/docs: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /contrib/debian/compat: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /contrib/debian/dirs: -------------------------------------------------------------------------------- 1 | usr/bin 2 | -------------------------------------------------------------------------------- /lib/.gitignore: -------------------------------------------------------------------------------- 1 | # make install artefact 2 | liblz4.pc 3 | -------------------------------------------------------------------------------- /contrib/debian/liblz4.install: -------------------------------------------------------------------------------- 1 | usr/lib/liblz4.so.* 2 | usr/bin/* 3 | -------------------------------------------------------------------------------- /contrib/gen_manual/.gitignore: -------------------------------------------------------------------------------- 1 | # build artefact 2 | gen_manual 3 | -------------------------------------------------------------------------------- /tests/goldenSamples/skip.bin: -------------------------------------------------------------------------------- 1 | Z*M -------------------------------------------------------------------------------- /contrib/debian/liblz4-dev.install: -------------------------------------------------------------------------------- 1 | usr/include/lz4* 2 | usr/lib/liblz4.so 3 | -------------------------------------------------------------------------------- /.cirrus.yml: -------------------------------------------------------------------------------- 1 | freebsd_instance: 2 | image_family: freebsd-12-2 3 | 4 | task: 5 | script: pkg install -y gmake && gmake test 6 | -------------------------------------------------------------------------------- /ossfuzz/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # build artefacts 3 | round_trip_frame_uncompressed_fuzzer 4 | 5 | # test artefacts 6 | 7 | # local tests 8 | 9 | -------------------------------------------------------------------------------- /tests/check_liblz4_version.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | set -e 3 | 4 | # written as a script shell, because pipe management in python is horrible 5 | ldd $1 | grep liblz4 6 | 7 | -------------------------------------------------------------------------------- /contrib/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | include /usr/share/cdbs/1/rules/debhelper.mk 4 | include /usr/share/cdbs/1/class/cmake.mk 5 | 6 | 7 | DEB_CMAKE_EXTRA_FLAGS := -DCMAKE_BUILD_TYPE=RelWithDebInfo ../../build/cmake 8 | -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | /Makefile.lz4* 2 | /printVersion 3 | /doubleBuffer 4 | /dictionaryRandomAccess 5 | /ringBuffer 6 | /ringBufferHC 7 | /lineCompress 8 | /frameCompress 9 | /fileCompress 10 | /simpleBuffer 11 | /*.exe 12 | -------------------------------------------------------------------------------- /programs/.gitignore: -------------------------------------------------------------------------------- 1 | # local binary (Makefile) 2 | lz4 3 | unlz4 4 | lz4cat 5 | lz4c 6 | lz4c32 7 | lz4-wlib 8 | datagen 9 | frametest 10 | frametest32 11 | fullbench 12 | fullbench32 13 | fuzzer 14 | fuzzer32 15 | *.exe 16 | 17 | # tests files 18 | tmp* 19 | 20 | # artefacts 21 | *.dSYM 22 | -------------------------------------------------------------------------------- /examples/printVersion.c: -------------------------------------------------------------------------------- 1 | // LZ4 trivial example : print Library version number 2 | // by Takayuki Matsuoka 3 | 4 | 5 | #include 6 | #include "lz4.h" 7 | 8 | int main(int argc, char** argv) 9 | { 10 | (void)argc; (void)argv; 11 | printf("Hello World ! LZ4 Library version = %d\n", LZ4_versionNumber()); 12 | return 0; 13 | } 14 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # LZ4 examples 2 | 3 | All examples are GPL-v2 licensed. 4 | 5 | ## Documents 6 | 7 | - [Streaming API Basics](streaming_api_basics.md) 8 | - Examples 9 | - [Double Buffer](blockStreaming_doubleBuffer.md) 10 | - [Line by Line Text Compression](blockStreaming_lineByLine.md) 11 | - [Dictionary Random Access](dictionaryRandomAccess.md) 12 | -------------------------------------------------------------------------------- /contrib/debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: liblz4 3 | Upstream-Contact: Yann Collet 4 | Source: https://github.com/lz4/lz4 5 | 6 | Files: * 7 | Copyright: (C) 2011-2020 Yann Collet 8 | License: GPL-2+ 9 | The full text of license: https://github.com/lz4/lz4/blob/dev/lib/LICENSE 10 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # build artefacts 3 | datagen 4 | frametest 5 | frametest32 6 | fullbench 7 | fullbench32 8 | fuzzer 9 | fuzzer32 10 | fasttest 11 | roundTripTest 12 | checkTag 13 | checkFrame 14 | decompress-partial 15 | decompress-partial-usingDict 16 | abiTest 17 | freestanding 18 | 19 | # test artefacts 20 | tmp* 21 | versionsTest 22 | abiTests 23 | lz4_all.c 24 | 25 | # local tests 26 | afl 27 | -------------------------------------------------------------------------------- /ossfuzz/lz4_helpers.h: -------------------------------------------------------------------------------- 1 | #ifndef LZ4_HELPERS 2 | #define LZ4_HELPERS 3 | 4 | #include "lz4frame.h" 5 | 6 | LZ4F_frameInfo_t FUZZ_randomFrameInfo(uint32_t* seed); 7 | 8 | LZ4F_preferences_t FUZZ_randomPreferences(uint32_t* seed); 9 | 10 | size_t FUZZ_decompressFrame(void* dst, const size_t dstCapacity, 11 | const void* src, const size_t srcSize); 12 | 13 | #endif /* LZ4_HELPERS */ 14 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set the default behavior 2 | * text eol=lf 3 | 4 | # Explicitly declare source files 5 | *.c text eol=lf 6 | *.h text eol=lf 7 | 8 | # Denote files that should not be modified. 9 | *.odt binary 10 | *.png binary 11 | 12 | # Visual Studio 13 | *.sln text eol=crlf 14 | *.vcxproj* text eol=crlf 15 | *.vcproj* text eol=crlf 16 | *.suo binary 17 | *.rc text eol=crlf 18 | 19 | # Windows 20 | *.bat text eol=crlf 21 | *.cmd text eol=crlf 22 | -------------------------------------------------------------------------------- /lib/liblz4.pc.in: -------------------------------------------------------------------------------- 1 | # LZ4 - Fast LZ compression algorithm 2 | # Copyright (C) 2011-2020, Yann Collet. 3 | # BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 4 | 5 | prefix=@PREFIX@ 6 | libdir=@LIBDIR@ 7 | includedir=@INCLUDEDIR@ 8 | 9 | Name: lz4 10 | Description: extremely fast lossless compression algorithm library 11 | URL: http://www.lz4.org/ 12 | Version: @VERSION@ 13 | Libs: -L${libdir} -llz4 14 | Cflags: -I${includedir} 15 | -------------------------------------------------------------------------------- /contrib/debian/changelog: -------------------------------------------------------------------------------- 1 | liblz4 (1.7.2) unstable; urgency=low 2 | 3 | * Changed : moved to versioning; package, cli and library have same version number 4 | * Improved: Small decompression speed boost (+4%) 5 | * Improved: Performance on ARMv6 and ARMv7 6 | * Added : Debianization, by Evgeniy Polyakov 7 | * Makefile: Generates object files (*.o) for faster (re)compilation on low power systems 8 | * Fix : cli : crash on some invalid inputs 9 | 10 | -- Yann Collet Sun, 28 Jun 2015 01:00:00 +0000 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | *.ko 4 | 5 | # Libraries 6 | *.lib 7 | *.a 8 | 9 | # Shared objects (inc. Windows DLLs) 10 | *.dll 11 | *.so 12 | *.so.* 13 | *.dylib 14 | *.dSYM # apple 15 | 16 | # Executables 17 | *.exe 18 | *.out 19 | *.app 20 | lz4 21 | 22 | # IDE / editors files 23 | .clang_complete 24 | _codelite/ 25 | _codelite_lz4/ 26 | bin/ 27 | *.zip 28 | *.swp 29 | 30 | # analyzers 31 | infer-out 32 | 33 | # Mac 34 | .DS_Store 35 | *.dSYM 36 | 37 | # Windows / Msys 38 | nul 39 | ld.exe* 40 | 41 | # test artifacts 42 | *.lz4 43 | tmp* 44 | -------------------------------------------------------------------------------- /ossfuzz/ossfuzz.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | # This script is called by the oss-fuzz main project when compiling the fuzz 4 | # targets. This script is regression tested by travisoss.sh. 5 | 6 | # Save off the current folder as the build root. 7 | export BUILD_ROOT=$PWD 8 | 9 | echo "CC: $CC" 10 | echo "CXX: $CXX" 11 | echo "LIB_FUZZING_ENGINE: $LIB_FUZZING_ENGINE" 12 | echo "CFLAGS: $CFLAGS" 13 | echo "CXXFLAGS: $CXXFLAGS" 14 | echo "OUT: $OUT" 15 | 16 | export MAKEFLAGS+="-j$(nproc)" 17 | 18 | pushd ossfuzz 19 | make V=1 all 20 | popd 21 | 22 | # Copy the fuzzers to the target directory. 23 | cp -v ossfuzz/*_fuzzer $OUT/ 24 | -------------------------------------------------------------------------------- /contrib/meson/meson/contrib/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # Copyright (c) 2022-present Tristan Partin 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | 11 | subdir('gen_manual') 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This repository uses 2 different licenses : 2 | - all files in the `lib` directory use a BSD 2-Clause license 3 | - all other files use a GPLv2 license, unless explicitly stated otherwise 4 | 5 | Relevant license is reminded at the top of each source file, 6 | and with presence of COPYING or LICENSE file in associated directories. 7 | 8 | This model is selected to emphasize that 9 | files in the `lib` directory are designed to be included into 3rd party applications, 10 | while all other files, in `programs`, `tests` or `examples`, 11 | are intended to be used "as is", as part of their intended scenarios, 12 | with no intention to support 3rd party integration use cases. 13 | -------------------------------------------------------------------------------- /contrib/gen_manual/gen-lz4-manual.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | LIBVER_MAJOR_SCRIPT=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/lz4.h` 4 | LIBVER_MINOR_SCRIPT=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/lz4.h` 5 | LIBVER_PATCH_SCRIPT=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/lz4.h` 6 | LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT 7 | 8 | echo LZ4_VERSION=$LIBVER_SCRIPT 9 | ./gen_manual "lz4 $LIBVER_SCRIPT" ../../lib/lz4.h ./lz4_manual.html 10 | ./gen_manual "lz4frame $LIBVER_SCRIPT" ../../lib/lz4frame.h ./lz4frame_manual.html 11 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Installation 2 | ============= 3 | 4 | ``` 5 | make 6 | make install # this command may require root access 7 | ``` 8 | 9 | LZ4's `Makefile` supports standard [Makefile conventions], 10 | including [staged installs], [redirection], or [command redefinition]. 11 | It is compatible with parallel builds (`-j#`). 12 | 13 | [Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html 14 | [staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html 15 | [redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html 16 | [command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.circleci/images/primary/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM circleci/buildpack-deps:bionic 2 | 3 | RUN sudo apt-get -y -qq update 4 | RUN sudo apt-get -y install software-properties-common 5 | RUN sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test 6 | RUN sudo apt-get -y install cmake 7 | RUN sudo apt-get -y install qemu-system-ppc qemu-user-static qemu-system-arm 8 | RUN sudo apt-get -y install libc6-dev-armel-cross libc6-dev-arm64-cross libc6-dev-i386 9 | RUN sudo apt-get -y install clang clang-tools 10 | RUN sudo apt-get -y install gcc-5 gcc-5-multilib gcc-6 11 | RUN sudo apt-get -y install valgrind 12 | RUN sudo apt-get -y install gcc-multilib-powerpc-linux-gnu gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu 13 | -------------------------------------------------------------------------------- /contrib/debian/control: -------------------------------------------------------------------------------- 1 | Source: liblz4 2 | Section: devel 3 | Priority: optional 4 | Maintainer: Evgeniy Polyakov 5 | Build-Depends: 6 | cmake (>= 2.6), 7 | debhelper (>= 7.0.50~), 8 | cdbs 9 | Standards-Version: 3.8.0 10 | Homepage: http://www.lz4.org/ 11 | Vcs-Git: git://github.com/lz4/lz4.git 12 | Vcs-Browser: https://github.com/lz4/lz4 13 | 14 | Package: liblz4 15 | Architecture: any 16 | Depends: ${shlibs:Depends}, ${misc:Depends} 17 | Description: Extremely Fast Compression algorithm http://www.lz4.org 18 | 19 | Package: liblz4-dev 20 | Architecture: any 21 | Depends: ${shlibs:Depends}, ${misc:Depends} 22 | Description: Extremely Fast Compression algorithm http://www.lz4.org 23 | Development files. 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Expected behavior** 14 | Please describe what you expected to happen. 15 | 16 | **To Reproduce** 17 | Steps to reproduce the behavior: 18 | 1. Go to '...' 19 | 2. Click on '....' 20 | 3. Scroll down to '....' 21 | 4. See error '...' 22 | If applicable, add screenshots to help explain your problem. 23 | 24 | **System (please complete the following information):** 25 | - OS: [e.g. Mac] 26 | - Version [e.g. 22] 27 | - Compiler [e.g. gcc] 28 | - Build System [e.g. Makefile] 29 | - Other hardware specs [e.g. Core 2 duo...] 30 | 31 | **Additional context** 32 | Add any other context about the problem here. 33 | -------------------------------------------------------------------------------- /ossfuzz/travisoss.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -ex 4 | 5 | # Clone the oss-fuzz repository 6 | git clone https://github.com/google/oss-fuzz.git /tmp/ossfuzz 7 | 8 | if [[ ! -d /tmp/ossfuzz/projects/lz4 ]] 9 | then 10 | echo "Could not find the lz4 project in ossfuzz" 11 | exit 1 12 | fi 13 | 14 | # Modify the oss-fuzz Dockerfile so that we're checking out the current branch on travis. 15 | if [ "x${TRAVIS_PULL_REQUEST}" = "xfalse" ] 16 | then 17 | sed -i "s@https://github.com/lz4/lz4.git@-b ${TRAVIS_BRANCH} https://github.com/lz4/lz4.git@" /tmp/ossfuzz/projects/lz4/Dockerfile 18 | else 19 | sed -i "s@https://github.com/lz4/lz4.git@-b ${TRAVIS_PULL_REQUEST_BRANCH} https://github.com/${TRAVIS_PULL_REQUEST_SLUG}.git@" /tmp/ossfuzz/projects/lz4/Dockerfile 20 | fi 21 | 22 | # Try and build the fuzzers 23 | pushd /tmp/ossfuzz 24 | python infra/helper.py build_image --pull lz4 25 | python infra/helper.py build_fuzzers lz4 26 | popd 27 | -------------------------------------------------------------------------------- /programs/lz4-exe.rc.in: -------------------------------------------------------------------------------- 1 | 1 VERSIONINFO 2 | FILEVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0 3 | PRODUCTVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0 4 | FILEFLAGSMASK 0 5 | FILEOS 0x40000 6 | FILETYPE 1 7 | { 8 | BLOCK "StringFileInfo" 9 | { 10 | BLOCK "040904B0" 11 | { 12 | VALUE "CompanyName", "Yann Collet" 13 | VALUE "FileDescription", "Extremely fast compression" 14 | VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0" 15 | VALUE "InternalName", "@PROGNAME@" 16 | VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet" 17 | VALUE "OriginalFilename", "@PROGNAME@.@EXT@" 18 | VALUE "ProductName", "LZ4" 19 | VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0" 20 | } 21 | } 22 | BLOCK "VarFileInfo" 23 | { 24 | VALUE "Translation", 0x0409, 1200 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /contrib/meson/README.md: -------------------------------------------------------------------------------- 1 | Meson build system for lz4 2 | ========================== 3 | 4 | Meson is a build system designed to optimize programmer productivity. 5 | It aims to do this by providing simple, out-of-the-box support for 6 | modern software development tools and practices, such as unit tests, 7 | coverage reports, Valgrind, CCache and the like. 8 | 9 | This Meson build system is provided with no guarantee. 10 | 11 | ## How to build 12 | 13 | `cd` to this meson directory (`contrib/meson`) 14 | 15 | ```sh 16 | meson setup --buildtype=release -Ddefault_library=shared -Dprograms=true builddir 17 | cd builddir 18 | ninja # to build 19 | ninja install # to install 20 | ``` 21 | 22 | You might want to install it in staging directory: 23 | 24 | ```sh 25 | DESTDIR=./staging ninja install 26 | ``` 27 | 28 | To configure build options, use: 29 | 30 | ```sh 31 | meson configure 32 | ``` 33 | 34 | See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html). 35 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | 3 | matrix: 4 | fast_finish: true 5 | include: 6 | 7 | - name: aarch64 real-hw tests 8 | arch: arm64 9 | script: 10 | - make test 11 | 12 | - name: PPC64LE real-hw tests 13 | arch: ppc64le 14 | script: 15 | - make test 16 | 17 | - name: IBM s390x real-hw tests 18 | arch: s390x 19 | script: 20 | - make test 21 | 22 | # tag-specific test 23 | - name: tag build 24 | if: tag =~ ^v[0-9]\.[0-9] 25 | os: linux 26 | script: 27 | - make -C tests checkTag 28 | - tests/checkTag "$TRAVIS_BRANCH" 29 | 30 | # oss-fuzz compilation test 31 | - name: Compile OSS-Fuzz targets 32 | script: 33 | - ./ossfuzz/travisoss.sh 34 | 35 | # Unicode lint 36 | # See https://github.com/lz4/lz4/issues/1018 37 | - name: Run Unicode lint 38 | script: 39 | - ./tests/unicode_lint.sh 40 | 41 | allow_failures: 42 | - env: ALLOW_FAILURES=true 43 | -------------------------------------------------------------------------------- /contrib/meson/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ############################################################################# 9 | 10 | # This is a dummy meson file. 11 | # The intention is that it can be easily moved to the root of the project 12 | # (together with meson_options.txt) and packaged for wrapdb. 13 | 14 | project( 15 | 'lz4', 16 | ['c'], 17 | license: 'BSD-2-Clause-Patent AND GPL-2.0-or-later', 18 | default_options: [ 19 | 'c_std=c99', 20 | 'buildtype=release', 21 | 'warning_level=3' 22 | ], 23 | version: 'DUMMY', 24 | meson_version: '>=0.49.0' 25 | ) 26 | 27 | subdir('meson') 28 | -------------------------------------------------------------------------------- /tests/test_install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | set -e 3 | 4 | 5 | make="make -C $lz4_root" 6 | unamestr=$(uname) 7 | if [ "$unamestr" = 'Linux' ]; then 8 | make="make -C $lz4_root" 9 | elif [ "$unamestr" = 'FreeBSD' ] || [ "$unamestr" = 'OpenBSD' ]; then 10 | make="gmake -C $lz4_root" 11 | fi 12 | 13 | for cmd in install uninstall; do 14 | for upper in DUMMY PREFIX EXEC_PREFIX LIBDIR INCLUDEDIR PKGCONFIGDIR BINDIR MANDIR MAN1DIR ; do 15 | lower=$(echo $upper | tr '[:upper:]' '[:lower:]') 16 | tmp_lower="$(pwd)/tmp-lower-$lower/" 17 | tmp_upper="$(pwd)/tmp-upper-$lower/" 18 | echo $make $cmd DESTDIR="$tmp_upper" $upper="test" 19 | $make $cmd DESTDIR="$tmp_upper" $upper="test" >/dev/null 20 | echo $make $cmd DESTDIR="$tmp_lower" $lower="test" 21 | $make $cmd DESTDIR="$tmp_lower" $lower="test" >/dev/null 22 | command diff -r "$tmp_lower" "$tmp_upper" && echo "SAME!" || false 23 | if [ "x$cmd" = "xuninstall" ]; then 24 | test -z "$(find "$tmp_lower" -type f)" && echo "EMPTY!" || false 25 | rm -rf "$tmp_upper" "$tmp_lower" 26 | fi 27 | done 28 | done 29 | -------------------------------------------------------------------------------- /contrib/djgpp/README.MD: -------------------------------------------------------------------------------- 1 | # lz4 for DOS/djgpp 2 | This file details on how to compile lz4.exe, and liblz4.a for use on DOS/djgpp using 3 | Andrew Wu's build-djgpp cross compilers ([GH][0], [Binaries][1]) on OSX, Linux. 4 | 5 | ## Setup 6 | * Download a djgpp tarball [binaries][1] for your platform. 7 | * Extract and install it (`tar jxvf djgpp-linux64-gcc492.tar.bz2`). Note the path. We'll assume `/home/user/djgpp`. 8 | * Add the `bin` folder to your `PATH`. In bash, do `export PATH=/home/user/djgpp/bin:$PATH`. 9 | * The `Makefile` in `contrib/djgpp/` sets up `CC`, `AR`, `LD` for you. So, `CC=i586-pc-msdosdjgpp-gcc`, `AR=i586-pc-msdosdjgpp-ar`, `LD=i586-pc-msdosdjgpp-gcc`. 10 | 11 | ## Building LZ4 for DOS 12 | In the base dir of lz4 and with `contrib/djgpp/Makefile`, try: 13 | Try: 14 | * `make -f contrib/djgpp/Makefile` 15 | * `make -f contrib/djgpp/Makefile liblz4.a` 16 | * `make -f contrib/djgpp/Makefile lz4.exe` 17 | * `make -f contrib/djgpp/Makefile DESTDIR=/home/user/dos install`, however it doesn't make much sense on a \*nix. 18 | * You can also do `make -f contrib/djgpp/Makefile uninstall` 19 | 20 | [0]: https://github.com/andrewwutw/build-djgpp 21 | [1]: https://github.com/andrewwutw/build-djgpp/releases 22 | -------------------------------------------------------------------------------- /contrib/snap/snapcraft.yaml: -------------------------------------------------------------------------------- 1 | name: lz4 2 | version: 1.9.3 3 | summary: Extremely Fast Compression algorithm 4 | description: > 5 | LZ4 is lossless compression algorithm, providing compression 6 | speed > 500 MB/s per core, scalable with multi-cores CPU. It features an 7 | extremely fast decoder, with speed in multiple GB/s per core, typically 8 | reaching RAM speed limits on multi-core systems. 9 | . 10 | Speed can be tuned dynamically, selecting an "acceleration" factor which 11 | trades compression ratio for faster speed. On the other end, a high 12 | compression derivative, LZ4_HC, is also provided, trading CPU time for 13 | improved compression ratio. All versions feature the same decompression 14 | speed. 15 | . 16 | LZ4 is also compatible with dictionary compression, and can ingest any 17 | input file as dictionary, including those created by Zstandard Dictionary 18 | Builder. (note: only the final 64KB are used). 19 | . 20 | LZ4 library is provided as open-source software using BSD 2-Clause license. 21 | confinement: strict 22 | grade: stable 23 | 24 | apps: 25 | lz4: 26 | command: usr/local/bin/lz4 27 | plugs: [home] 28 | parts: 29 | lz4: 30 | source: ../ 31 | plugin: make 32 | -------------------------------------------------------------------------------- /lib/liblz4-dll.rc.in: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | // DLL version information. 4 | 1 VERSIONINFO 5 | FILEVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0 6 | PRODUCTVERSION @LIBVER_MAJOR@,@LIBVER_MINOR@,@LIBVER_PATCH@,0 7 | FILEFLAGSMASK VS_FFI_FILEFLAGSMASK 8 | #ifdef _DEBUG 9 | FILEFLAGS VS_FF_DEBUG | VS_FF_PRERELEASE 10 | #else 11 | FILEFLAGS 0 12 | #endif 13 | FILEOS VOS_NT_WINDOWS32 14 | FILETYPE VFT_DLL 15 | FILESUBTYPE VFT2_UNKNOWN 16 | BEGIN 17 | BLOCK "StringFileInfo" 18 | BEGIN 19 | BLOCK "040904B0" 20 | BEGIN 21 | VALUE "CompanyName", "Yann Collet" 22 | VALUE "FileDescription", "Extremely fast compression" 23 | VALUE "FileVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0" 24 | VALUE "InternalName", "@LIBLZ4@" 25 | VALUE "LegalCopyright", "Copyright (C) 2013-2020, Yann Collet" 26 | VALUE "OriginalFilename", "@LIBLZ4@.dll" 27 | VALUE "ProductName", "LZ4" 28 | VALUE "ProductVersion", "@LIBVER_MAJOR@.@LIBVER_MINOR@.@LIBVER_PATCH@.0" 29 | END 30 | END 31 | BLOCK "VarFileInfo" 32 | BEGIN 33 | VALUE "Translation", 0x0409, 1200 34 | END 35 | END 36 | -------------------------------------------------------------------------------- /contrib/gen_manual/README.md: -------------------------------------------------------------------------------- 1 | gen_manual - a program for automatic generation of manual from source code 2 | ========================================================================== 3 | 4 | #### Introduction 5 | 6 | This simple C++ program generates a single-page HTML manual from `lz4.h`. 7 | 8 | The format of recognized comment blocks is following: 9 | - comments of type `/*!` mean: this is a function declaration; switch comments with declarations 10 | - comments of type `/**` and `/*-` mean: this is a comment; use a `

` header for the first line 11 | - comments of type `/*=` and `/**=` mean: use a `

` header and show also all functions until first empty line 12 | - comments of type `/*X` where `X` is different from above-mentioned are ignored 13 | 14 | Moreover: 15 | - `LZ4LIB_API` is removed to improve readability 16 | - `typedef` are detected and included even if uncommented 17 | - comments of type `/**<` and `/*!<` are detected and only function declaration is highlighted (bold) 18 | 19 | 20 | #### Usage 21 | 22 | The program requires 3 parameters: 23 | ``` 24 | gen_manual [lz4_version] [input_file] [output_html] 25 | ``` 26 | 27 | To compile program and generate lz4 manual we have used: 28 | ``` 29 | make 30 | ./gen_manual.exe 1.7.3 ../../lib/lz4.h lz4_manual.html 31 | ``` 32 | -------------------------------------------------------------------------------- /contrib/snap/README.md: -------------------------------------------------------------------------------- 1 | Snap Packaging 2 | -------------- 3 | 4 | This directory contains the config required to generate a snap package 5 | of lz4. Snaps are universal Linux packages that allow you to easily 6 | build your application from any source and ship it to any Linux 7 | distribution by publishing it to https://snapcraft.io/. A key attribute 8 | of a snap package is that it is (ideally) confined such that it 9 | executes within a controlled environment with all its dependencies 10 | bundled with it and does not share dependencies with of from any other 11 | package on the system (with a couple of minor exceptions). 12 | 13 | The basic anatomy and workflow is: 14 | 15 | * ensure snap.snapcraft.yaml is up-to-date e.g. with version info 16 | 17 | * build the snap by installing the snapcraft package and running it 18 | 19 | * push snap/* changes to the repo (excluding any crud generated by a build of course) 20 | 21 | * register yourself as owner of lz4 name in snapstore 22 | 23 | * publish new snap to the snap store 24 | 25 | * install snap by doing 'snap install lz4' on any Linux distro 26 | 27 | * all installed copies of lz4 will be automatically updated to your new version 28 | 29 | For more information on Snaps see https://docs.snapcraft.io and https://forum.snapcraft.io/ 30 | -------------------------------------------------------------------------------- /contrib/meson/meson/examples/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # Copyright (c) 2022-present Tristan Partin 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | 11 | lz4_source_root = '../../../..' 12 | 13 | examples = { 14 | 'printVersion': 'printVersion.c', 15 | 'doubleBuffer': 'blockStreaming_doubleBuffer.c', 16 | 'dictionaryRandomAccess': 'dictionaryRandomAccess.c', 17 | 'ringBuffer': 'blockStreaming_ringBuffer.c', 18 | 'ringBufferHC': 'HCStreaming_ringBuffer.c', 19 | 'lineCompress': 'blockStreaming_lineByLine.c', 20 | 'frameCompress': 'frameCompress.c', 21 | 'compressFunctions': 'compress_functions.c', 22 | 'simpleBuffer': 'simple_buffer.c', 23 | } 24 | 25 | foreach e, src : examples 26 | executable( 27 | e, 28 | lz4_source_root / 'examples' / src, 29 | dependencies: [liblz4_internal_dep], 30 | install: false 31 | ) 32 | endforeach 33 | -------------------------------------------------------------------------------- /contrib/meson/meson/contrib/gen_manual/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # Copyright (c) 2022-present Tristan Partin 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | 11 | lz4_source_root = '../../../../..' 12 | 13 | add_languages('cpp') 14 | 15 | sources = files( 16 | lz4_source_root / 'contrib/gen_manual/gen_manual.cpp' 17 | ) 18 | 19 | gen_manual = executable( 20 | 'gen_manual', 21 | sources, 22 | native: true, 23 | install: false 24 | ) 25 | 26 | manual_pages = ['lz4', 'lz4frame'] 27 | 28 | foreach mp : manual_pages 29 | custom_target( 30 | '@0@_manual.html'.format(mp), 31 | build_by_default: true, 32 | input: lz4_source_root / 'lib/@0@.h'.format(mp), 33 | output: '@0@_manual.html'.format(mp), 34 | command: [ 35 | gen_manual, 36 | lz4_version, 37 | '@INPUT@', 38 | '@OUTPUT@', 39 | ], 40 | install: false 41 | ) 42 | endforeach 43 | -------------------------------------------------------------------------------- /contrib/meson/meson_options.txt: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # Copyright (c) 2022-present Tristan Partin 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | 11 | option('debug-level', type: 'integer', min: 0, max: 7, value: 1, 12 | description: 'Enable run-time debug. See lib/lz4hc.c') 13 | option('unstable', type: 'boolean', value: false, 14 | description: 'Expose unstable interfaces') 15 | option('programs', type: 'boolean', value: false, 16 | description: 'Enable programs build') 17 | option('tests', type: 'boolean', value: false, 18 | description: 'Enable tests build') 19 | option('contrib', type: 'boolean', value: false, 20 | description: 'Enable contrib build') 21 | option('examples', type: 'boolean', value: false, 22 | description: 'Enable examples build') 23 | option('memory-usage', type: 'integer', min: 0, value: 0, 24 | description: 'See LZ4_MEMORY_USAGE. 0 means use the LZ4 default') 25 | -------------------------------------------------------------------------------- /lib/dll/example/fullbench-dll.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio Express 2012 for Windows Desktop 3 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}" 4 | EndProject 5 | Global 6 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 7 | Debug|Win32 = Debug|Win32 8 | Debug|x64 = Debug|x64 9 | Release|Win32 = Release|Win32 10 | Release|x64 = Release|x64 11 | EndGlobalSection 12 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 13 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32 14 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32 15 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64 16 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64 17 | {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32 18 | {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32 19 | {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64 20 | {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64 21 | EndGlobalSection 22 | GlobalSection(SolutionProperties) = preSolution 23 | HideSolutionNode = FALSE 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /contrib/djgpp/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, lpsantil 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | 25 | -------------------------------------------------------------------------------- /lib/LICENSE: -------------------------------------------------------------------------------- 1 | LZ4 Library 2 | Copyright (c) 2011-2020, Yann Collet 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, 6 | are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright notice, this 12 | list of conditions and the following disclaimer in the documentation and/or 13 | other materials provided with the distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 19 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 22 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /tests/unicode_lint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # `unicode_lint.sh' determines whether source files under the ./lib/, ./tests/ and ./programs/ directories 4 | # contain Unicode characters, and fails if any do. 5 | # 6 | # See https://github.com/lz4/lz4/issues/1018 7 | 8 | echo "Ensure no unicode character is present in source files *.{c,h}" 9 | pass=true 10 | 11 | # Scan ./lib/ for Unicode in source (*.c, *.h) files 12 | echo "Scanning lib/" 13 | result=$( 14 | find ./lib/ -regex '.*\.\(c\|h\)$' -exec grep -P -n "[^\x00-\x7F]" {} \; -exec echo "{}: FAIL" \; 15 | ) 16 | if [[ $result ]]; then 17 | echo "$result" 18 | pass=false 19 | fi 20 | 21 | # Scan ./programs/ for Unicode in source (*.c, *.h) files 22 | echo "Scanning programs/" 23 | result=$( 24 | find ./programs/ -regex '.*\.\(c\|h\)$' -exec grep -P -n "[^\x00-\x7F]" {} \; -exec echo "{}: FAIL" \; 25 | ) 26 | if [[ $result ]]; then 27 | echo "$result" 28 | pass=false 29 | fi 30 | 31 | # Scan ./tests/ for Unicode in source (*.c, *.h) files 32 | echo "Scanning tests/" 33 | result=$( 34 | find ./tests/ -regex '.*\.\(c\|h\)$' -exec grep -P -n "[^\x00-\x7F]" {} \; -exec echo "{}: FAIL" \; 35 | ) 36 | if [[ $result ]]; then 37 | echo "$result" 38 | pass=false 39 | fi 40 | 41 | if [ "$pass" = true ]; then 42 | echo "All tests successful: no unicode character detected" 43 | echo "Result: PASS" 44 | exit 0 45 | else 46 | echo "Result: FAIL" 47 | exit 1 48 | fi 49 | -------------------------------------------------------------------------------- /ossfuzz/fuzz_data_producer.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "fuzz_helpers.h" 7 | #include "lz4frame.h" 8 | #include "lz4hc.h" 9 | 10 | /* Struct used for maintaining the state of the data */ 11 | typedef struct FUZZ_dataProducer_s FUZZ_dataProducer_t; 12 | 13 | /* Returns a data producer state struct. Use for producer initialization. */ 14 | FUZZ_dataProducer_t *FUZZ_dataProducer_create(const uint8_t *data, size_t size); 15 | 16 | /* Frees the data producer */ 17 | void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer); 18 | 19 | /* Returns 32 bits from the end of data */ 20 | uint32_t FUZZ_dataProducer_retrieve32(FUZZ_dataProducer_t *producer); 21 | 22 | /* Returns value between [min, max] */ 23 | uint32_t FUZZ_getRange_from_uint32(uint32_t seed, uint32_t min, uint32_t max); 24 | 25 | /* Combination of above two functions for non adaptive use cases. ie where size is not involved */ 26 | uint32_t FUZZ_dataProducer_range32(FUZZ_dataProducer_t *producer, uint32_t min, 27 | uint32_t max); 28 | 29 | /* Returns lz4 preferences */ 30 | LZ4F_preferences_t FUZZ_dataProducer_preferences(FUZZ_dataProducer_t* producer); 31 | 32 | /* Returns lz4 frame info */ 33 | LZ4F_frameInfo_t FUZZ_dataProducer_frameInfo(FUZZ_dataProducer_t* producer); 34 | 35 | /* Returns the size of the remaining bytes of data in the producer */ 36 | size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); 37 | -------------------------------------------------------------------------------- /contrib/meson/meson/GetLz4LibraryVersion.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # ############################################################################# 3 | # Copyright (c) 2018-present lzutao 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | import re 11 | 12 | 13 | def find_version_tuple(filepath): 14 | version_file_data = None 15 | with open(filepath) as fd: 16 | version_file_data = fd.read() 17 | 18 | patterns = r"""#\s*define\s+LZ4_VERSION_MAJOR\s+([0-9]+).*$ 19 | #\s*define\s+LZ4_VERSION_MINOR\s+([0-9]+).*$ 20 | #\s*define\s+LZ4_VERSION_RELEASE\s+([0-9]+).*$ 21 | """ 22 | regex = re.compile(patterns, re.MULTILINE) 23 | version_match = regex.search(version_file_data) 24 | if version_match: 25 | return version_match.groups() 26 | raise Exception("Unable to find version string.") 27 | 28 | 29 | def main(): 30 | import argparse 31 | parser = argparse.ArgumentParser(description='Print lz4 version from lib/lz4.h') 32 | parser.add_argument('file', help='path to lib/lz4.h') 33 | args = parser.parse_args() 34 | version_tuple = find_version_tuple(args.file) 35 | print('.'.join(version_tuple)) 36 | 37 | 38 | if __name__ == '__main__': 39 | main() 40 | -------------------------------------------------------------------------------- /ossfuzz/round_trip_hc_fuzzer.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This fuzz target performs a lz4 round-trip test (compress & decompress), 3 | * compares the result with the original, and calls abort() on corruption. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "fuzz_helpers.h" 12 | #include "fuzz_data_producer.h" 13 | #include "lz4.h" 14 | #include "lz4hc.h" 15 | 16 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) 17 | { 18 | FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size); 19 | int const level = FUZZ_dataProducer_range32(producer, 20 | LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX); 21 | size = FUZZ_dataProducer_remainingBytes(producer); 22 | 23 | size_t const dstCapacity = LZ4_compressBound(size); 24 | char* const dst = (char*)malloc(dstCapacity); 25 | char* const rt = (char*)malloc(size); 26 | 27 | FUZZ_ASSERT(dst); 28 | FUZZ_ASSERT(rt); 29 | 30 | /* Compression must succeed and round trip correctly. */ 31 | int const dstSize = LZ4_compress_HC((const char*)data, dst, size, 32 | dstCapacity, level); 33 | FUZZ_ASSERT(dstSize > 0); 34 | 35 | int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); 36 | FUZZ_ASSERT_MSG(rtSize == size, "Incorrect size"); 37 | FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); 38 | 39 | free(dst); 40 | free(rt); 41 | FUZZ_dataProducer_free(producer); 42 | 43 | return 0; 44 | } 45 | -------------------------------------------------------------------------------- /ossfuzz/round_trip_frame_fuzzer.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This fuzz target performs a lz4 round-trip test (compress & decompress), 3 | * compares the result with the original, and calls abort() on corruption. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "fuzz_helpers.h" 12 | #include "lz4.h" 13 | #include "lz4frame.h" 14 | #include "lz4_helpers.h" 15 | #include "fuzz_data_producer.h" 16 | 17 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) 18 | { 19 | FUZZ_dataProducer_t* producer = FUZZ_dataProducer_create(data, size); 20 | LZ4F_preferences_t const prefs = FUZZ_dataProducer_preferences(producer); 21 | size = FUZZ_dataProducer_remainingBytes(producer); 22 | 23 | size_t const dstCapacity = LZ4F_compressFrameBound(LZ4_compressBound(size), &prefs); 24 | char* const dst = (char*)malloc(dstCapacity); 25 | char* const rt = (char*)malloc(FUZZ_dataProducer_remainingBytes(producer)); 26 | 27 | FUZZ_ASSERT(dst); 28 | FUZZ_ASSERT(rt); 29 | 30 | /* Compression must succeed and round trip correctly. */ 31 | size_t const dstSize = 32 | LZ4F_compressFrame(dst, dstCapacity, data, size, &prefs); 33 | FUZZ_ASSERT(!LZ4F_isError(dstSize)); 34 | size_t const rtSize = FUZZ_decompressFrame(rt, size, dst, dstSize); 35 | FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size"); 36 | FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); 37 | 38 | free(dst); 39 | free(rt); 40 | FUZZ_dataProducer_free(producer); 41 | 42 | return 0; 43 | } 44 | -------------------------------------------------------------------------------- /contrib/meson/meson/programs/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # Copyright (c) 2022-present Tristan Partin 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | 11 | lz4_source_root = '../../../..' 12 | 13 | sources = files( 14 | lz4_source_root / 'programs/bench.c', 15 | lz4_source_root / 'programs/datagen.c', 16 | lz4_source_root / 'programs/lz4cli.c', 17 | lz4_source_root / 'programs/lz4io.c', 18 | ) 19 | 20 | lz4 = executable( 21 | 'lz4', 22 | sources, 23 | include_directories: include_directories(lz4_source_root / 'programs'), 24 | dependencies: [liblz4_internal_dep], 25 | export_dynamic: get_option('debug') and host_machine.system() == 'windows', 26 | install: true 27 | ) 28 | 29 | install_man(lz4_source_root / 'programs/lz4.1') 30 | 31 | if meson.version().version_compare('>=0.61.0') 32 | foreach alias : ['lz4c', 'lz4cat', 'unlz4'] 33 | install_symlink( 34 | alias, 35 | install_dir: get_option('bindir'), 36 | pointing_to: 'lz4' 37 | ) 38 | install_symlink( 39 | '@0@.1'.format(alias), 40 | install_dir: get_option('mandir') / 'man1', 41 | pointing_to: 'lz4.1' 42 | ) 43 | endforeach 44 | endif 45 | -------------------------------------------------------------------------------- /ossfuzz/compress_frame_fuzzer.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This fuzz target attempts to compress the fuzzed data with the simple 3 | * compression function with an output buffer that may be too small to 4 | * ensure that the compressor never crashes. 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "fuzz_helpers.h" 13 | #include "lz4.h" 14 | #include "lz4frame.h" 15 | #include "lz4_helpers.h" 16 | #include "fuzz_data_producer.h" 17 | 18 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) 19 | { 20 | FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size); 21 | LZ4F_preferences_t const prefs = FUZZ_dataProducer_preferences(producer); 22 | size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer); 23 | size = FUZZ_dataProducer_remainingBytes(producer); 24 | 25 | size_t const compressBound = LZ4F_compressFrameBound(size, &prefs); 26 | size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, compressBound); 27 | 28 | char* const dst = (char*)malloc(dstCapacity); 29 | char* const rt = (char*)malloc(size); 30 | 31 | FUZZ_ASSERT(dst!=NULL); 32 | FUZZ_ASSERT(rt!=NULL); 33 | 34 | /* If compression succeeds it must round trip correctly. */ 35 | size_t const dstSize = 36 | LZ4F_compressFrame(dst, dstCapacity, data, size, &prefs); 37 | if (!LZ4F_isError(dstSize)) { 38 | size_t const rtSize = FUZZ_decompressFrame(rt, size, dst, dstSize); 39 | FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size"); 40 | FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); 41 | } 42 | 43 | free(dst); 44 | free(rt); 45 | FUZZ_dataProducer_free(producer); 46 | 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /programs/datagen.h: -------------------------------------------------------------------------------- 1 | /* 2 | datagen.h - compressible data generator header 3 | Copyright (C) Yann Collet 2012-2020 4 | 5 | GPL v2 License 6 | 7 | This program is free software; you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation; either version 2 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License along 18 | with this program; if not, write to the Free Software Foundation, Inc., 19 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 20 | 21 | You can contact the author at : 22 | - LZ4 source repository : https://github.com/lz4/lz4 23 | - Public forum : https://groups.google.com/forum/#!forum/lz4c 24 | */ 25 | 26 | 27 | #include /* size_t */ 28 | 29 | void RDG_genOut(unsigned long long size, double matchProba, double litProba, unsigned seed); 30 | void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed); 31 | /* RDG_genOut 32 | Generate 'size' bytes of compressible data into stdout. 33 | Compressibility can be controlled using 'matchProba'. 34 | 'LitProba' is optional, and affect variability of bytes. If litProba==0.0, default value is used. 35 | Generated data can be selected using 'seed'. 36 | If (matchProba, litProba and seed) are equal, the function always generate the same content. 37 | 38 | RDG_genBuffer 39 | Same as RDG_genOut, but generate data into provided buffer 40 | */ 41 | -------------------------------------------------------------------------------- /ossfuzz/fuzz.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Fuzz target interface. 3 | * Fuzz targets have some common parameters passed as macros during compilation. 4 | * Check the documentation for each individual fuzzer for more parameters. 5 | * 6 | * @param FUZZ_RNG_SEED_SIZE: 7 | * The number of bytes of the source to look at when constructing a seed 8 | * for the deterministic RNG. These bytes are discarded before passing 9 | * the data to lz4 functions. Every fuzzer initializes the RNG exactly 10 | * once before doing anything else, even if it is unused. 11 | * Default: 4. 12 | * @param LZ4_DEBUG: 13 | * This is a parameter for the lz4 library. Defining `LZ4_DEBUG=1` 14 | * enables assert() statements in the lz4 library. Higher levels enable 15 | * logging, so aren't recommended. Defining `LZ4_DEBUG=1` is 16 | * recommended. 17 | * @param LZ4_FORCE_MEMORY_ACCESS: 18 | * This flag controls how the zstd library accesses unaligned memory. 19 | * It can be undefined, or 0 through 2. If it is undefined, it selects 20 | * the method to use based on the compiler. If testing with UBSAN set 21 | * MEM_FORCE_MEMORY_ACCESS=0 to use the standard compliant method. 22 | * @param FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION 23 | * This is the canonical flag to enable deterministic builds for fuzzing. 24 | * Changes to zstd for fuzzing are gated behind this define. 25 | * It is recommended to define this when building zstd for fuzzing. 26 | */ 27 | 28 | #ifndef FUZZ_H 29 | #define FUZZ_H 30 | 31 | #ifndef FUZZ_RNG_SEED_SIZE 32 | # define FUZZ_RNG_SEED_SIZE 4 33 | #endif 34 | 35 | #include 36 | #include 37 | 38 | #ifdef __cplusplus 39 | extern "C" { 40 | #endif 41 | 42 | int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size); 43 | 44 | #ifdef __cplusplus 45 | } 46 | #endif 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /ossfuzz/lz4_helpers.c: -------------------------------------------------------------------------------- 1 | #include "fuzz_helpers.h" 2 | #include "lz4_helpers.h" 3 | #include "lz4hc.h" 4 | 5 | LZ4F_frameInfo_t FUZZ_randomFrameInfo(uint32_t* seed) 6 | { 7 | LZ4F_frameInfo_t info = LZ4F_INIT_FRAMEINFO; 8 | info.blockSizeID = FUZZ_rand32(seed, LZ4F_max64KB - 1, LZ4F_max4MB); 9 | if (info.blockSizeID < LZ4F_max64KB) { 10 | info.blockSizeID = LZ4F_default; 11 | } 12 | info.blockMode = FUZZ_rand32(seed, LZ4F_blockLinked, LZ4F_blockIndependent); 13 | info.contentChecksumFlag = FUZZ_rand32(seed, LZ4F_noContentChecksum, 14 | LZ4F_contentChecksumEnabled); 15 | info.blockChecksumFlag = FUZZ_rand32(seed, LZ4F_noBlockChecksum, 16 | LZ4F_blockChecksumEnabled); 17 | return info; 18 | } 19 | 20 | LZ4F_preferences_t FUZZ_randomPreferences(uint32_t* seed) 21 | { 22 | LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES; 23 | prefs.frameInfo = FUZZ_randomFrameInfo(seed); 24 | prefs.compressionLevel = FUZZ_rand32(seed, 0, LZ4HC_CLEVEL_MAX + 3) - 3; 25 | prefs.autoFlush = FUZZ_rand32(seed, 0, 1); 26 | prefs.favorDecSpeed = FUZZ_rand32(seed, 0, 1); 27 | return prefs; 28 | } 29 | 30 | size_t FUZZ_decompressFrame(void* dst, const size_t dstCapacity, 31 | const void* src, const size_t srcSize) 32 | { 33 | LZ4F_decompressOptions_t opts; 34 | memset(&opts, 0, sizeof(opts)); 35 | opts.stableDst = 1; 36 | LZ4F_dctx* dctx; 37 | LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); 38 | FUZZ_ASSERT(dctx); 39 | 40 | size_t dstSize = dstCapacity; 41 | size_t srcConsumed = srcSize; 42 | size_t const rc = 43 | LZ4F_decompress(dctx, dst, &dstSize, src, &srcConsumed, &opts); 44 | FUZZ_ASSERT(!LZ4F_isError(rc)); 45 | FUZZ_ASSERT(rc == 0); 46 | FUZZ_ASSERT(srcConsumed == srcSize); 47 | 48 | LZ4F_freeDecompressionContext(dctx); 49 | 50 | return dstSize; 51 | } 52 | -------------------------------------------------------------------------------- /contrib/meson/meson/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # Copyright (c) 2022-present Tristan Partin 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | 11 | cc = meson.get_compiler('c') 12 | 13 | pkgconfig = import('pkgconfig') 14 | 15 | lz4_source_root = '../../..' 16 | 17 | lz4_version = meson.project_version() 18 | 19 | lz4_h_file = lz4_source_root / 'lib/lz4.h' 20 | GetLz4LibraryVersion_py = find_program('GetLz4LibraryVersion.py') 21 | lz4_version = run_command(GetLz4LibraryVersion_py, lz4_h_file, check: true).stdout().strip() 22 | message('Project version is now: @0@'.format(lz4_version)) 23 | 24 | add_project_arguments('-DXXH_NAMESPACE=LZ4_', language: 'c') 25 | 26 | if get_option('debug') 27 | add_project_arguments(cc.get_supported_arguments([ 28 | '-Wcast-qual', 29 | '-Wcast-align', 30 | '-Wshadow', 31 | '-Wswitch-enum', 32 | '-Wdeclaration-after-statement', 33 | '-Wstrict-prototypes', 34 | '-Wundef', 35 | '-Wpointer-arith', 36 | '-Wstrict-aliasing=1', 37 | '-DLZ4_DEBUG=@0@'.format(get_option('debug-level')), 38 | ] 39 | ), 40 | language: 'c', 41 | ) 42 | endif 43 | 44 | if get_option('memory-usage') > 0 45 | add_project_arguments( 46 | '-DLZ4_MEMORY_USAGE=@0@'.format(get_option('memory-usage')), 47 | language: 'c' 48 | ) 49 | endif 50 | 51 | subdir('lib') 52 | 53 | if get_option('programs') 54 | subdir('programs') 55 | endif 56 | 57 | if get_option('tests') 58 | subdir('tests') 59 | endif 60 | 61 | if get_option('contrib') 62 | subdir('contrib') 63 | endif 64 | 65 | if get_option('examples') 66 | subdir('examples') 67 | endif 68 | -------------------------------------------------------------------------------- /contrib/meson/meson/tests/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # Copyright (c) 2022-present Tristan Partin 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | 11 | lz4_source_root = '../../../..' 12 | 13 | exes = { 14 | 'fullbench': { 15 | 'sources': files(lz4_source_root / 'tests/fullbench.c'), 16 | 'include_directories': include_directories(lz4_source_root / 'programs'), 17 | }, 18 | 'fuzzer': { 19 | 'sources': files(lz4_source_root / 'tests/fuzzer.c'), 20 | 'include_directories': include_directories(lz4_source_root / 'programs'), 21 | }, 22 | 'frametest': { 23 | 'sources': files(lz4_source_root / 'tests/frametest.c'), 24 | 'include_directories': include_directories(lz4_source_root / 'programs'), 25 | }, 26 | 'roundTripTest': { 27 | 'sources': files(lz4_source_root / 'tests/roundTripTest.c'), 28 | }, 29 | 'datagen': { 30 | 'sources': files(lz4_source_root / 'tests/datagencli.c'), 31 | 'objects': lz4.extract_objects(lz4_source_root / 'programs/datagen.c'), 32 | 'include_directories': include_directories(lz4_source_root / 'programs'), 33 | }, 34 | 'checkFrame': { 35 | 'sources': files(lz4_source_root / 'tests/checkFrame.c'), 36 | 'include_directories': include_directories(lz4_source_root / 'programs'), 37 | }, 38 | 'checkTag': { 39 | 'sources': files(lz4_source_root / 'tests/checkTag.c'), 40 | }, 41 | } 42 | 43 | foreach e, attrs : exes 44 | executable( 45 | e, 46 | attrs.get('sources'), 47 | objects: attrs.get('objects', []), 48 | dependencies: [liblz4_internal_dep], 49 | include_directories: attrs.get('include_directories', []), 50 | install: false 51 | ) 52 | endforeach 53 | -------------------------------------------------------------------------------- /ossfuzz/standaloneengine.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "fuzz.h" 6 | 7 | /** 8 | * Main procedure for standalone fuzzing engine. 9 | * 10 | * Reads filenames from the argument array. For each filename, read the file 11 | * into memory and then call the fuzzing interface with the data. 12 | */ 13 | int main(int argc, char **argv) 14 | { 15 | int ii; 16 | for(ii = 1; ii < argc; ii++) 17 | { 18 | FILE *infile; 19 | printf("[%s] ", argv[ii]); 20 | 21 | /* Try and open the file. */ 22 | infile = fopen(argv[ii], "rb"); 23 | if(infile) 24 | { 25 | uint8_t *buffer = NULL; 26 | size_t buffer_len; 27 | 28 | printf("Opened.. "); 29 | 30 | /* Get the length of the file. */ 31 | fseek(infile, 0L, SEEK_END); 32 | buffer_len = ftell(infile); 33 | 34 | /* Reset the file indicator to the beginning of the file. */ 35 | fseek(infile, 0L, SEEK_SET); 36 | 37 | /* Allocate a buffer for the file contents. */ 38 | buffer = (uint8_t *)calloc(buffer_len, sizeof(uint8_t)); 39 | if(buffer) 40 | { 41 | /* Read all the text from the file into the buffer. */ 42 | fread(buffer, sizeof(uint8_t), buffer_len, infile); 43 | printf("Read %zu bytes, fuzzing.. ", buffer_len); 44 | 45 | /* Call the fuzzer with the data. */ 46 | LLVMFuzzerTestOneInput(buffer, buffer_len); 47 | 48 | printf("complete !!"); 49 | 50 | /* Free the buffer as it's no longer needed. */ 51 | free(buffer); 52 | buffer = NULL; 53 | } 54 | else 55 | { 56 | fprintf(stderr, 57 | "[%s] Failed to allocate %zu bytes \n", 58 | argv[ii], 59 | buffer_len); 60 | } 61 | 62 | /* Close the file as it's no longer needed. */ 63 | fclose(infile); 64 | infile = NULL; 65 | } 66 | else 67 | { 68 | /* Failed to open the file. Maybe wrong name or wrong permissions? */ 69 | fprintf(stderr, "[%s] Open failed. \n", argv[ii]); 70 | } 71 | 72 | printf("\n"); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /tests/test_custom_block_sizes.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | set -e 3 | 4 | LZ4=../lz4 5 | CHECKFRAME=./checkFrame 6 | DATAGEN=./datagen 7 | 8 | failures="" 9 | 10 | TMPFILE=/tmp/test_custom_block_sizes.$$ 11 | TMPFILE1=/tmp/test_custom_block_sizes1.$$ 12 | TMPFILE2=/tmp/test_custom_block_sizes2.$$ 13 | $DATAGEN -g12345678 > $TMPFILE1 14 | $DATAGEN -g12345678 > $TMPFILE2 15 | 16 | echo Testing -B31 17 | $LZ4 -f -B31 $TMPFILE1 && failures="31 (should fail) " 18 | 19 | for blocksize in 32 65535 65536 20 | do 21 | echo Testing -B$blocksize 22 | $LZ4 -f -B$blocksize $TMPFILE1 23 | $LZ4 -f -B$blocksize $TMPFILE2 24 | cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 25 | $CHECKFRAME -B$blocksize -b4 $TMPFILE.lz4 || failures="$failures $blocksize " 26 | done 27 | 28 | for blocksize in 65537 262143 262144 29 | do 30 | echo Testing -B$blocksize 31 | $LZ4 -f -B$blocksize $TMPFILE1 32 | $LZ4 -f -B$blocksize $TMPFILE2 33 | cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 34 | $CHECKFRAME -B$blocksize -b5 $TMPFILE.lz4 || failures="$failures $blocksize " 35 | done 36 | 37 | for blocksize in 262145 1048575 1048576 38 | do 39 | echo Testing -B$blocksize 40 | $LZ4 -f -B$blocksize $TMPFILE1 41 | $LZ4 -f -B$blocksize $TMPFILE2 42 | cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 43 | $CHECKFRAME -B$blocksize -b6 $TMPFILE.lz4 || failures="$failures $blocksize " 44 | done 45 | 46 | for blocksize in 1048577 4194303 4194304 47 | do 48 | echo Testing -B$blocksize 49 | $LZ4 -f -B$blocksize $TMPFILE1 50 | $LZ4 -f -B$blocksize $TMPFILE2 51 | cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 52 | $CHECKFRAME -B$blocksize -b7 $TMPFILE.lz4 || failures="$failures $blocksize " 53 | done 54 | 55 | for blocksize in 4194305 10485760 56 | do 57 | echo Testing -B$blocksize 58 | $LZ4 -f -B$blocksize $TMPFILE1 59 | $LZ4 -f -B$blocksize $TMPFILE2 60 | cat $TMPFILE1.lz4 $TMPFILE2.lz4 > $TMPFILE.lz4 61 | $CHECKFRAME -B4194304 -b7 $TMPFILE.lz4 || failures="$failures $blocksize " 62 | done 63 | 64 | rm $TMPFILE.lz4 $TMPFILE1 $TMPFILE1.lz4 $TMPFILE2 $TMPFILE2.lz4 65 | if [ "$failures" = "" ] 66 | then 67 | echo ---- All tests passed 68 | exit 0 69 | else 70 | echo ---- The following tests had failures: $failures 71 | exit 1 72 | fi 73 | -------------------------------------------------------------------------------- /lib/lz4frame_static.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 auto-framing library 3 | Header File for static linking only 4 | Copyright (C) 2011-2020, Yann Collet. 5 | 6 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | You can contact the author at : 32 | - LZ4 source repository : https://github.com/lz4/lz4 33 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 34 | */ 35 | 36 | #ifndef LZ4FRAME_STATIC_H_0398209384 37 | #define LZ4FRAME_STATIC_H_0398209384 38 | 39 | /* The declarations that formerly were made here have been merged into 40 | * lz4frame.h, protected by the LZ4F_STATIC_LINKING_ONLY macro. Going forward, 41 | * it is recommended to simply include that header directly. 42 | */ 43 | 44 | #define LZ4F_STATIC_LINKING_ONLY 45 | #include "lz4frame.h" 46 | 47 | #endif /* LZ4FRAME_STATIC_H_0398209384 */ 48 | -------------------------------------------------------------------------------- /ossfuzz/compress_fuzzer.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This fuzz target attempts to compress the fuzzed data with the simple 3 | * compression function with an output buffer that may be too small to 4 | * ensure that the compressor never crashes. 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "fuzz_helpers.h" 13 | #include "fuzz_data_producer.h" 14 | #include "lz4.h" 15 | 16 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) 17 | { 18 | FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size); 19 | size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer); 20 | size = FUZZ_dataProducer_remainingBytes(producer); 21 | 22 | size_t const compressBound = LZ4_compressBound(size); 23 | size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, compressBound); 24 | 25 | char* const dst = (char*)malloc(dstCapacity); 26 | char* const rt = (char*)malloc(size); 27 | 28 | FUZZ_ASSERT(dst); 29 | FUZZ_ASSERT(rt); 30 | 31 | /* If compression succeeds it must round trip correctly. */ 32 | { 33 | int const dstSize = LZ4_compress_default((const char*)data, dst, 34 | size, dstCapacity); 35 | if (dstSize > 0) { 36 | int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); 37 | FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size"); 38 | FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); 39 | } 40 | } 41 | 42 | if (dstCapacity > 0) { 43 | /* Compression succeeds and must round trip correctly. */ 44 | int compressedSize = size; 45 | int const dstSize = LZ4_compress_destSize((const char*)data, dst, 46 | &compressedSize, dstCapacity); 47 | FUZZ_ASSERT(dstSize > 0); 48 | int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); 49 | FUZZ_ASSERT_MSG(rtSize == compressedSize, "Incorrect regenerated size"); 50 | FUZZ_ASSERT_MSG(!memcmp(data, rt, compressedSize), "Corruption!"); 51 | } 52 | 53 | free(dst); 54 | free(rt); 55 | FUZZ_dataProducer_free(producer); 56 | 57 | return 0; 58 | } 59 | -------------------------------------------------------------------------------- /lib/dll/example/Makefile: -------------------------------------------------------------------------------- 1 | # ########################################################################## 2 | # LZ4 programs - Makefile 3 | # Copyright (C) Yann Collet 2016-2020 4 | # 5 | # GPL v2 License 6 | # 7 | # This program is free software; you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation; either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License along 18 | # with this program; if not, write to the Free Software Foundation, Inc., 19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 20 | # 21 | # You can contact the author at : 22 | # - LZ4 homepage : http://www.lz4.org 23 | # - LZ4 source repository : https://github.com/lz4/lz4 24 | # ########################################################################## 25 | 26 | VOID := /dev/null 27 | LZ4DIR := ../include 28 | LIBDIR := ../static 29 | DLLDIR := ../dll 30 | 31 | CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make 32 | CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ 33 | -Wdeclaration-after-statement -Wstrict-prototypes \ 34 | -Wpointer-arith -Wstrict-aliasing=1 35 | CFLAGS += $(MOREFLAGS) 36 | CPPFLAGS:= -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ 37 | FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) 38 | 39 | 40 | # Define *.exe as extension for Windows systems 41 | ifneq (,$(filter Windows%,$(OS))) 42 | EXT =.exe 43 | else 44 | EXT = 45 | endif 46 | 47 | .PHONY: default fullbench-dll fullbench-lib 48 | 49 | 50 | default: all 51 | 52 | all: fullbench-dll fullbench-lib 53 | 54 | 55 | fullbench-lib: fullbench.c xxhash.c 56 | $(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/liblz4_static.lib 57 | 58 | fullbench-dll: fullbench.c xxhash.c 59 | $(CC) $(FLAGS) $^ -o $@$(EXT) -DLZ4_DLL_IMPORT=1 $(DLLDIR)/liblz4.dll 60 | 61 | clean: 62 | @$(RM) fullbench-dll$(EXT) fullbench-lib$(EXT) \ 63 | @echo Cleaning completed 64 | -------------------------------------------------------------------------------- /tests/decompress-partial.c: -------------------------------------------------------------------------------- 1 | #include "stdio.h" 2 | #include "string.h" 3 | #include "lz4.h" 4 | 5 | const char source[] = 6 | "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod\n" 7 | "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim\n" 8 | "veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea\n" 9 | "commodo consequat. Duis aute irure dolor in reprehenderit in voluptate\n" 10 | "velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat\n" 11 | "cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id\n" 12 | "est laborum.\n" 13 | "\n" 14 | "Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium\n" 15 | "doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore\n" 16 | "veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim\n" 17 | "ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia\n" 18 | "consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque\n" 19 | "porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur,\n" 20 | "adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore\n" 21 | "et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis\n" 22 | "nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid\n" 23 | "ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea\n" 24 | "voluptate velit esse quam nihil molestiae consequatur, vel illum qui\n" 25 | "dolorem eum fugiat quo voluptas nulla pariatur?\n"; 26 | 27 | #define BUFFER_SIZE 2048 28 | 29 | int main(void) 30 | { 31 | int srcLen = (int)strlen(source); 32 | char cmpBuffer[BUFFER_SIZE]; 33 | char outBuffer[BUFFER_SIZE]; 34 | int cmpSize; 35 | int i; 36 | 37 | cmpSize = LZ4_compress_default(source, cmpBuffer, srcLen, BUFFER_SIZE); 38 | 39 | for (i = cmpSize; i < cmpSize + 10; ++i) { 40 | int result = LZ4_decompress_safe_partial(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE); 41 | if ((result < 0) || (result != srcLen) || memcmp(source, outBuffer, srcLen)) { 42 | printf("test decompress-partial error \n"); 43 | return -1; 44 | } 45 | } 46 | 47 | printf("test decompress-partial OK \n"); 48 | return 0; 49 | } 50 | -------------------------------------------------------------------------------- /examples/dictionaryRandomAccess.md: -------------------------------------------------------------------------------- 1 | # LZ4 API Example : Dictionary Random Access 2 | 3 | `dictionaryRandomAccess.c` is LZ4 API example which implements dictionary compression and random access decompression. 4 | 5 | Please note that the output file is not compatible with lz4frame and is platform dependent. 6 | 7 | 8 | ## What's the point of this example ? 9 | 10 | - Dictionary based compression for homogeneous files. 11 | - Random access to compressed blocks. 12 | 13 | 14 | ## How the compression works 15 | 16 | Reads the dictionary from a file, and uses it as the history for each block. 17 | This allows each block to be independent, but maintains compression ratio. 18 | 19 | ``` 20 | Dictionary 21 | + 22 | | 23 | v 24 | +---------+ 25 | | Block#1 | 26 | +----+----+ 27 | | 28 | v 29 | {Out#1} 30 | 31 | 32 | Dictionary 33 | + 34 | | 35 | v 36 | +---------+ 37 | | Block#2 | 38 | +----+----+ 39 | | 40 | v 41 | {Out#2} 42 | ``` 43 | 44 | After writing the magic bytes `TEST` and then the compressed blocks, write out the jump table. 45 | The last 4 bytes is an integer containing the number of blocks in the stream. 46 | If there are `N` blocks, then just before the last 4 bytes is `N + 1` 4 byte integers containing the offsets at the beginning and end of each block. 47 | Let `Offset#K` be the total number of bytes written after writing out `Block#K` *including* the magic bytes for simplicity. 48 | 49 | ``` 50 | +------+---------+ +---------+---+----------+ +----------+-----+ 51 | | TEST | Block#1 | ... | Block#N | 4 | Offset#1 | ... | Offset#N | N+1 | 52 | +------+---------+ +---------+---+----------+ +----------+-----+ 53 | ``` 54 | 55 | ## How the decompression works 56 | 57 | Decompression will do reverse order. 58 | 59 | - Seek to the last 4 bytes of the file and read the number of offsets. 60 | - Read each offset into an array. 61 | - Seek to the first block containing data we want to read. 62 | We know where to look because we know each block contains a fixed amount of uncompressed data, except possibly the last. 63 | - Decompress it and write what data we need from it to the file. 64 | - Read the next block. 65 | - Decompress it and write that page to the file. 66 | 67 | Continue these procedures until all the required data has been read. 68 | -------------------------------------------------------------------------------- /.github/workflows/README.md: -------------------------------------------------------------------------------- 1 | This directory contains [GitHub Actions](https://github.com/features/actions) workflow files. 2 | 3 | # Known issues 4 | 5 | ## USAN, ASAN (`lz4-ubsan-x64`, `lz4-ubsan-x86`, `lz4-asan-x64`) 6 | 7 | For now, `lz4-ubsan-*` ignores the exit code of `make usan` and `make usan32`. 8 | Because there are several issues which may take relatively long time to resolve. 9 | 10 | We'll fully enable it when we ensure `make usan` is ready for all commits and PRs. 11 | 12 | See [#983](https://github.com/lz4/lz4/pull/983) for details. 13 | 14 | 15 | ## C Compilers (`lz4-c-compilers`) 16 | 17 | - Our test doesn't use `gcc-4.5` due to installation issue of its package. (`apt-get install gcc-4.5` fails on GH-Actions VM) 18 | 19 | - Currently, the following 32bit executable tests fail with all versions of `clang`. 20 | - `CC=clang-X CFLAGS='-O3' make V=1 -C tests clean test-lz4c32` 21 | - `CC=clang-X CFLAGS='-O3 -mx32' make V=1 -C tests clean test-lz4c32` 22 | - See [#991](https://github.com/lz4/lz4/issues/991) for details. 23 | 24 | - Currently, the following 32bit executable tests fail with `gcc-11` 25 | - `CC=gcc-11 CFLAGS='-O3' make V=1 -C tests clean test-lz4c32` 26 | - `CC=gcc-11 CFLAGS='-O3 -mx32' make V=1 -C tests clean test-lz4c32` 27 | - See [#991](https://github.com/lz4/lz4/issues/991) for details. 28 | 29 | 30 | ## cppcheck (`lz4-cppcheck`) 31 | 32 | This test script ignores the exit code of `make cppcheck`. 33 | Because this project doesn't 100% follow their recommendation. 34 | Also sometimes it reports false positives. 35 | 36 | 37 | 38 | # Notes 39 | 40 | - You can investigate various information at the right pane of GitHub 41 | Actions report page. 42 | 43 | | Item | Section in the right pane | 44 | | ------------------------- | ------------------------------------- | 45 | | OS, VM | Set up job | 46 | | git repo, commit hash | Run actions/checkout@v2 | 47 | | Version of tools | Environment info | 48 | 49 | 50 | 51 | # Difference with `.travis.yml` 52 | 53 | The following tests are not included yet. 54 | 55 | - name: Compile OSS-Fuzz targets 56 | 57 | The following tests will not be included due to limitation of GH-Actions. 58 | 59 | - name: aarch64 real-hw tests 60 | - name: PPC64LE real-hw tests 61 | - name: IBM s390x real-hw tests 62 | -------------------------------------------------------------------------------- /contrib/meson/meson/lib/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # Copyright (c) 2022-present Tristan Partin 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | 11 | lz4_source_root = '../../../..' 12 | 13 | sources = files( 14 | lz4_source_root / 'lib/lz4.c', 15 | lz4_source_root / 'lib/lz4frame.c', 16 | lz4_source_root / 'lib/lz4hc.c', 17 | lz4_source_root / 'lib/xxhash.c' 18 | ) 19 | 20 | c_args = [] 21 | 22 | if host_machine.system() == 'windows' and get_option('default_library') != 'static' 23 | c_args += '-DLZ4_DLL_EXPORT=1' 24 | endif 25 | 26 | if get_option('unstable') 27 | compile_args += '-DLZ4_STATIC_LINKING_ONLY' 28 | if get_option('default_library') != 'static' 29 | c_args += '-DLZ4_PUBLISH_STATIC_FUNCTIONS' 30 | endif 31 | endif 32 | 33 | liblz4 = library( 34 | 'lz4', 35 | sources, 36 | install: true, 37 | version: lz4_version, 38 | gnu_symbol_visibility: 'hidden' 39 | ) 40 | 41 | liblz4_dep = declare_dependency( 42 | link_with: liblz4, 43 | include_directories: include_directories(lz4_source_root / 'lib') 44 | ) 45 | 46 | if get_option('tests') or get_option('programs') or get_option('examples') 47 | liblz4_internal = static_library( 48 | 'lz4-internal', 49 | objects: liblz4.extract_all_objects(recursive: true), 50 | gnu_symbol_visibility: 'hidden' 51 | ) 52 | 53 | liblz4_internal_dep = declare_dependency( 54 | link_with: liblz4_internal, 55 | include_directories: include_directories(lz4_source_root / 'lib') 56 | ) 57 | endif 58 | 59 | pkgconfig.generate( 60 | liblz4, 61 | name: 'lz4', 62 | filebase: 'liblz4', 63 | description: 'extremely fast lossless compression algorithm library', 64 | version: lz4_version, 65 | url: 'http://www.lz4.org/' 66 | ) 67 | 68 | install_headers( 69 | lz4_source_root / 'lib/lz4.h', 70 | lz4_source_root / 'lib/lz4hc.h', 71 | lz4_source_root / 'lib/lz4frame.h' 72 | ) 73 | 74 | if get_option('default_library') != 'shared' 75 | install_headers(lz4_source_root / 'lib/lz4frame_static.h') 76 | endif 77 | -------------------------------------------------------------------------------- /ossfuzz/compress_hc_fuzzer.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This fuzz target attempts to compress the fuzzed data with the simple 3 | * compression function with an output buffer that may be too small to 4 | * ensure that the compressor never crashes. 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "fuzz_helpers.h" 13 | #include "fuzz_data_producer.h" 14 | #include "lz4.h" 15 | #include "lz4hc.h" 16 | 17 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) 18 | { 19 | FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size); 20 | size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer); 21 | size_t const levelSeed = FUZZ_dataProducer_retrieve32(producer); 22 | size = FUZZ_dataProducer_remainingBytes(producer); 23 | 24 | size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, size); 25 | int const level = FUZZ_getRange_from_uint32(levelSeed, LZ4HC_CLEVEL_MIN, LZ4HC_CLEVEL_MAX); 26 | 27 | char* const dst = (char*)malloc(dstCapacity); 28 | char* const rt = (char*)malloc(size); 29 | 30 | FUZZ_ASSERT(dst); 31 | FUZZ_ASSERT(rt); 32 | 33 | /* If compression succeeds it must round trip correctly. */ 34 | { 35 | int const dstSize = LZ4_compress_HC((const char*)data, dst, size, 36 | dstCapacity, level); 37 | if (dstSize > 0) { 38 | int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); 39 | FUZZ_ASSERT_MSG(rtSize == size, "Incorrect regenerated size"); 40 | FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); 41 | } 42 | } 43 | 44 | if (dstCapacity > 0) { 45 | /* Compression succeeds and must round trip correctly. */ 46 | void* state = malloc(LZ4_sizeofStateHC()); 47 | FUZZ_ASSERT(state); 48 | int compressedSize = size; 49 | int const dstSize = LZ4_compress_HC_destSize(state, (const char*)data, 50 | dst, &compressedSize, 51 | dstCapacity, level); 52 | FUZZ_ASSERT(dstSize > 0); 53 | int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); 54 | FUZZ_ASSERT_MSG(rtSize == compressedSize, "Incorrect regenerated size"); 55 | FUZZ_ASSERT_MSG(!memcmp(data, rt, compressedSize), "Corruption!"); 56 | free(state); 57 | } 58 | 59 | free(dst); 60 | free(rt); 61 | FUZZ_dataProducer_free(producer); 62 | 63 | return 0; 64 | } 65 | -------------------------------------------------------------------------------- /ossfuzz/decompress_frame_fuzzer.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This fuzz target attempts to decompress the fuzzed data with the simple 3 | * decompression function to ensure the decompressor never crashes. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "fuzz_helpers.h" 12 | #include "fuzz_data_producer.h" 13 | #include "lz4.h" 14 | #define LZ4F_STATIC_LINKING_ONLY 15 | #include "lz4frame.h" 16 | #include "lz4_helpers.h" 17 | 18 | static void decompress(LZ4F_dctx* dctx, void* dst, size_t dstCapacity, 19 | const void* src, size_t srcSize, 20 | const void* dict, size_t dictSize, 21 | const LZ4F_decompressOptions_t* opts) 22 | { 23 | LZ4F_resetDecompressionContext(dctx); 24 | if (dictSize == 0) 25 | LZ4F_decompress(dctx, dst, &dstCapacity, src, &srcSize, opts); 26 | else 27 | LZ4F_decompress_usingDict(dctx, dst, &dstCapacity, src, &srcSize, 28 | dict, dictSize, opts); 29 | } 30 | 31 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) 32 | { 33 | FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size); 34 | size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer); 35 | size_t const dictSizeSeed = FUZZ_dataProducer_retrieve32(producer); 36 | size = FUZZ_dataProducer_remainingBytes(producer); 37 | 38 | size_t const dstCapacity = FUZZ_getRange_from_uint32( 39 | dstCapacitySeed, 0, 4 * size); 40 | size_t const largeDictSize = 64 * 1024; 41 | size_t const dictSize = FUZZ_getRange_from_uint32( 42 | dictSizeSeed, 0, largeDictSize); 43 | 44 | char* const dst = (char*)malloc(dstCapacity); 45 | char* const dict = (char*)malloc(dictSize); 46 | LZ4F_decompressOptions_t opts; 47 | LZ4F_dctx* dctx; 48 | LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); 49 | 50 | FUZZ_ASSERT(dctx); 51 | FUZZ_ASSERT(dst); 52 | FUZZ_ASSERT(dict); 53 | 54 | /* Prepare the dictionary. The data doesn't matter for decompression. */ 55 | memset(dict, 0, dictSize); 56 | 57 | 58 | /* Decompress using multiple configurations. */ 59 | memset(&opts, 0, sizeof(opts)); 60 | opts.stableDst = 0; 61 | decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts); 62 | opts.stableDst = 1; 63 | decompress(dctx, dst, dstCapacity, data, size, NULL, 0, &opts); 64 | opts.stableDst = 0; 65 | decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts); 66 | opts.stableDst = 1; 67 | decompress(dctx, dst, dstCapacity, data, size, dict, dictSize, &opts); 68 | 69 | LZ4F_freeDecompressionContext(dctx); 70 | free(dst); 71 | free(dict); 72 | FUZZ_dataProducer_free(producer); 73 | 74 | return 0; 75 | } 76 | -------------------------------------------------------------------------------- /ossfuzz/fuzz_data_producer.c: -------------------------------------------------------------------------------- 1 | #include "fuzz_data_producer.h" 2 | 3 | struct FUZZ_dataProducer_s{ 4 | const uint8_t *data; 5 | size_t size; 6 | }; 7 | 8 | FUZZ_dataProducer_t* FUZZ_dataProducer_create(const uint8_t* data, size_t size) { 9 | FUZZ_dataProducer_t* const producer = malloc(sizeof(FUZZ_dataProducer_t)); 10 | 11 | FUZZ_ASSERT(producer != NULL); 12 | 13 | producer->data = data; 14 | producer->size = size; 15 | return producer; 16 | } 17 | 18 | void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer) { free(producer); } 19 | 20 | uint32_t FUZZ_dataProducer_retrieve32(FUZZ_dataProducer_t *producer) { 21 | const uint8_t* data = producer->data; 22 | const size_t size = producer->size; 23 | if (size == 0) { 24 | return 0; 25 | } else if (size < 4) { 26 | producer->size -= 1; 27 | return (uint32_t)data[size - 1]; 28 | } else { 29 | producer->size -= 4; 30 | return *(data + size - 4); 31 | } 32 | } 33 | 34 | uint32_t FUZZ_getRange_from_uint32(uint32_t seed, uint32_t min, uint32_t max) 35 | { 36 | uint32_t range = max - min; 37 | if (range == 0xffffffff) { 38 | return seed; 39 | } 40 | return min + seed % (range + 1); 41 | } 42 | 43 | uint32_t FUZZ_dataProducer_range32(FUZZ_dataProducer_t* producer, 44 | uint32_t min, uint32_t max) 45 | { 46 | size_t const seed = FUZZ_dataProducer_retrieve32(producer); 47 | return FUZZ_getRange_from_uint32(seed, min, max); 48 | } 49 | 50 | LZ4F_frameInfo_t FUZZ_dataProducer_frameInfo(FUZZ_dataProducer_t* producer) 51 | { 52 | LZ4F_frameInfo_t info = LZ4F_INIT_FRAMEINFO; 53 | info.blockSizeID = FUZZ_dataProducer_range32(producer, LZ4F_max64KB - 1, LZ4F_max4MB); 54 | if (info.blockSizeID < LZ4F_max64KB) { 55 | info.blockSizeID = LZ4F_default; 56 | } 57 | info.blockMode = FUZZ_dataProducer_range32(producer, LZ4F_blockLinked, LZ4F_blockIndependent); 58 | info.contentChecksumFlag = FUZZ_dataProducer_range32(producer, LZ4F_noContentChecksum, 59 | LZ4F_contentChecksumEnabled); 60 | info.blockChecksumFlag = FUZZ_dataProducer_range32(producer, LZ4F_noBlockChecksum, 61 | LZ4F_blockChecksumEnabled); 62 | return info; 63 | } 64 | 65 | LZ4F_preferences_t FUZZ_dataProducer_preferences(FUZZ_dataProducer_t* producer) 66 | { 67 | LZ4F_preferences_t prefs = LZ4F_INIT_PREFERENCES; 68 | prefs.frameInfo = FUZZ_dataProducer_frameInfo(producer); 69 | prefs.compressionLevel = FUZZ_dataProducer_range32(producer, 0, LZ4HC_CLEVEL_MAX + 3) - 3; 70 | prefs.autoFlush = FUZZ_dataProducer_range32(producer, 0, 1); 71 | prefs.favorDecSpeed = FUZZ_dataProducer_range32(producer, 0, 1); 72 | return prefs; 73 | } 74 | 75 | size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){ 76 | return producer->size; 77 | } 78 | -------------------------------------------------------------------------------- /tests/checkTag.c: -------------------------------------------------------------------------------- 1 | /* 2 | checkTag.c - Version validation tool for LZ4 3 | Copyright (C) Yann Collet 2018-2020 4 | 5 | GPL v2 License 6 | 7 | This program is free software; you can redistribute it and/or modify 8 | it under the terms of the GNU General Public License as published by 9 | the Free Software Foundation; either version 2 of the License, or 10 | (at your option) any later version. 11 | 12 | This program is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU General Public License for more details. 16 | 17 | You should have received a copy of the GNU General Public License along 18 | with this program; if not, write to the Free Software Foundation, Inc., 19 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 20 | 21 | You can contact the author at : 22 | - LZ4 homepage : http://www.lz4.org 23 | - LZ4 source repo : https://github.com/lz4/lz4 24 | */ 25 | 26 | /* checkTag command : 27 | * $ ./checkTag tag 28 | * checkTag validates tags of following format : v[0-9].[0-9].[0-9]{any} 29 | * The tag is then compared to LZ4 version number. 30 | * They are compatible if first 3 digits are identical. 31 | * Anything beyond that is free, and doesn't impact validation. 32 | * Example : tag v1.8.1.2 is compatible with version 1.8.1 33 | * When tag and version are not compatible, program exits with error code 1. 34 | * When they are compatible, it exists with a code 0. 35 | * checkTag is intended to be used in automated testing environment. 36 | */ 37 | 38 | #include /* printf */ 39 | #include /* strlen, strncmp */ 40 | #include "lz4.h" /* LZ4_VERSION_STRING */ 41 | 42 | 43 | /* validate() : 44 | * @return 1 if tag is compatible, 0 if not. 45 | */ 46 | static int validate(const char* const tag) 47 | { 48 | size_t const tagLength = strlen(tag); 49 | size_t const verLength = strlen(LZ4_VERSION_STRING); 50 | 51 | if (tagLength < 2) return 0; 52 | if (tag[0] != 'v') return 0; 53 | if (tagLength <= verLength) return 0; 54 | 55 | if (strncmp(LZ4_VERSION_STRING, tag+1, verLength)) return 0; 56 | 57 | return 1; 58 | } 59 | 60 | int main(int argc, const char** argv) 61 | { 62 | const char* const exeName = argv[0]; 63 | const char* const tag = argv[1]; 64 | if (argc!=2) { 65 | printf("incorrect usage : %s tag \n", exeName); 66 | return 2; 67 | } 68 | 69 | printf("Version : %s \n", LZ4_VERSION_STRING); 70 | printf("Tag : %s \n", tag); 71 | 72 | if (validate(tag)) { 73 | printf("OK : tag is compatible with lz4 version \n"); 74 | return 0; 75 | } 76 | 77 | printf("!! error : tag and versions are not compatible !! \n"); 78 | return 1; 79 | } 80 | -------------------------------------------------------------------------------- /ossfuzz/Makefile: -------------------------------------------------------------------------------- 1 | # ########################################################################## 2 | # LZ4 oss fuzzer - Makefile 3 | # 4 | # GPL v2 License 5 | # 6 | # This program is free software; you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation; either version 2 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19 | # 20 | # You can contact the author at : 21 | # - LZ4 homepage : http://www.lz4.org 22 | # - LZ4 source repository : https://github.com/lz4/lz4 23 | # ########################################################################## 24 | # compress_fuzzer : OSS Fuzz test tool 25 | # decompress_fuzzer : OSS Fuzz test tool 26 | # ########################################################################## 27 | 28 | LZ4DIR := ../lib 29 | LIB_FUZZING_ENGINE ?= 30 | 31 | DEBUGLEVEL?= 1 32 | DEBUGFLAGS = -g -DLZ4_DEBUG=$(DEBUGLEVEL) 33 | 34 | LZ4_CFLAGS = $(CFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) 35 | LZ4_CXXFLAGS = $(CXXFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) 36 | LZ4_CPPFLAGS = $(CPPFLAGS) -I$(LZ4DIR) -DXXH_NAMESPACE=LZ4_ \ 37 | -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION 38 | 39 | FUZZERS := \ 40 | compress_fuzzer \ 41 | decompress_fuzzer \ 42 | round_trip_fuzzer \ 43 | round_trip_stream_fuzzer \ 44 | compress_hc_fuzzer \ 45 | round_trip_hc_fuzzer \ 46 | compress_frame_fuzzer \ 47 | round_trip_frame_fuzzer \ 48 | round_trip_frame_uncompressed_fuzzer \ 49 | decompress_frame_fuzzer 50 | 51 | .PHONY: all 52 | all: $(FUZZERS) 53 | 54 | # Include a rule to build the static library if calling this target 55 | # directly. 56 | $(LZ4DIR)/liblz4.a: 57 | $(MAKE) -C $(LZ4DIR) CFLAGS="$(LZ4_CFLAGS)" liblz4.a 58 | 59 | %.o: %.c 60 | $(CC) -c $(LZ4_CFLAGS) $(LZ4_CPPFLAGS) $< -o $@ 61 | 62 | # Generic rule for generating fuzzers 63 | ifeq ($(LIB_FUZZING_ENGINE),) 64 | LIB_FUZZING_DEPS := standaloneengine.o 65 | else 66 | LIB_FUZZING_DEPS := 67 | endif 68 | %_fuzzer: %_fuzzer.o lz4_helpers.o fuzz_data_producer.o $(LZ4DIR)/liblz4.a $(LIB_FUZZING_DEPS) 69 | $(CXX) $(LZ4_CXXFLAGS) $(LZ4_CPPFLAGS) $(LDFLAGS) $(LIB_FUZZING_ENGINE) $^ -o $@$(EXT) 70 | 71 | %_fuzzer_clean: 72 | $(RM) $*_fuzzer $*_fuzzer.o standaloneengine.o 73 | 74 | .PHONY: clean 75 | clean: compress_fuzzer_clean decompress_fuzzer_clean \ 76 | compress_frame_fuzzer_clean compress_hc_fuzzer_clean \ 77 | decompress_frame_fuzzer_clean round_trip_frame_fuzzer_clean \ 78 | round_trip_fuzzer_clean round_trip_hc_fuzzer_clean round_trip_stream_fuzzer_clean 79 | $(MAKE) -C $(LZ4DIR) clean 80 | -------------------------------------------------------------------------------- /programs/bench.h: -------------------------------------------------------------------------------- 1 | /* 2 | bench.h - Demo program to benchmark open-source compression algorithm 3 | Copyright (C) Yann Collet 2012-2020 4 | 5 | This program is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation; either version 2 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License along 16 | with this program; if not, write to the Free Software Foundation, Inc., 17 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 18 | 19 | You can contact the author at : 20 | - LZ4 source repository : https://github.com/lz4/lz4 21 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 22 | */ 23 | #ifndef BENCH_H_125623623633 24 | #define BENCH_H_125623623633 25 | 26 | #include 27 | 28 | /* BMK_benchFiles() : 29 | * Benchmark all files provided through array @fileNamesTable. 30 | * All files must be valid, otherwise benchmark fails. 31 | * Roundtrip measurements are done for each file individually, but 32 | * unless BMK_setBenchSeparately() is set, all results are agglomerated. 33 | * The method benchmarks all compression levels from @cLevelStart to @cLevelLast, 34 | * both inclusive, providing one result per compression level. 35 | * If @cLevelLast <= @cLevelStart, BMK_benchFiles() benchmarks @cLevelStart only. 36 | * @dictFileName is optional, it's possible to provide NULL. 37 | * When provided, compression and decompression use the specified file as dictionary. 38 | * Only one dictionary can be provided, in which case it's applied to all benchmarked files. 39 | **/ 40 | int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, 41 | int cLevelStart, int cLevelLast, 42 | const char* dictFileName); 43 | 44 | /* Set Parameters */ 45 | void BMK_setNbSeconds(unsigned nbSeconds); /* minimum benchmark duration, in seconds, for both compression and decompression */ 46 | void BMK_setBlockSize(size_t blockSize); /* Internally cut input file(s) into independent blocks of specified size */ 47 | void BMK_setNotificationLevel(unsigned level); /* Influence verbosity level */ 48 | void BMK_setBenchSeparately(int separate); /* When providing multiple files, output one result per file */ 49 | void BMK_setDecodeOnlyMode(int set); /* v1.9.4+: set benchmark mode to decode only */ 50 | void BMK_skipChecksums(int skip); /* v1.9.4+: only useful for DecodeOnlyMode; do not calculate checksum when present, to save CPU time */ 51 | 52 | void BMK_setAdditionalParam(int additionalParam); /* hidden param, influence output format, for python parsing */ 53 | 54 | #endif /* BENCH_H_125623623633 */ 55 | -------------------------------------------------------------------------------- /ossfuzz/fuzz_helpers.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-present, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree), 8 | * meaning you may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /** 12 | * Helper functions for fuzzing. 13 | */ 14 | 15 | #ifndef FUZZ_HELPERS_H 16 | #define FUZZ_HELPERS_H 17 | 18 | #include "fuzz.h" 19 | #include "xxhash.h" 20 | #include 21 | #include 22 | #include 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | #define LZ4_COMMONDEFS_ONLY 29 | #ifndef LZ4_SRC_INCLUDED 30 | #include "lz4.c" /* LZ4_count, constants, mem */ 31 | #endif 32 | 33 | #define MIN(a,b) ( (a) < (b) ? (a) : (b) ) 34 | #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) 35 | 36 | #define FUZZ_QUOTE_IMPL(str) #str 37 | #define FUZZ_QUOTE(str) FUZZ_QUOTE_IMPL(str) 38 | 39 | /** 40 | * Asserts for fuzzing that are always enabled. 41 | */ 42 | #define FUZZ_ASSERT_MSG(cond, msg) \ 43 | ((cond) ? (void)0 \ 44 | : (fprintf(stderr, "%s: %u: Assertion: `%s' failed. %s\n", __FILE__, \ 45 | __LINE__, FUZZ_QUOTE(cond), (msg)), \ 46 | abort())) 47 | #define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), ""); 48 | 49 | #if defined(__GNUC__) 50 | #define FUZZ_STATIC static __inline __attribute__((unused)) 51 | #elif defined(__cplusplus) || \ 52 | (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) 53 | #define FUZZ_STATIC static inline 54 | #elif defined(_MSC_VER) 55 | #define FUZZ_STATIC static __inline 56 | #else 57 | #define FUZZ_STATIC static 58 | #endif 59 | 60 | /** 61 | * Deterministically constructs a seed based on the fuzz input. 62 | * Consumes up to the first FUZZ_RNG_SEED_SIZE bytes of the input. 63 | */ 64 | FUZZ_STATIC uint32_t FUZZ_seed(uint8_t const **src, size_t* size) { 65 | uint8_t const *data = *src; 66 | size_t const toHash = MIN(FUZZ_RNG_SEED_SIZE, *size); 67 | *size -= toHash; 68 | *src += toHash; 69 | return XXH32(data, toHash, 0); 70 | } 71 | 72 | #define FUZZ_rotl32(x, r) (((x) << (r)) | ((x) >> (32 - (r)))) 73 | 74 | FUZZ_STATIC uint32_t FUZZ_rand(uint32_t *state) { 75 | static const uint32_t prime1 = 2654435761U; 76 | static const uint32_t prime2 = 2246822519U; 77 | uint32_t rand32 = *state; 78 | rand32 *= prime1; 79 | rand32 += prime2; 80 | rand32 = FUZZ_rotl32(rand32, 13); 81 | *state = rand32; 82 | return rand32 >> 5; 83 | } 84 | 85 | /* Returns a random number in the range [min, max]. */ 86 | FUZZ_STATIC uint32_t FUZZ_rand32(uint32_t *state, uint32_t min, uint32_t max) { 87 | uint32_t random = FUZZ_rand(state); 88 | return min + (random % (max - min + 1)); 89 | } 90 | 91 | #ifdef __cplusplus 92 | } 93 | #endif 94 | 95 | #endif 96 | -------------------------------------------------------------------------------- /examples/blockStreaming_doubleBuffer.md: -------------------------------------------------------------------------------- 1 | # LZ4 Streaming API Example : Double Buffer 2 | by *Takayuki Matsuoka* 3 | 4 | `blockStreaming_doubleBuffer.c` is LZ4 Streaming API example which implements double buffer (de)compression. 5 | 6 | Please note : 7 | 8 | - Firstly, read "LZ4 Streaming API Basics". 9 | - This is relatively advanced application example. 10 | - Output file is not compatible with lz4frame and platform dependent. 11 | 12 | 13 | ## What's the point of this example ? 14 | 15 | - Handle huge file in small amount of memory 16 | - Always better compression ratio than Block API 17 | - Uniform block size 18 | 19 | 20 | ## How the compression works 21 | 22 | First of all, allocate "Double Buffer" for input and LZ4 compressed data buffer for output. 23 | Double buffer has two pages, "first" page (Page#1) and "second" page (Page#2). 24 | 25 | ``` 26 | Double Buffer 27 | 28 | Page#1 Page#2 29 | +---------+---------+ 30 | | Block#1 | | 31 | +----+----+---------+ 32 | | 33 | v 34 | {Out#1} 35 | 36 | 37 | Prefix Dependency 38 | +---------+ 39 | | | 40 | v | 41 | +---------+----+----+ 42 | | Block#1 | Block#2 | 43 | +---------+----+----+ 44 | | 45 | v 46 | {Out#2} 47 | 48 | 49 | External Dictionary Mode 50 | +---------+ 51 | | | 52 | | v 53 | +----+----+---------+ 54 | | Block#3 | Block#2 | 55 | +----+----+---------+ 56 | | 57 | v 58 | {Out#3} 59 | 60 | 61 | Prefix Dependency 62 | +---------+ 63 | | | 64 | v | 65 | +---------+----+----+ 66 | | Block#3 | Block#4 | 67 | +---------+----+----+ 68 | | 69 | v 70 | {Out#4} 71 | ``` 72 | 73 | Next, read first block to double buffer's first page. And compress it by `LZ4_compress_continue()`. 74 | For the first time, LZ4 doesn't know any previous dependencies, 75 | so it just compress the line without dependencies and generates compressed block {Out#1} to LZ4 compressed data buffer. 76 | After that, write {Out#1} to the file. 77 | 78 | Next, read second block to double buffer's second page. And compress it. 79 | This time, LZ4 can use dependency to Block#1 to improve compression ratio. 80 | This dependency is called "Prefix mode". 81 | 82 | Next, read third block to double buffer's *first* page, and compress it. 83 | Also this time, LZ4 can use dependency to Block#2. 84 | This dependency is called "External Dictonaly mode". 85 | 86 | Continue these procedure to the end of the file. 87 | 88 | 89 | ## How the decompression works 90 | 91 | Decompression will do reverse order. 92 | 93 | - Read first compressed block. 94 | - Decompress it to the first page and write that page to the file. 95 | - Read second compressed block. 96 | - Decompress it to the second page and write that page to the file. 97 | - Read third compressed block. 98 | - Decompress it to the *first* page and write that page to the file. 99 | 100 | Continue these procedure to the end of the compressed file. 101 | -------------------------------------------------------------------------------- /contrib/gen_manual/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (C) Przemyslaw Skibinski 2016-present 3 | # All rights reserved. 4 | # 5 | # BSD license 6 | # Redistribution and use in source and binary forms, with or without modification, 7 | # are permitted provided that the following conditions are met: 8 | # 9 | # * Redistributions of source code must retain the above copyright notice, this 10 | # list of conditions and the following disclaimer. 11 | # 12 | # * Redistributions in binary form must reproduce the above copyright notice, this 13 | # list of conditions and the following disclaimer in the documentation and/or 14 | # other materials provided with the distribution. 15 | # 16 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 17 | # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 20 | # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 | # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 23 | # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | # 27 | # You can contact the author at : 28 | # - LZ4 source repository : https://github.com/Cyan4973/lz4 29 | # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c 30 | # ################################################################ 31 | 32 | 33 | CXXFLAGS ?= -O2 34 | CXXFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment 35 | CPPFLAGS += $(MOREFLAGS) 36 | FLAGS = $(CXXFLAGS) $(CPPFLAGS) $(LDFLAGS) 37 | 38 | LZ4API = ../../lib/lz4.h 39 | LZ4MANUAL = ../../doc/lz4_manual.html 40 | LZ4FAPI = ../../lib/lz4frame.h 41 | LZ4FMANUAL = ../../doc/lz4frame_manual.html 42 | LIBVER_MAJOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)` 43 | LIBVER_MINOR_SCRIPT:=`sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)` 44 | LIBVER_PATCH_SCRIPT:=`sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < $(LZ4API)` 45 | LIBVER_SCRIPT:= $(LIBVER_MAJOR_SCRIPT).$(LIBVER_MINOR_SCRIPT).$(LIBVER_PATCH_SCRIPT) 46 | LZ4VER := $(shell echo $(LIBVER_SCRIPT)) 47 | 48 | # Define *.exe as extension for Windows systems 49 | ifneq (,$(filter Windows%,$(OS))) 50 | EXT =.exe 51 | else 52 | EXT = 53 | endif 54 | 55 | 56 | .PHONY: default 57 | default: gen_manual 58 | 59 | gen_manual: gen_manual.cpp 60 | $(CXX) $(FLAGS) $^ -o $@$(EXT) 61 | 62 | $(LZ4MANUAL) : gen_manual $(LZ4API) 63 | echo "Update lz4 manual in /doc" 64 | ./gen_manual $(LZ4VER) $(LZ4API) $@ 65 | 66 | $(LZ4FMANUAL) : gen_manual $(LZ4FAPI) 67 | echo "Update lz4frame manual in /doc" 68 | ./gen_manual $(LZ4VER) $(LZ4FAPI) $@ 69 | 70 | .PHONY: manuals 71 | manuals: $(LZ4MANUAL) $(LZ4FMANUAL) 72 | 73 | .PHONY: clean 74 | clean: 75 | @$(RM) gen_manual$(EXT) 76 | @echo Cleaning completed 77 | -------------------------------------------------------------------------------- /lib/dll/example/README.md: -------------------------------------------------------------------------------- 1 | LZ4 Windows binary package 2 | ==================================== 3 | 4 | #### The package contents 5 | 6 | - `lz4.exe` : Command Line Utility, supporting gzip-like arguments 7 | - `dll\msys-lz4-1.dll` : The DLL of LZ4 library, compiled by msys 8 | - `dll\liblz4.dll.a` : The import library of LZ4 library for Visual C++ 9 | - `example\` : The example of usage of LZ4 library 10 | - `include\` : Header files required with LZ4 library 11 | - `static\liblz4_static.lib` : The static LZ4 library 12 | 13 | 14 | #### Usage of Command Line Interface 15 | 16 | Command Line Interface (CLI) supports gzip-like arguments. 17 | By default CLI takes an input file and compresses it to an output file: 18 | ``` 19 | Usage: lz4 [arg] [input] [output] 20 | ``` 21 | The full list of commands for CLI can be obtained with `-h` or `-H`. The ratio can 22 | be improved with commands from `-3` to `-16` but higher levels also have slower 23 | compression. CLI includes in-memory compression benchmark module with compression 24 | levels starting from `-b` and ending with `-e` with iteration time of `-i` seconds. 25 | CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined 26 | into `-b1e18i1`. 27 | 28 | 29 | #### The example of usage of static and dynamic LZ4 libraries with gcc/MinGW 30 | 31 | Use `cd example` and `make` to build `fullbench-dll` and `fullbench-lib`. 32 | `fullbench-dll` uses a dynamic LZ4 library from the `dll` directory. 33 | `fullbench-lib` uses a static LZ4 library from the `lib` directory. 34 | 35 | 36 | #### Using LZ4 DLL with gcc/MinGW 37 | 38 | The header files from `include\` and the dynamic library `dll\msys-lz4-1.dll` 39 | are required to compile a project using gcc/MinGW. 40 | The dynamic library has to be added to linking options. 41 | It means that if a project that uses LZ4 consists of a single `test-dll.c` 42 | file it should be linked with `dll\msys-lz4-1.dll`. For example: 43 | ``` 44 | gcc $(CFLAGS) -Iinclude\ test-dll.c -o test-dll dll\msys-lz4-1.dll 45 | ``` 46 | The compiled executable will require LZ4 DLL which is available at `dll\msys-lz4-1.dll`. 47 | 48 | 49 | #### The example of usage of static and dynamic LZ4 libraries with Visual C++ 50 | 51 | Open `example\fullbench-dll.sln` to compile `fullbench-dll` that uses a 52 | dynamic LZ4 library from the `dll` directory. The solution works with Visual C++ 53 | 2010 or newer. When one will open the solution with Visual C++ newer than 2010 54 | then the solution will be upgraded to the current version. 55 | 56 | 57 | #### Using LZ4 DLL with Visual C++ 58 | 59 | The header files from `include\` and the import library `dll\liblz4.dll.a` 60 | are required to compile a project using Visual C++. 61 | 62 | 1. The header files should be added to `Additional Include Directories` that can 63 | be found in project properties `C/C++` then `General`. 64 | 2. The import library has to be added to `Additional Dependencies` that can 65 | be found in project properties `Linker` then `Input`. 66 | If one will provide only the name `liblz4.dll.a` without a full path to the library 67 | the directory has to be added to `Linker\General\Additional Library Directories`. 68 | 69 | The compiled executable will require LZ4 DLL which is available at `dll\msys-lz4-1.dll`. 70 | -------------------------------------------------------------------------------- /lib/lz4file.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 file library 3 | Header File 4 | Copyright (C) 2022, Xiaomi Inc. 5 | BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | You can contact the author at : 31 | - LZ4 source repository : https://github.com/lz4/lz4 32 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 33 | */ 34 | #if defined (__cplusplus) 35 | extern "C" { 36 | #endif 37 | 38 | #ifndef LZ4FILE_H 39 | #define LZ4FILE_H 40 | 41 | #include 42 | #include "lz4frame_static.h" 43 | 44 | typedef struct LZ4_readFile_s LZ4_readFile_t; 45 | typedef struct LZ4_writeFile_s LZ4_writeFile_t; 46 | 47 | /*! LZ4F_readOpen() : 48 | * Set read lz4file handle. 49 | * `lz4f` will set a lz4file handle. 50 | * `fp` must be the return value of the lz4 file opened by fopen. 51 | */ 52 | LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_readOpen(LZ4_readFile_t** lz4fRead, FILE* fp); 53 | 54 | /*! LZ4F_read() : 55 | * Read lz4file content to buffer. 56 | * `lz4f` must use LZ4_readOpen to set first. 57 | * `buf` read data buffer. 58 | * `size` read data buffer size. 59 | */ 60 | LZ4FLIB_STATIC_API size_t LZ4F_read(LZ4_readFile_t* lz4fRead, void* buf, size_t size); 61 | 62 | /*! LZ4F_readClose() : 63 | * Close lz4file handle. 64 | * `lz4f` must use LZ4_readOpen to set first. 65 | */ 66 | LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_readClose(LZ4_readFile_t* lz4fRead); 67 | 68 | /*! LZ4F_writeOpen() : 69 | * Set write lz4file handle. 70 | * `lz4f` will set a lz4file handle. 71 | * `fp` must be the return value of the lz4 file opened by fopen. 72 | */ 73 | LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_writeOpen(LZ4_writeFile_t** lz4fWrite, FILE* fp, const LZ4F_preferences_t* prefsPtr); 74 | 75 | /*! LZ4F_write() : 76 | * Write buffer to lz4file. 77 | * `lz4f` must use LZ4F_writeOpen to set first. 78 | * `buf` write data buffer. 79 | * `size` write data buffer size. 80 | */ 81 | LZ4FLIB_STATIC_API size_t LZ4F_write(LZ4_writeFile_t* lz4fWrite, void* buf, size_t size); 82 | 83 | /*! LZ4F_writeClose() : 84 | * Close lz4file handle. 85 | * `lz4f` must use LZ4F_writeOpen to set first. 86 | */ 87 | LZ4FLIB_STATIC_API LZ4F_errorCode_t LZ4F_writeClose(LZ4_writeFile_t* lz4fWrite); 88 | 89 | #endif /* LZ4FILE_H */ 90 | 91 | #if defined (__cplusplus) 92 | } 93 | #endif 94 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | Programs and scripts for automated testing of LZ4 2 | ======================================================= 3 | 4 | This directory contains the following programs and scripts: 5 | - `datagen` : Synthetic and parametrable data generator, for tests 6 | - `frametest` : Test tool that checks lz4frame integrity on target platform 7 | - `fullbench` : Precisely measure speed for each lz4 inner functions 8 | - `fuzzer` : Test tool, to check lz4 integrity on target platform 9 | - `test-lz4-speed.py` : script for testing lz4 speed difference between commits 10 | - `test-lz4-versions.py` : compatibility test between lz4 versions stored on Github 11 | 12 | 13 | #### `test-lz4-versions.py` - script for testing lz4 interoperability between versions 14 | 15 | This script creates `versionsTest` directory to which lz4 repository is cloned. 16 | Then all tagged (released) versions of lz4 are compiled. 17 | In the following step interoperability between lz4 versions is checked. 18 | 19 | 20 | #### `test-lz4-speed.py` - script for testing lz4 speed difference between commits 21 | 22 | This script creates `speedTest` directory to which lz4 repository is cloned. 23 | Then it compiles all branches of lz4 and performs a speed benchmark for a given list of files (the `testFileNames` parameter). 24 | After `sleepTime` (an optional parameter, default 300 seconds) seconds the script checks repository for new commits. 25 | If a new commit is found it is compiled and a speed benchmark for this commit is performed. 26 | The results of the speed benchmark are compared to the previous results. 27 | If compression or decompression speed for one of lz4 levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted. 28 | If second results are also lower than `lowerLimit` the warning e-mail is sent to recipients from the list (the `emails` parameter). 29 | 30 | Additional remarks: 31 | - To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel 32 | - Using the script with virtual machines can lead to large variations of speed results 33 | - The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75) 34 | - The script sends e-mails using `mutt`; if `mutt` is not available it sends e-mails without attachments using `mail`; if both are not available it only prints a warning 35 | 36 | 37 | The example usage with two test files, one e-mail address, and with an additional message: 38 | ``` 39 | ./test-lz4-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60 40 | ``` 41 | 42 | To run the script in background please use: 43 | ``` 44 | nohup ./test-lz4-speed.py testFileNames emails & 45 | ``` 46 | 47 | The full list of parameters: 48 | ``` 49 | positional arguments: 50 | testFileNames file names list for speed benchmark 51 | emails list of e-mail addresses to send warnings 52 | 53 | optional arguments: 54 | -h, --help show this help message and exit 55 | --message MESSAGE attach an additional message to e-mail 56 | --lowerLimit LOWERLIMIT 57 | send email if speed is lower than given limit 58 | --maxLoadAvg MAXLOADAVG 59 | maximum load average to start testing 60 | --lastCLevel LASTCLEVEL 61 | last compression level for testing 62 | --sleepTime SLEEPTIME 63 | frequency of repository checking in seconds 64 | ``` 65 | 66 | 67 | #### License 68 | 69 | All files in this directory are licensed under GPL-v2. 70 | See [COPYING](COPYING) for details. 71 | The text of the license is also included at the top of each source file. 72 | -------------------------------------------------------------------------------- /ossfuzz/decompress_fuzzer.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This fuzz target attempts to decompress the fuzzed data with the simple 3 | * decompression function to ensure the decompressor never crashes. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "fuzz_helpers.h" 12 | #include "fuzz_data_producer.h" 13 | #include "lz4.h" 14 | 15 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) 16 | { 17 | FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size); 18 | size_t const dstCapacitySeed = FUZZ_dataProducer_retrieve32(producer); 19 | size = FUZZ_dataProducer_remainingBytes(producer); 20 | 21 | size_t const dstCapacity = FUZZ_getRange_from_uint32(dstCapacitySeed, 0, 4 * size); 22 | size_t const smallDictSize = size + 1; 23 | size_t const largeDictSize = 64 * 1024 - 1; 24 | size_t const dictSize = MAX(smallDictSize, largeDictSize); 25 | char* const dst = (char*)malloc(dstCapacity); 26 | char* const dict = (char*)malloc(dictSize + size); 27 | char* const largeDict = dict; 28 | char* const dataAfterDict = dict + dictSize; 29 | char* const smallDict = dataAfterDict - smallDictSize; 30 | 31 | FUZZ_ASSERT(dst); 32 | FUZZ_ASSERT(dict); 33 | 34 | /* Prepare the dictionary. The data doesn't matter for decompression. */ 35 | memset(dict, 0, dictSize); 36 | memcpy(dataAfterDict, data, size); 37 | 38 | /* Decompress using each possible dictionary configuration. */ 39 | /* No dictionary. */ 40 | LZ4_decompress_safe_usingDict((char const*)data, dst, size, 41 | dstCapacity, NULL, 0); 42 | /* Small external dictionary. */ 43 | LZ4_decompress_safe_usingDict((char const*)data, dst, size, 44 | dstCapacity, smallDict, smallDictSize); 45 | /* Large external dictionary. */ 46 | LZ4_decompress_safe_usingDict((char const*)data, dst, size, 47 | dstCapacity, largeDict, largeDictSize); 48 | /* Small prefix. */ 49 | LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size, 50 | dstCapacity, smallDict, smallDictSize); 51 | /* Large prefix. */ 52 | LZ4_decompress_safe_usingDict((char const*)dataAfterDict, dst, size, 53 | dstCapacity, largeDict, largeDictSize); 54 | /* Partial decompression. */ 55 | LZ4_decompress_safe_partial((char const*)data, dst, size, 56 | dstCapacity, dstCapacity); 57 | /* Partial decompression using each possible dictionary configuration. */ 58 | /* Partial decompression with no dictionary. */ 59 | LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size, 60 | dstCapacity, dstCapacity, NULL, 0); 61 | /* Partial decompression with small external dictionary. */ 62 | LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size, 63 | dstCapacity, dstCapacity, smallDict, smallDictSize); 64 | /* Partial decompression with large external dictionary. */ 65 | LZ4_decompress_safe_partial_usingDict((char const*)data, dst, size, 66 | dstCapacity, dstCapacity, largeDict, largeDictSize); 67 | /* Partial decompression with small prefix. */ 68 | LZ4_decompress_safe_partial_usingDict((char const*)dataAfterDict, dst, size, 69 | dstCapacity, dstCapacity, smallDict, smallDictSize); 70 | /* Partial decompression wtih large prefix. */ 71 | LZ4_decompress_safe_partial_usingDict((char const*)dataAfterDict, dst, size, 72 | dstCapacity, dstCapacity, largeDict, largeDictSize); 73 | free(dst); 74 | free(dict); 75 | FUZZ_dataProducer_free(producer); 76 | 77 | return 0; 78 | } 79 | -------------------------------------------------------------------------------- /programs/README.md: -------------------------------------------------------------------------------- 1 | Command Line Interface for LZ4 library 2 | ============================================ 3 | 4 | ### Build 5 | The Command Line Interface (CLI) can be generated 6 | using the `make` command without any additional parameters. 7 | 8 | The `Makefile` script supports all [standard conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html), 9 | including standard targets (`all`, `install`, `clean`, etc.) 10 | and standard variables (`CC`, `CFLAGS`, `CPPFLAGS`, etc.). 11 | 12 | For advanced use cases, there are targets to different variations of the CLI: 13 | - `lz4` : default CLI, with a command line syntax close to gzip 14 | - `lz4c` : Same as `lz4` with additional support legacy lz4 commands (incompatible with gzip) 15 | - `lz4c32` : Same as `lz4c`, but forced to compile in 32-bits mode 16 | 17 | The CLI generates and decodes [LZ4-compressed frames](../doc/lz4_Frame_format.md). 18 | 19 | 20 | #### Aggregation of parameters 21 | CLI supports aggregation of parameters i.e. `-b1`, `-e18`, and `-i1` can be joined into `-b1e18i1`. 22 | 23 | 24 | #### Benchmark in Command Line Interface 25 | CLI includes in-memory compression benchmark module for lz4. 26 | The benchmark is conducted using a given filename. 27 | The file is read into memory. 28 | It makes benchmark more precise as it eliminates I/O overhead. 29 | 30 | The benchmark measures ratio, compressed size, compression and decompression speed. 31 | One can select compression levels starting from `-b` and ending with `-e`. 32 | The `-i` parameter selects a number of seconds used for each of tested levels. 33 | 34 | 35 | 36 | #### Usage of Command Line Interface 37 | The full list of commands can be obtained with `-h` or `-H` parameter: 38 | ``` 39 | Usage : 40 | lz4 [arg] [input] [output] 41 | 42 | input : a filename 43 | with no FILE, or when FILE is - or stdin, read standard input 44 | Arguments : 45 | -1 : Fast compression (default) 46 | -9 : High compression 47 | -d : decompression (default for .lz4 extension) 48 | -z : force compression 49 | -D FILE: use FILE as dictionary 50 | -f : overwrite output without prompting 51 | -k : preserve source files(s) (default) 52 | --rm : remove source file(s) after successful de/compression 53 | -h/-H : display help/long help and exit 54 | 55 | Advanced arguments : 56 | -V : display Version number and exit 57 | -v : verbose mode 58 | -q : suppress warnings; specify twice to suppress errors too 59 | -c : force write to standard output, even if it is the console 60 | -t : test compressed file integrity 61 | -m : multiple input files (implies automatic output filenames) 62 | -r : operate recursively on directories (sets also -m) 63 | -l : compress using Legacy format (Linux kernel compression) 64 | -B# : cut file into blocks of size # bytes [32+] 65 | or predefined block size [4-7] (default: 7) 66 | -BD : Block dependency (improve compression ratio) 67 | -BX : enable block checksum (default:disabled) 68 | --no-frame-crc : disable stream checksum (default:enabled) 69 | --content-size : compressed frame includes original size (default:not present) 70 | --[no-]sparse : sparse mode (default:enabled on file, disabled on stdout) 71 | --favor-decSpeed: compressed files decompress faster, but are less compressed 72 | --fast[=#]: switch to ultra fast compression level (default: 1) 73 | 74 | Benchmark arguments : 75 | -b# : benchmark file(s), using # compression level (default : 1) 76 | -e# : test all compression levels from -bX to # (default : 1) 77 | -i# : minimum evaluation time in seconds (default : 3s)``` 78 | ``` 79 | 80 | #### License 81 | 82 | All files in this directory are licensed under GPL-v2. 83 | See [COPYING](COPYING) for details. 84 | The text of the license is also included at the top of each source file. 85 | -------------------------------------------------------------------------------- /Makefile.inc: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # LZ4 - Makefile common definitions 3 | # Copyright (C) Yann Collet 2020 4 | # All rights reserved. 5 | # 6 | # BSD license 7 | # Redistribution and use in source and binary forms, with or without modification, 8 | # are permitted provided that the following conditions are met: 9 | # 10 | # * Redistributions of source code must retain the above copyright notice, this 11 | # list of conditions and the following disclaimer. 12 | # 13 | # * Redistributions in binary form must reproduce the above copyright notice, this 14 | # list of conditions and the following disclaimer in the documentation and/or 15 | # other materials provided with the distribution. 16 | # 17 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 21 | # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 | # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 | # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 24 | # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | # 28 | # You can contact the author at : 29 | # - LZ4 source repository : https://github.com/lz4/lz4 30 | # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c 31 | # ################################################################ 32 | 33 | UNAME ?= uname 34 | 35 | TARGET_OS ?= $(shell $(UNAME)) 36 | ifeq ($(TARGET_OS),) 37 | TARGET_OS ?= $(OS) 38 | endif 39 | 40 | ifneq (,$(filter Windows%,$(TARGET_OS))) 41 | LIBLZ4 = liblz4-$(LIBVER_MAJOR) 42 | LIBLZ4_EXP = liblz4.lib 43 | WINBASED = yes 44 | else 45 | LIBLZ4_EXP = liblz4.dll.a 46 | ifneq (,$(filter MINGW%,$(TARGET_OS))) 47 | LIBLZ4 = liblz4 48 | WINBASED = yes 49 | else 50 | ifneq (,$(filter MSYS%,$(TARGET_OS))) 51 | LIBLZ4 = msys-lz4-$(LIBVER_MAJOR) 52 | WINBASED = yes 53 | else 54 | ifneq (,$(filter CYGWIN%,$(TARGET_OS))) 55 | LIBLZ4 = cyglz4-$(LIBVER_MAJOR) 56 | WINBASED = yes 57 | else 58 | LIBLZ4 = liblz4.$(SHARED_EXT_VER) 59 | WINBASED = no 60 | EXT = 61 | endif 62 | endif 63 | endif 64 | endif 65 | 66 | ifeq ($(WINBASED),yes) 67 | EXT = .exe 68 | WINDRES = windres 69 | endif 70 | 71 | #determine if dev/nul based on host environment 72 | ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell $(UNAME)))) 73 | VOID := /dev/null 74 | else 75 | ifneq (,$(filter Windows%,$(OS))) 76 | VOID := nul 77 | else 78 | VOID := /dev/null 79 | endif 80 | endif 81 | 82 | ifneq (,$(filter Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku MidnightBSD MINGW% CYGWIN% MSYS%,$(shell $(UNAME)))) 83 | POSIX_ENV = Yes 84 | else 85 | POSIX_ENV = No 86 | endif 87 | 88 | # Avoid symlinks when targeting Windows or building on a Windows host 89 | ifeq ($(WINBASED),yes) 90 | LN_SF = cp -p 91 | else 92 | ifneq (,$(filter MINGW% MSYS% CYGWIN%,$(shell $(UNAME)))) 93 | LN_SF = cp -p 94 | else 95 | ifneq (,$(filter Windows%,$(OS))) 96 | LN_SF = cp -p 97 | else 98 | LN_SF = ln -sf 99 | endif 100 | endif 101 | endif 102 | 103 | ifneq (,$(filter $(shell $(UNAME)),SunOS)) 104 | INSTALL ?= ginstall 105 | else 106 | INSTALL ?= install 107 | endif 108 | 109 | INSTALL_PROGRAM ?= $(INSTALL) -m 755 110 | INSTALL_DATA ?= $(INSTALL) -m 644 111 | INSTALL_DIR ?= $(INSTALL) -d -m 755 112 | -------------------------------------------------------------------------------- /examples/streaming_api_basics.md: -------------------------------------------------------------------------------- 1 | # LZ4 Streaming API Basics 2 | by *Takayuki Matsuoka* 3 | ## LZ4 API sets 4 | 5 | LZ4 has the following API sets : 6 | 7 | - "Auto Framing" API (lz4frame.h) : 8 | This is most recommended API for usual application. 9 | It guarantees interoperability with other LZ4 framing format compliant tools/libraries 10 | such as LZ4 command line utility, node-lz4, etc. 11 | - "Block" API : This is recommended for simple purpose. 12 | It compresses single raw memory block to LZ4 memory block and vice versa. 13 | - "Streaming" API : This is designed for complex things. 14 | For example, compress huge stream data in restricted memory environment. 15 | 16 | Basically, you should use "Auto Framing" API. 17 | But if you want to write advanced application, it's time to use Block or Streaming APIs. 18 | 19 | 20 | ## What is difference between Block and Streaming API ? 21 | 22 | Block API (de)compresses a single contiguous memory block. 23 | In other words, LZ4 library finds redundancy from a single contiguous memory block. 24 | Streaming API does same thing but (de)compresses multiple adjacent contiguous memory blocks. 25 | So Streaming API could find more redundancy than Block API. 26 | 27 | The following figure shows difference between API and block sizes. 28 | In these figures, the original data is split into 4KiBytes contiguous chunks. 29 | 30 | ``` 31 | Original Data 32 | +---------------+---------------+----+----+----+ 33 | | 4KiB Chunk A | 4KiB Chunk B | C | D |... | 34 | +---------------+---------------+----+----+----+ 35 | 36 | Example (1) : Block API, 4KiB Block 37 | +---------------+---------------+----+----+----+ 38 | | 4KiB Chunk A | 4KiB Chunk B | C | D |... | 39 | +---------------+---------------+----+----+----+ 40 | | Block #1 | Block #2 | #3 | #4 |... | 41 | +---------------+---------------+----+----+----+ 42 | 43 | (No Dependency) 44 | 45 | 46 | Example (2) : Block API, 8KiB Block 47 | +---------------+---------------+----+----+----+ 48 | | 4KiB Chunk A | 4KiB Chunk B | C | D |... | 49 | +---------------+---------------+----+----+----+ 50 | | Block #1 |Block #2 |... | 51 | +--------------------+----------+-------+-+----+ 52 | ^ | ^ | 53 | | | | | 54 | +--------------+ +----+ 55 | Internal Dependency Internal Dependency 56 | 57 | 58 | Example (3) : Streaming API, 4KiB Block 59 | +---------------+---------------+-----+----+----+ 60 | | 4KiB Chunk A | 4KiB Chunk B | C | D |... | 61 | +---------------+---------------+-----+----+----+ 62 | | Block #1 | Block #2 | #3 | #4 |... | 63 | +---------------+----+----------+-+---+-+--+----+ 64 | ^ | ^ | ^ | 65 | | | | | | | 66 | +--------------+ +--------+ +---+ 67 | Dependency Dependency Dependency 68 | ``` 69 | 70 | - In example (1), there is no dependency. 71 | All blocks are compressed independently. 72 | - In example (2), naturally 8KiBytes block has internal dependency. 73 | But still block #1 and #2 are compressed independently. 74 | - In example (3), block #2 has dependency to #1, 75 | also #3 has dependency to #2 and #1, #4 has #3, #2 and #1, and so on. 76 | 77 | Here, we can observe difference between example (2) and (3). 78 | In (2), there's no dependency between chunk B and C, but (3) has dependency between B and C. 79 | This dependency improves compression ratio. 80 | 81 | 82 | ## Restriction of Streaming API 83 | 84 | For efficiency, Streaming API doesn't keep a mirror copy of dependent (de)compressed memory. 85 | This means users should keep these dependent (de)compressed memory explicitly. 86 | Usually, "Dependent memory" is previous adjacent contiguous memory up to 64KiBytes. 87 | LZ4 will not access further memories. 88 | -------------------------------------------------------------------------------- /contrib/djgpp/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015, Louis P. Santillan 2 | # All rights reserved. 3 | # See LICENSE for licensing details. 4 | DESTDIR ?= /opt/local 5 | 6 | # Pulled the code below from lib/Makefile. Might be nicer to derive this somehow without sed 7 | # Version numbers 8 | VERSION ?= 129 9 | RELEASE ?= r$(VERSION) 10 | LIBVER_MAJOR=$(shell sed -n '/define LZ4_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lib/lz4.h) 11 | LIBVER_MINOR=$(shell sed -n '/define LZ4_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lib/lz4.h) 12 | LIBVER_PATCH=$(shell sed -n '/define LZ4_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < lib/lz4.h) 13 | LIBVER=$(LIBVER_MAJOR).$(LIBVER_MINOR).$(LIBVER_PATCH) 14 | 15 | ###################################################################### 16 | 17 | CROSS ?= i586-pc-msdosdjgpp 18 | CC = $(CROSS)-gcc 19 | AR = $(CROSS)-ar 20 | LD = $(CROSS)-gcc 21 | 22 | CFLAGS ?= -O3 -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-qual -Wcast-align -Wstrict-prototypes -pedantic -DLZ4_VERSION=\"$(RELEASE)\" 23 | LDFLAGS ?= -s 24 | SRC = programs/bench.c programs/lz4io.c programs/lz4cli.c 25 | OBJ = $(SRC:.c=.o) 26 | SDEPS = $(SRC:.c=.d) 27 | IDIR = lib 28 | EDIR = . 29 | EXE = lz4.exe 30 | LNK = lz4 31 | LDIR = lib 32 | LSRC = lib/lz4.c lib/lz4hc.c lib/lz4frame.c lib/xxhash.c 33 | INC = $(LSRC:.c=.h) 34 | LOBJ = $(LSRC:.c=.o) 35 | LSDEPS = $(LSRC:.c=.d) 36 | LIB = $(LDIR)/lib$(LNK).a 37 | 38 | # Since LDFLAGS defaults to "-s", probably better to override unless 39 | # you have a default you would like to maintain 40 | ifeq ($(WITH_DEBUG), 1) 41 | CFLAGS += -g 42 | LDFLAGS += -g 43 | endif 44 | 45 | # Since LDFLAGS defaults to "-s", probably better to override unless 46 | # you have a default you would like to maintain 47 | ifeq ($(WITH_PROFILING), 1) 48 | CFLAGS += -pg 49 | LDFLAGS += -pg 50 | endif 51 | 52 | %.o: %.c $(INC) Makefile 53 | $(CC) $(CFLAGS) -MMD -MP -I$(IDIR) -c $< -o $@ 54 | 55 | %.exe: %.o $(LIB) Makefile 56 | $(LD) $< -L$(LDIR) -l$(LNK) $(LDFLAGS) $(LIBDEP) -o $@ 57 | 58 | ###################################################################### 59 | ######################## DO NOT MODIFY BELOW ######################### 60 | ###################################################################### 61 | 62 | .PHONY: all install uninstall showconfig gstat gpush 63 | 64 | all: $(LIB) $(EXE) 65 | 66 | $(LIB): $(LOBJ) 67 | $(AR) -rcs $@ $^ 68 | 69 | $(EXE): $(LOBJ) $(OBJ) 70 | $(LD) $(LDFLAGS) $(LOBJ) $(OBJ) -o $(EDIR)/$@ 71 | 72 | clean: 73 | rm -f $(OBJ) $(EXE) $(LOBJ) $(LIB) *.tmp $(SDEPS) $(LSDEPS) $(TSDEPS) 74 | 75 | install: $(INC) $(LIB) $(EXE) 76 | mkdir -p $(DESTDIR)/bin $(DESTDIR)/include $(DESTDIR)/lib 77 | rm -f .footprint 78 | echo $(DESTDIR)/bin/$(EXE) >> .footprint 79 | cp -v $(EXE) $(DESTDIR)/bin/ 80 | @for T in $(LIB); \ 81 | do ( \ 82 | echo $(DESTDIR)/$$T >> .footprint; \ 83 | cp -v --parents $$T $(DESTDIR) \ 84 | ); done 85 | @for T in $(INC); \ 86 | do ( \ 87 | echo $(DESTDIR)/include/`basename -a $$T` >> .footprint; \ 88 | cp -v $$T $(DESTDIR)/include/ \ 89 | ); done 90 | 91 | uninstall: .footprint 92 | @for T in $(shell cat .footprint); do rm -v $$T; done 93 | 94 | -include $(SDEPS) $(LSDEPS) 95 | 96 | showconfig: 97 | @echo "PWD="$(PWD) 98 | @echo "VERSION="$(VERSION) 99 | @echo "RELEASE="$(RELEASE) 100 | @echo "LIBVER_MAJOR="$(LIBVER_MAJOR) 101 | @echo "LIBVER_MINOR="$(LIBVER_MINOR) 102 | @echo "LIBVER_PATCH="$(LIBVER_PATCH) 103 | @echo "LIBVER="$(LIBVER) 104 | @echo "CROSS="$(CROSS) 105 | @echo "CC="$(CC) 106 | @echo "AR="$(AR) 107 | @echo "LD="$(LD) 108 | @echo "DESTDIR="$(DESTDIR) 109 | @echo "CFLAGS="$(CFLAGS) 110 | @echo "LDFLAGS="$(LDFLAGS) 111 | @echo "SRC="$(SRC) 112 | @echo "OBJ="$(OBJ) 113 | @echo "IDIR="$(IDIR) 114 | @echo "INC="$(INC) 115 | @echo "EDIR="$(EDIR) 116 | @echo "EXE="$(EXE) 117 | @echo "LDIR="$(LDIR) 118 | @echo "LSRC="$(LSRC) 119 | @echo "LOBJ="$(LOBJ) 120 | @echo "LNK="$(LNK) 121 | @echo "LIB="$(LIB) 122 | @echo "SDEPS="$(SDEPS) 123 | @echo "LSDEPS="$(LSDEPS) 124 | 125 | gstat: 126 | git status 127 | 128 | gpush: 129 | git commit 130 | git push 131 | -------------------------------------------------------------------------------- /examples/blockStreaming_lineByLine.md: -------------------------------------------------------------------------------- 1 | # LZ4 Streaming API Example : Line by Line Text Compression 2 | by *Takayuki Matsuoka* 3 | 4 | `blockStreaming_lineByLine.c` is LZ4 Streaming API example which implements line by line incremental (de)compression. 5 | 6 | Please note the following restrictions : 7 | 8 | - Firstly, read "LZ4 Streaming API Basics". 9 | - This is relatively advanced application example. 10 | - Output file is not compatible with lz4frame and platform dependent. 11 | 12 | 13 | ## What's the point of this example ? 14 | 15 | - Line by line incremental (de)compression. 16 | - Handle huge file in small amount of memory 17 | - Generally better compression ratio than Block API 18 | - Non-uniform block size 19 | 20 | 21 | ## How the compression works 22 | 23 | First of all, allocate "Ring Buffer" for input and LZ4 compressed data buffer for output. 24 | 25 | ``` 26 | (1) 27 | Ring Buffer 28 | 29 | +--------+ 30 | | Line#1 | 31 | +---+----+ 32 | | 33 | v 34 | {Out#1} 35 | 36 | 37 | (2) 38 | Prefix Mode Dependency 39 | +----+ 40 | | | 41 | v | 42 | +--------+-+------+ 43 | | Line#1 | Line#2 | 44 | +--------+---+----+ 45 | | 46 | v 47 | {Out#2} 48 | 49 | 50 | (3) 51 | Prefix Prefix 52 | +----+ +----+ 53 | | | | | 54 | v | v | 55 | +--------+-+------+-+------+ 56 | | Line#1 | Line#2 | Line#3 | 57 | +--------+--------+---+----+ 58 | | 59 | v 60 | {Out#3} 61 | 62 | 63 | (4) 64 | External Dictionary Mode 65 | +----+ +----+ 66 | | | | | 67 | v | v | 68 | ------+--------+-+------+-+--------+ 69 | | .... | Line#X | Line#X+1 | 70 | ------+--------+--------+-----+----+ 71 | ^ | 72 | | v 73 | | {Out#X+1} 74 | | 75 | Reset 76 | 77 | 78 | (5) 79 | Prefix 80 | +-----+ 81 | | | 82 | v | 83 | ------+--------+--------+----------+--+-------+ 84 | | .... | Line#X | Line#X+1 | Line#X+2 | 85 | ------+--------+--------+----------+-----+----+ 86 | ^ | 87 | | v 88 | | {Out#X+2} 89 | | 90 | Reset 91 | ``` 92 | 93 | Next (see (1)), read first line to ringbuffer and compress it by `LZ4_compress_continue()`. 94 | For the first time, LZ4 doesn't know any previous dependencies, 95 | so it just compress the line without dependencies and generates compressed line {Out#1} to LZ4 compressed data buffer. 96 | After that, write {Out#1} to the file and forward ringbuffer offset. 97 | 98 | Do the same things to second line (see (2)). 99 | But in this time, LZ4 can use dependency to Line#1 to improve compression ratio. 100 | This dependency is called "Prefix mode". 101 | 102 | Eventually, we'll reach end of ringbuffer at Line#X (see (4)). 103 | This time, we should reset ringbuffer offset. 104 | After resetting, at Line#X+1 pointer is not adjacent, but LZ4 still maintain its memory. 105 | This is called "External Dictionary Mode". 106 | 107 | In Line#X+2 (see (5)), finally LZ4 forget almost all memories but still remains Line#X+1. 108 | This is the same situation as Line#2. 109 | 110 | Continue these procedures to the end of text file. 111 | 112 | 113 | ## How the decompression works 114 | 115 | Decompression will do reverse order. 116 | 117 | - Read compressed line from the file to buffer. 118 | - Decompress it to the ringbuffer. 119 | - Output decompressed plain text line to the file. 120 | - Forward ringbuffer offset. If offset exceeds end of the ringbuffer, reset it. 121 | 122 | Continue these procedures to the end of the compressed file. 123 | -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | # ########################################################################## 2 | # LZ4 examples - Makefile 3 | # Copyright (C) Yann Collet 2011-2020 4 | # 5 | # GPL v2 License 6 | # 7 | # This program is free software; you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation; either version 2 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License along 18 | # with this program; if not, write to the Free Software Foundation, Inc., 19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 20 | # 21 | # You can contact the author at : 22 | # - LZ4 source repository : https://github.com/lz4/lz4 23 | # - LZ4 forum froup : https://groups.google.com/forum/#!forum/lz4c 24 | # ########################################################################## 25 | # This makefile compile and test 26 | # example programs, using (mostly) LZ4 streaming library, 27 | # kindly provided by Takayuki Matsuoka 28 | # ########################################################################## 29 | 30 | CPPFLAGS += -I../lib 31 | CFLAGS ?= -O3 32 | CFLAGS += -std=gnu99 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Wstrict-prototypes 33 | FLAGS := $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(MOREFLAGS) 34 | 35 | TESTFILE = Makefile 36 | LZ4DIR := ../lib 37 | LZ4 = ../programs/lz4 38 | 39 | include ../Makefile.inc 40 | 41 | default: all 42 | 43 | all: printVersion doubleBuffer dictionaryRandomAccess ringBuffer ringBufferHC \ 44 | lineCompress frameCompress fileCompress simpleBuffer 45 | 46 | $(LZ4DIR)/liblz4.a: $(LZ4DIR)/lz4.c $(LZ4DIR)/lz4hc.c $(LZ4DIR)/lz4frame.c $(LZ4DIR)/lz4.h $(LZ4DIR)/lz4hc.h $(LZ4DIR)/lz4frame.h $(LZ4DIR)/lz4frame_static.h 47 | $(MAKE) -C $(LZ4DIR) liblz4.a 48 | 49 | printVersion: printVersion.c $(LZ4DIR)/liblz4.a 50 | $(CC) $(FLAGS) $^ -o $@$(EXT) 51 | 52 | doubleBuffer: blockStreaming_doubleBuffer.c $(LZ4DIR)/liblz4.a 53 | $(CC) $(FLAGS) $^ -o $@$(EXT) 54 | 55 | dictionaryRandomAccess: dictionaryRandomAccess.c $(LZ4DIR)/liblz4.a 56 | $(CC) $(FLAGS) $^ -o $@$(EXT) 57 | 58 | ringBuffer : blockStreaming_ringBuffer.c $(LZ4DIR)/liblz4.a 59 | $(CC) $(FLAGS) $^ -o $@$(EXT) 60 | 61 | ringBufferHC: HCStreaming_ringBuffer.c $(LZ4DIR)/liblz4.a 62 | $(CC) $(FLAGS) $^ -o $@$(EXT) 63 | 64 | lineCompress: blockStreaming_lineByLine.c $(LZ4DIR)/liblz4.a 65 | $(CC) $(FLAGS) $^ -o $@$(EXT) 66 | 67 | frameCompress: frameCompress.c $(LZ4DIR)/liblz4.a 68 | $(CC) $(FLAGS) $^ -o $@$(EXT) 69 | 70 | fileCompress: fileCompress.c $(LZ4DIR)/liblz4.a 71 | $(CC) $(FLAGS) $^ -o $@$(EXT) 72 | 73 | compressFunctions: compress_functions.c $(LZ4DIR)/liblz4.a 74 | $(CC) $(FLAGS) $^ -o $@$(EXT) -lrt 75 | 76 | simpleBuffer: simple_buffer.c $(LZ4DIR)/liblz4.a 77 | $(CC) $(FLAGS) $^ -o $@$(EXT) 78 | 79 | $(LZ4) : 80 | $(MAKE) -C ../programs lz4 81 | 82 | test : all $(LZ4) 83 | @echo "\n=== Print Version ===" 84 | ./printVersion$(EXT) 85 | @echo "\n=== Simple compression example ===" 86 | ./simpleBuffer$(EXT) 87 | @echo "\n=== Double-buffer ===" 88 | ./doubleBuffer$(EXT) $(TESTFILE) 89 | @echo "\n=== Ring Buffer ===" 90 | ./ringBuffer$(EXT) $(TESTFILE) 91 | @echo "\n=== Ring Buffer + LZ4 HC ===" 92 | ./ringBufferHC$(EXT) $(TESTFILE) 93 | @echo "\n=== Compress line by line ===" 94 | ./lineCompress$(EXT) $(TESTFILE) 95 | @echo "\n=== Dictionary Random Access ===" 96 | ./dictionaryRandomAccess$(EXT) $(TESTFILE) $(TESTFILE) 1100 1400 97 | @echo "\n=== Frame compression ===" 98 | ./frameCompress$(EXT) $(TESTFILE) 99 | $(LZ4) -vt $(TESTFILE).lz4 100 | @echo "\n=== file compression ===" 101 | ./fileCompress$(EXT) $(TESTFILE) 102 | $(LZ4) -vt $(TESTFILE).lz4 103 | 104 | .PHONY: cxxtest 105 | cxxtest: CFLAGS := -O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror 106 | cxxtest: clean 107 | CC=$(CXX) $(MAKE) -C . all CFLAGS="$(CFLAGS)" 108 | 109 | clean: 110 | @rm -f core *.o *.dec *-0 *-9 *-8192 *.lz4s *.lz4 \ 111 | printVersion$(EXT) doubleBuffer$(EXT) dictionaryRandomAccess$(EXT) \ 112 | ringBuffer$(EXT) ringBufferHC$(EXT) lineCompress$(EXT) frameCompress$(EXT) \ 113 | fileCompress$(EXT) compressFunctions$(EXT) simpleBuffer$(EXT) 114 | @echo Cleaning completed 115 | -------------------------------------------------------------------------------- /tests/decompress-partial-usingDict.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "lz4.h" 6 | 7 | const char source[] = 8 | "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod\n" 9 | "tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim\n" 10 | "veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea\n" 11 | "commodo consequat. Duis aute irure dolor in reprehenderit in voluptate\n" 12 | "velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat\n" 13 | "cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id\n" 14 | "est laborum.\n" 15 | "\n" 16 | "Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium\n" 17 | "doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore\n" 18 | "veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim\n" 19 | "ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia\n" 20 | "consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque\n" 21 | "porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur,\n" 22 | "adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore\n" 23 | "et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis\n" 24 | "nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid\n" 25 | "ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea\n" 26 | "voluptate velit esse quam nihil molestiae consequatur, vel illum qui\n" 27 | "dolorem eum fugiat quo voluptas nulla pariatur?\n"; 28 | 29 | #define BUFFER_SIZE 2048 30 | 31 | int main(void) 32 | { 33 | int srcLen = (int)strlen(source); 34 | size_t const smallSize = 1024; 35 | size_t const largeSize = 64 * 1024 - 1; 36 | char cmpBuffer[BUFFER_SIZE]; 37 | char* const buffer = (char*)malloc(BUFFER_SIZE + largeSize); 38 | char* outBuffer = buffer + largeSize; 39 | char* const dict = (char*)malloc(largeSize); 40 | char* const largeDict = dict; 41 | char* const smallDict = dict + largeSize - smallSize; 42 | int i; 43 | int cmpSize; 44 | 45 | printf("starting test decompress-partial-usingDict : \n"); 46 | assert(buffer != NULL); 47 | assert(dict != NULL); 48 | 49 | cmpSize = LZ4_compress_default(source, cmpBuffer, srcLen, BUFFER_SIZE); 50 | 51 | for (i = cmpSize; i < cmpSize + 10; ++i) { 52 | int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, NULL, 0); 53 | if ( (result < 0) 54 | || (result != srcLen) 55 | || memcmp(source, outBuffer, (size_t)srcLen) ) { 56 | printf("test decompress-partial-usingDict with no dict error \n"); 57 | return -1; 58 | } 59 | } 60 | 61 | for (i = cmpSize; i < cmpSize + 10; ++i) { 62 | int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, outBuffer - smallSize, smallSize); 63 | if ( (result < 0) 64 | || (result != srcLen) 65 | || memcmp(source, outBuffer, (size_t)srcLen) ) { 66 | printf("test decompress-partial-usingDict with small prefix error \n"); 67 | return -1; 68 | } 69 | } 70 | 71 | for (i = cmpSize; i < cmpSize + 10; ++i) { 72 | int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, buffer, largeSize); 73 | if ( (result < 0) 74 | || (result != srcLen) 75 | || memcmp(source, outBuffer, (size_t)srcLen) ) { 76 | printf("test decompress-partial-usingDict with large prefix error \n"); 77 | return -1; 78 | } 79 | } 80 | 81 | for (i = cmpSize; i < cmpSize + 10; ++i) { 82 | int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, smallDict, smallSize); 83 | if ( (result < 0) 84 | || (result != srcLen) 85 | || memcmp(source, outBuffer, (size_t)srcLen) ) { 86 | printf("test decompress-partial-usingDict with small external dict error \n"); 87 | return -1; 88 | } 89 | } 90 | 91 | for (i = cmpSize; i < cmpSize + 10; ++i) { 92 | int result = LZ4_decompress_safe_partial_usingDict(cmpBuffer, outBuffer, i, srcLen, BUFFER_SIZE, largeDict, largeSize); 93 | if ( (result < 0) 94 | || (result != srcLen) 95 | || memcmp(source, outBuffer, (size_t)srcLen) ) { 96 | printf("test decompress-partial-usingDict with large external dict error \n"); 97 | return -1; 98 | } 99 | } 100 | 101 | printf("test decompress-partial-usingDict OK \n"); 102 | return 0; 103 | } 104 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # This configuration was automatically generated from a CircleCI 1.0 config. 2 | # It should include any build commands you had along with commands that CircleCI 3 | # inferred from your project structure. We strongly recommend you read all the 4 | # comments in this file to understand the structure of CircleCI 2.0, as the idiom 5 | # for configuration has changed substantially in 2.0 to allow arbitrary jobs rather 6 | # than the prescribed lifecycle of 1.0. In general, we recommend using this generated 7 | # configuration as a reference rather than using it in production, though in most 8 | # cases it should duplicate the execution of your original 1.0 config. 9 | version: 2 10 | jobs: 11 | build: 12 | working_directory: ~/lz4/lz4 13 | # Parallelism is broken in this file : it just plays the same tests twice. 14 | # The script will have to be modified to support parallelism properly 15 | # In the meantime, set it to 1. 16 | parallelism: 1 17 | shell: /bin/bash --login 18 | # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did. 19 | # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables . 20 | environment: 21 | CIRCLE_ARTIFACTS: /tmp/circleci-artifacts 22 | CIRCLE_TEST_REPORTS: /tmp/circleci-test-results 23 | # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages. 24 | # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images. 25 | # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job. 26 | # We have selected a pre-built image that mirrors the build environment we use on 27 | # the 1.0 platform, but we recommend you choose an image more tailored to the needs 28 | # of each job. For more information on choosing an image (or alternatively using a 29 | # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/ 30 | # To see the list of pre-built images that CircleCI provides for most common languages see 31 | # https://circleci.com/docs/2.0/circleci-images/ 32 | docker: 33 | - image: fbopensource/lz4-circleci-primary:0.0.4 34 | steps: 35 | # Machine Setup 36 | # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each 37 | # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out. 38 | - checkout 39 | # Prepare for artifact and test results collection equivalent to how it was done on 1.0. 40 | # In many cases you can simplify this from what is generated here. 41 | # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/' 42 | - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS 43 | # Test 44 | # This would typically be a build job when using workflows, possibly combined with build 45 | # This is based on your 1.0 configuration file or project settings 46 | - run: CFLAGS= make clangtest && make clean 47 | - run: g++ -v; make cxxtest && make clean 48 | - run: gcc -v; g++ -v; make ctocpptest && make clean 49 | - run: gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -Werror" make check && make clean 50 | - run: gcc-5 -v; CC=gcc-5 CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean 51 | - run: gcc-6 -v; CC=gcc-6 MOREFLAGS="-O2 -Werror" make check && make clean 52 | - run: make cmake && make clean 53 | - run: make -C tests test-lz4 54 | - run: make -C tests test-lz4c 55 | - run: make -C tests test-frametest 56 | - run: make -C tests test-fuzzer && make clean 57 | - run: make -C lib all && make clean 58 | - run: pyenv global 3.4.4; make versionsTest MOREFLAGS=-I/usr/include/x86_64-linux-gnu && make clean 59 | - run: make travis-install && make clean 60 | - run: gcc -v; CFLAGS="-O2 -m32 -Werror" CPPFLAGS=-I/usr/include/x86_64-linux-gnu make check && make clean 61 | - run: clang -v; make staticAnalyze && make clean 62 | - run: make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static && make clean 63 | - run: make platformTest CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS=-m64 && make clean 64 | - run: make platformTest CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static && make clean 65 | - run: make platformTest CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static && make clean 66 | # Teardown 67 | # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each 68 | # Save test results 69 | - store_test_results: 70 | path: /tmp/circleci-test-results 71 | # Save artifacts 72 | - store_artifacts: 73 | path: /tmp/circleci-artifacts 74 | - store_artifacts: 75 | path: /tmp/circleci-test-results 76 | -------------------------------------------------------------------------------- /ossfuzz/round_trip_fuzzer.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This fuzz target performs a lz4 round-trip test (compress & decompress), 3 | * compares the result with the original, and calls abort() on corruption. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "fuzz_helpers.h" 12 | #include "lz4.h" 13 | #include "fuzz_data_producer.h" 14 | 15 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) 16 | { 17 | FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size); 18 | size_t const partialCapacitySeed = FUZZ_dataProducer_retrieve32(producer); 19 | size = FUZZ_dataProducer_remainingBytes(producer); 20 | 21 | size_t const partialCapacity = FUZZ_getRange_from_uint32(partialCapacitySeed, 0, size); 22 | size_t const dstCapacity = LZ4_compressBound(size); 23 | size_t const largeSize = 64 * 1024 - 1; 24 | size_t const smallSize = 1024; 25 | char* const dstPlusLargePrefix = (char*)malloc(dstCapacity + largeSize); 26 | FUZZ_ASSERT(dstPlusLargePrefix); 27 | char* const dstPlusSmallPrefix = dstPlusLargePrefix + largeSize - smallSize; 28 | char* const largeDict = (char*)malloc(largeSize); 29 | FUZZ_ASSERT(largeDict); 30 | char* const smallDict = largeDict + largeSize - smallSize; 31 | char* const dst = dstPlusLargePrefix + largeSize; 32 | char* const rt = (char*)malloc(size); 33 | FUZZ_ASSERT(rt); 34 | 35 | /* Compression must succeed and round trip correctly. */ 36 | int const dstSize = LZ4_compress_default((const char*)data, dst, 37 | size, dstCapacity); 38 | FUZZ_ASSERT(dstSize > 0); 39 | 40 | int const rtSize = LZ4_decompress_safe(dst, rt, dstSize, size); 41 | FUZZ_ASSERT_MSG(rtSize == size, "Incorrect size"); 42 | FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); 43 | 44 | /* Partial decompression must succeed. */ 45 | { 46 | char* const partial = (char*)malloc(partialCapacity); 47 | FUZZ_ASSERT(partial); 48 | int const partialSize = LZ4_decompress_safe_partial( 49 | dst, partial, dstSize, partialCapacity, partialCapacity); 50 | FUZZ_ASSERT(partialSize >= 0); 51 | FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size"); 52 | FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!"); 53 | free(partial); 54 | } 55 | /* Partial decompression using dict with no dict. */ 56 | { 57 | char* const partial = (char*)malloc(partialCapacity); 58 | FUZZ_ASSERT(partial); 59 | int const partialSize = LZ4_decompress_safe_partial_usingDict( 60 | dst, partial, dstSize, partialCapacity, partialCapacity, NULL, 0); 61 | FUZZ_ASSERT(partialSize >= 0); 62 | FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size"); 63 | FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!"); 64 | free(partial); 65 | } 66 | /* Partial decompression using dict with small prefix as dict */ 67 | { 68 | char* const partial = (char*)malloc(partialCapacity); 69 | FUZZ_ASSERT(partial); 70 | int const partialSize = LZ4_decompress_safe_partial_usingDict( 71 | dst, partial, dstSize, partialCapacity, partialCapacity, dstPlusSmallPrefix, smallSize); 72 | FUZZ_ASSERT(partialSize >= 0); 73 | FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size"); 74 | FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!"); 75 | free(partial); 76 | } 77 | /* Partial decompression using dict with large prefix as dict */ 78 | { 79 | char* const partial = (char*)malloc(partialCapacity); 80 | FUZZ_ASSERT(partial); 81 | int const partialSize = LZ4_decompress_safe_partial_usingDict( 82 | dst, partial, dstSize, partialCapacity, partialCapacity, dstPlusLargePrefix, largeSize); 83 | FUZZ_ASSERT(partialSize >= 0); 84 | FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size"); 85 | FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!"); 86 | free(partial); 87 | } 88 | /* Partial decompression using dict with small external dict */ 89 | { 90 | char* const partial = (char*)malloc(partialCapacity); 91 | FUZZ_ASSERT(partial); 92 | int const partialSize = LZ4_decompress_safe_partial_usingDict( 93 | dst, partial, dstSize, partialCapacity, partialCapacity, smallDict, smallSize); 94 | FUZZ_ASSERT(partialSize >= 0); 95 | FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size"); 96 | FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!"); 97 | free(partial); 98 | } 99 | /* Partial decompression using dict with large external dict */ 100 | { 101 | char* const partial = (char*)malloc(partialCapacity); 102 | FUZZ_ASSERT(partial); 103 | int const partialSize = LZ4_decompress_safe_partial_usingDict( 104 | dst, partial, dstSize, partialCapacity, partialCapacity, largeDict, largeSize); 105 | FUZZ_ASSERT(partialSize >= 0); 106 | FUZZ_ASSERT_MSG(partialSize == partialCapacity, "Incorrect size"); 107 | FUZZ_ASSERT_MSG(!memcmp(data, partial, partialSize), "Corruption!"); 108 | free(partial); 109 | } 110 | 111 | free(dstPlusLargePrefix); 112 | free(largeDict); 113 | free(rt); 114 | FUZZ_dataProducer_free(producer); 115 | 116 | return 0; 117 | } 118 | -------------------------------------------------------------------------------- /programs/lz4io.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4io.h - LZ4 File/Stream Interface 3 | Copyright (C) Yann Collet 2011-2020 4 | GPL v2 License 5 | 6 | This program is free software; you can redistribute it and/or modify 7 | it under the terms of the GNU General Public License as published by 8 | the Free Software Foundation; either version 2 of the License, or 9 | (at your option) any later version. 10 | 11 | This program is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU General Public License for more details. 15 | 16 | You should have received a copy of the GNU General Public License along 17 | with this program; if not, write to the Free Software Foundation, Inc., 18 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 19 | 20 | You can contact the author at : 21 | - LZ4 source repository : https://github.com/lz4/lz4 22 | - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c 23 | */ 24 | /* 25 | Note : this is stand-alone program. 26 | It is not part of LZ4 compression library, it is a user code of the LZ4 library. 27 | - The license of LZ4 library is BSD. 28 | - The license of xxHash library is BSD. 29 | - The license of this source file is GPLv2. 30 | */ 31 | 32 | #ifndef LZ4IO_H_237902873 33 | #define LZ4IO_H_237902873 34 | 35 | /*--- Dependency ---*/ 36 | #include /* size_t */ 37 | 38 | 39 | /* ************************************************** */ 40 | /* Special input/output values */ 41 | /* ************************************************** */ 42 | #define stdinmark "stdin" 43 | #define stdoutmark "stdout" 44 | #define NULL_OUTPUT "null" 45 | #ifdef _WIN32 46 | #define nulmark "nul" 47 | #else 48 | #define nulmark "/dev/null" 49 | #endif 50 | 51 | /* ************************************************** */ 52 | /* ****************** Type Definitions ************** */ 53 | /* ************************************************** */ 54 | 55 | typedef struct LZ4IO_prefs_s LZ4IO_prefs_t; 56 | 57 | LZ4IO_prefs_t* LZ4IO_defaultPreferences(void); 58 | void LZ4IO_freePreferences(LZ4IO_prefs_t* prefs); 59 | 60 | 61 | /* ************************************************** */ 62 | /* ****************** Functions ********************* */ 63 | /* ************************************************** */ 64 | 65 | /* if output_filename == stdoutmark, writes to stdout */ 66 | int LZ4IO_compressFilename(const char* input_filename, const char* output_filename, int compressionlevel, const LZ4IO_prefs_t* prefs); 67 | int LZ4IO_decompressFilename(const char* input_filename, const char* output_filename, const LZ4IO_prefs_t* prefs); 68 | 69 | /* if suffix == stdoutmark, writes to stdout */ 70 | int LZ4IO_compressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, int compressionlevel, const LZ4IO_prefs_t* prefs); 71 | int LZ4IO_decompressMultipleFilenames(const char** inFileNamesTable, int ifntSize, const char* suffix, const LZ4IO_prefs_t* prefs); 72 | 73 | 74 | /* ************************************************** */ 75 | /* ****************** Parameters ******************** */ 76 | /* ************************************************** */ 77 | 78 | int LZ4IO_setDictionaryFilename(LZ4IO_prefs_t* const prefs, const char* dictionaryFilename); 79 | 80 | /* Default setting : passThrough = 0; 81 | return : passThrough mode (0/1) */ 82 | int LZ4IO_setPassThrough(LZ4IO_prefs_t* const prefs, int yes); 83 | 84 | /* Default setting : overwrite = 1; 85 | return : overwrite mode (0/1) */ 86 | int LZ4IO_setOverwrite(LZ4IO_prefs_t* const prefs, int yes); 87 | 88 | /* Default setting : testMode = 0; 89 | return : testMode (0/1) */ 90 | int LZ4IO_setTestMode(LZ4IO_prefs_t* const prefs, int yes); 91 | 92 | /* blockSizeID : valid values : 4-5-6-7 93 | return : 0 if error, blockSize if OK */ 94 | size_t LZ4IO_setBlockSizeID(LZ4IO_prefs_t* const prefs, unsigned blockSizeID); 95 | 96 | /* blockSize : valid values : 32 -> 4MB 97 | return : 0 if error, actual blocksize if OK */ 98 | size_t LZ4IO_setBlockSize(LZ4IO_prefs_t* const prefs, size_t blockSize); 99 | 100 | /* Default setting : independent blocks */ 101 | typedef enum { LZ4IO_blockLinked=0, LZ4IO_blockIndependent} LZ4IO_blockMode_t; 102 | int LZ4IO_setBlockMode(LZ4IO_prefs_t* const prefs, LZ4IO_blockMode_t blockMode); 103 | 104 | /* Default setting : no block checksum */ 105 | int LZ4IO_setBlockChecksumMode(LZ4IO_prefs_t* const prefs, int xxhash); 106 | 107 | /* Default setting : stream checksum enabled */ 108 | int LZ4IO_setStreamChecksumMode(LZ4IO_prefs_t* const prefs, int xxhash); 109 | 110 | /* Default setting : 0 (no notification) */ 111 | int LZ4IO_setNotificationLevel(int level); 112 | 113 | /* Default setting : 0 (disabled) */ 114 | int LZ4IO_setSparseFile(LZ4IO_prefs_t* const prefs, int enable); 115 | 116 | /* Default setting : 0 == no content size present in frame header */ 117 | int LZ4IO_setContentSize(LZ4IO_prefs_t* const prefs, int enable); 118 | 119 | /* Default setting : 0 == src file preserved */ 120 | void LZ4IO_setRemoveSrcFile(LZ4IO_prefs_t* const prefs, unsigned flag); 121 | 122 | /* Default setting : 0 == favor compression ratio 123 | * Note : 1 only works for high compression levels (10+) */ 124 | void LZ4IO_favorDecSpeed(LZ4IO_prefs_t* const prefs, int favor); 125 | 126 | 127 | /* implement --list 128 | * @return 0 on success, 1 on error */ 129 | int LZ4IO_displayCompressedFilesInfo(const char** inFileNames, size_t ifnIdx); 130 | 131 | 132 | #endif /* LZ4IO_H_237902873 */ 133 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | LZ4 - Extremely fast compression 2 | ================================ 3 | 4 | LZ4 is lossless compression algorithm, 5 | providing compression speed > 500 MB/s per core, 6 | scalable with multi-cores CPU. 7 | It features an extremely fast decoder, 8 | with speed in multiple GB/s per core, 9 | typically reaching RAM speed limits on multi-core systems. 10 | 11 | Speed can be tuned dynamically, selecting an "acceleration" factor 12 | which trades compression ratio for faster speed. 13 | On the other end, a high compression derivative, LZ4_HC, is also provided, 14 | trading CPU time for improved compression ratio. 15 | All versions feature the same decompression speed. 16 | 17 | LZ4 is also compatible with [dictionary compression](https://github.com/facebook/zstd#the-case-for-small-data-compression), 18 | both at [API](https://github.com/lz4/lz4/blob/v1.8.3/lib/lz4frame.h#L481) and [CLI](https://github.com/lz4/lz4/blob/v1.8.3/programs/lz4.1.md#operation-modifiers) levels. 19 | It can ingest any input file as dictionary, though only the final 64KB are used. 20 | This capability can be combined with the [Zstandard Dictionary Builder](https://github.com/facebook/zstd/blob/v1.3.5/programs/zstd.1.md#dictionary-builder), 21 | in order to drastically improve compression performance on small files. 22 | 23 | 24 | LZ4 library is provided as open-source software using BSD 2-Clause license. 25 | 26 | 27 | |Branch |Status | 28 | |------------|---------| 29 | |dev | [![Build Status][travisDevBadge]][travisLink] [![Build status][AppveyorDevBadge]][AppveyorLink] | 30 | 31 | [travisDevBadge]: https://travis-ci.org/lz4/lz4.svg?branch=dev "Continuous Integration test suite" 32 | [travisLink]: https://travis-ci.org/lz4/lz4 33 | [AppveyorDevBadge]: https://ci.appveyor.com/api/projects/status/github/lz4/lz4?branch=dev&svg=true "Windows test suite" 34 | [AppveyorLink]: https://ci.appveyor.com/project/YannCollet/lz4-1lndh 35 | 36 | 37 | Benchmarks 38 | ------------------------- 39 | 40 | The benchmark uses [lzbench], from @inikep 41 | compiled with GCC v8.2.0 on Linux 64-bits (Ubuntu 4.18.0-17). 42 | The reference system uses a Core i7-9700K CPU @ 4.9GHz (w/ turbo boost). 43 | Benchmark evaluates the compression of reference [Silesia Corpus] 44 | in single-thread mode. 45 | 46 | [lzbench]: https://github.com/inikep/lzbench 47 | [Silesia Corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia 48 | 49 | | Compressor | Ratio | Compression | Decompression | 50 | | ---------- | ----- | ----------- | ------------- | 51 | | memcpy | 1.000 | 13700 MB/s | 13700 MB/s | 52 | |**LZ4 default (v1.9.0)** |**2.101**| **780 MB/s**| **4970 MB/s** | 53 | | LZO 2.09 | 2.108 | 670 MB/s | 860 MB/s | 54 | | QuickLZ 1.5.0 | 2.238 | 575 MB/s | 780 MB/s | 55 | | Snappy 1.1.4 | 2.091 | 565 MB/s | 1950 MB/s | 56 | | [Zstandard] 1.4.0 -1 | 2.883 | 515 MB/s | 1380 MB/s | 57 | | LZF v3.6 | 2.073 | 415 MB/s | 910 MB/s | 58 | | [zlib] deflate 1.2.11 -1| 2.730 | 100 MB/s | 415 MB/s | 59 | |**LZ4 HC -9 (v1.9.0)** |**2.721**| 41 MB/s | **4900 MB/s** | 60 | | [zlib] deflate 1.2.11 -6| 3.099 | 36 MB/s | 445 MB/s | 61 | 62 | [zlib]: http://www.zlib.net/ 63 | [Zstandard]: http://www.zstd.net/ 64 | 65 | LZ4 is also compatible and optimized for x32 mode, 66 | for which it provides additional speed performance. 67 | 68 | 69 | Installation 70 | ------------------------- 71 | 72 | ``` 73 | make 74 | make install # this command may require root permissions 75 | ``` 76 | 77 | LZ4's `Makefile` supports standard [Makefile conventions], 78 | including [staged installs], [redirection], or [command redefinition]. 79 | It is compatible with parallel builds (`-j#`). 80 | 81 | [Makefile conventions]: https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html 82 | [staged installs]: https://www.gnu.org/prep/standards/html_node/DESTDIR.html 83 | [redirection]: https://www.gnu.org/prep/standards/html_node/Directory-Variables.html 84 | [command redefinition]: https://www.gnu.org/prep/standards/html_node/Utilities-in-Makefiles.html 85 | 86 | ### Building LZ4 - Using vcpkg 87 | 88 | You can download and install LZ4 using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager: 89 | 90 | git clone https://github.com/Microsoft/vcpkg.git 91 | cd vcpkg 92 | ./bootstrap-vcpkg.sh 93 | ./vcpkg integrate install 94 | vcpkg install lz4 95 | 96 | The LZ4 port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. 97 | 98 | Documentation 99 | ------------------------- 100 | 101 | The raw LZ4 block compression format is detailed within [lz4_Block_format]. 102 | 103 | Arbitrarily long files or data streams are compressed using multiple blocks, 104 | for streaming requirements. These blocks are organized into a frame, 105 | defined into [lz4_Frame_format]. 106 | Interoperable versions of LZ4 must also respect the frame format. 107 | 108 | [lz4_Block_format]: doc/lz4_Block_format.md 109 | [lz4_Frame_format]: doc/lz4_Frame_format.md 110 | 111 | 112 | Other source versions 113 | ------------------------- 114 | 115 | Beyond the C reference source, 116 | many contributors have created versions of lz4 in multiple languages 117 | (Java, C#, Python, Perl, Ruby, etc.). 118 | A list of known source ports is maintained on the [LZ4 Homepage]. 119 | 120 | [LZ4 Homepage]: http://www.lz4.org 121 | -------------------------------------------------------------------------------- /ossfuzz/round_trip_frame_uncompressed_fuzzer.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This fuzz target performs a lz4 round-trip test (compress & decompress), 3 | * compares the result with the original, and calls abort() on corruption. 4 | */ 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "fuzz_data_producer.h" 12 | #include "fuzz_helpers.h" 13 | #include "lz4.h" 14 | #include "lz4_helpers.h" 15 | #include "lz4frame.h" 16 | #include "lz4frame_static.h" 17 | 18 | static void decompress(LZ4F_dctx *dctx, void *src, void *dst, 19 | size_t dstCapacity, size_t readSize) { 20 | size_t ret = 1; 21 | const void *srcPtr = (const char *) src; 22 | void *dstPtr = (char *) dst; 23 | const void *const srcEnd = (const char *) srcPtr + readSize; 24 | 25 | while (ret != 0) { 26 | while (srcPtr < srcEnd && ret != 0) { 27 | /* Any data within dst has been flushed at this stage */ 28 | size_t dstSize = dstCapacity; 29 | size_t srcSize = (const char *) srcEnd - (const char *) srcPtr; 30 | ret = LZ4F_decompress(dctx, dstPtr, &dstSize, srcPtr, &srcSize, 31 | /* LZ4F_decompressOptions_t */ NULL); 32 | FUZZ_ASSERT(!LZ4F_isError(ret)); 33 | 34 | /* Update input */ 35 | srcPtr = (const char *) srcPtr + srcSize; 36 | dstPtr = (char *) dstPtr + dstSize; 37 | } 38 | 39 | FUZZ_ASSERT(srcPtr <= srcEnd); 40 | } 41 | } 42 | 43 | static void compress_round_trip(const uint8_t *data, size_t size, 44 | FUZZ_dataProducer_t *producer, LZ4F_preferences_t const prefs) { 45 | 46 | // Choose random uncompressed offset start and end by producing seeds from random data, calculate the remaining 47 | // data size that will be used for compression later and use the seeds to actually calculate the offsets 48 | size_t const uncompressedOffsetSeed = FUZZ_dataProducer_retrieve32(producer); 49 | size_t const uncompressedEndOffsetSeed = FUZZ_dataProducer_retrieve32(producer); 50 | size = FUZZ_dataProducer_remainingBytes(producer); 51 | 52 | size_t const uncompressedOffset = FUZZ_getRange_from_uint32(uncompressedOffsetSeed, 0, size); 53 | size_t const uncompressedEndOffset = FUZZ_getRange_from_uint32(uncompressedEndOffsetSeed, uncompressedOffset, size); 54 | size_t const uncompressedSize = uncompressedEndOffset - uncompressedOffset; 55 | FUZZ_ASSERT(uncompressedOffset <= uncompressedEndOffset); 56 | FUZZ_ASSERT(uncompressedEndOffset <= size); 57 | 58 | const uint8_t *const uncompressedData = data + uncompressedOffset; 59 | 60 | size_t const dstCapacity = 61 | LZ4F_compressFrameBound(LZ4_compressBound(size), &prefs) + 62 | uncompressedSize; 63 | char *const dst = (char *) malloc(dstCapacity); 64 | size_t rtCapacity = dstCapacity; 65 | char *const rt = (char *) malloc(rtCapacity); 66 | 67 | FUZZ_ASSERT(dst); 68 | FUZZ_ASSERT(rt); 69 | 70 | /* Compression must succeed and round trip correctly. */ 71 | LZ4F_compressionContext_t ctx; 72 | size_t const ctxCreation = LZ4F_createCompressionContext(&ctx, LZ4F_VERSION); 73 | FUZZ_ASSERT(!LZ4F_isError(ctxCreation)); 74 | 75 | size_t const headerSize = LZ4F_compressBegin(ctx, dst, dstCapacity, &prefs); 76 | FUZZ_ASSERT(!LZ4F_isError(headerSize)); 77 | size_t compressedSize = headerSize; 78 | 79 | /* Compress data before uncompressed offset */ 80 | size_t lz4Return = LZ4F_compressUpdate(ctx, dst + compressedSize, dstCapacity, 81 | data, uncompressedOffset, NULL); 82 | FUZZ_ASSERT(!LZ4F_isError(lz4Return)); 83 | compressedSize += lz4Return; 84 | 85 | /* Add uncompressed data */ 86 | lz4Return = LZ4F_uncompressedUpdate(ctx, dst + compressedSize, dstCapacity, 87 | uncompressedData, uncompressedSize, NULL); 88 | FUZZ_ASSERT(!LZ4F_isError(lz4Return)); 89 | compressedSize += lz4Return; 90 | 91 | /* Compress data after uncompressed offset */ 92 | lz4Return = LZ4F_compressUpdate(ctx, dst + compressedSize, dstCapacity, 93 | data + uncompressedEndOffset, 94 | size - uncompressedEndOffset, NULL); 95 | FUZZ_ASSERT(!LZ4F_isError(lz4Return)); 96 | compressedSize += lz4Return; 97 | 98 | /* Finish compression */ 99 | lz4Return = LZ4F_compressEnd(ctx, dst + compressedSize, dstCapacity, NULL); 100 | FUZZ_ASSERT(!LZ4F_isError(lz4Return)); 101 | compressedSize += lz4Return; 102 | 103 | LZ4F_decompressOptions_t opts; 104 | memset(&opts, 0, sizeof(opts)); 105 | opts.stableDst = 1; 106 | LZ4F_dctx *dctx; 107 | LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION); 108 | FUZZ_ASSERT(dctx); 109 | 110 | decompress(dctx, dst, rt, rtCapacity, compressedSize); 111 | 112 | LZ4F_freeDecompressionContext(dctx); 113 | 114 | FUZZ_ASSERT_MSG(!memcmp(data, rt, size), "Corruption!"); 115 | 116 | free(dst); 117 | free(rt); 118 | 119 | FUZZ_dataProducer_free(producer); 120 | LZ4F_freeCompressionContext(ctx); 121 | } 122 | 123 | static void compress_independent_block_mode(const uint8_t *data, size_t size) { 124 | FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(data, size); 125 | LZ4F_preferences_t prefs = FUZZ_dataProducer_preferences(producer); 126 | prefs.frameInfo.blockMode = LZ4F_blockIndependent; 127 | compress_round_trip(data, size, producer, prefs); 128 | } 129 | 130 | 131 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { 132 | compress_independent_block_mode(data, size); 133 | return 0; 134 | } 135 | -------------------------------------------------------------------------------- /examples/blockStreaming_doubleBuffer.c: -------------------------------------------------------------------------------- 1 | // LZ4 streaming API example : double buffer 2 | // Copyright : Takayuki Matsuoka 3 | 4 | 5 | #if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ 6 | # define _CRT_SECURE_NO_WARNINGS 7 | # define snprintf sprintf_s 8 | #endif 9 | #include "lz4.h" 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | enum { 17 | BLOCK_BYTES = 1024 * 8, 18 | // BLOCK_BYTES = 1024 * 64, 19 | }; 20 | 21 | 22 | size_t write_int(FILE* fp, int i) { 23 | return fwrite(&i, sizeof(i), 1, fp); 24 | } 25 | 26 | size_t write_bin(FILE* fp, const void* array, size_t arrayBytes) { 27 | return fwrite(array, 1, arrayBytes, fp); 28 | } 29 | 30 | size_t read_int(FILE* fp, int* i) { 31 | return fread(i, sizeof(*i), 1, fp); 32 | } 33 | 34 | size_t read_bin(FILE* fp, void* array, size_t arrayBytes) { 35 | return fread(array, 1, arrayBytes, fp); 36 | } 37 | 38 | 39 | void test_compress(FILE* outFp, FILE* inpFp) 40 | { 41 | LZ4_stream_t lz4Stream_body; 42 | LZ4_stream_t* lz4Stream = &lz4Stream_body; 43 | 44 | char inpBuf[2][BLOCK_BYTES]; 45 | int inpBufIndex = 0; 46 | 47 | LZ4_initStream(lz4Stream, sizeof (*lz4Stream)); 48 | 49 | for(;;) { 50 | char* const inpPtr = inpBuf[inpBufIndex]; 51 | const int inpBytes = (int) read_bin(inpFp, inpPtr, BLOCK_BYTES); 52 | if(0 == inpBytes) { 53 | break; 54 | } 55 | 56 | { 57 | char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)]; 58 | const int cmpBytes = LZ4_compress_fast_continue( 59 | lz4Stream, inpPtr, cmpBuf, inpBytes, sizeof(cmpBuf), 1); 60 | if(cmpBytes <= 0) { 61 | break; 62 | } 63 | write_int(outFp, cmpBytes); 64 | write_bin(outFp, cmpBuf, (size_t) cmpBytes); 65 | } 66 | 67 | inpBufIndex = (inpBufIndex + 1) % 2; 68 | } 69 | 70 | write_int(outFp, 0); 71 | } 72 | 73 | 74 | void test_decompress(FILE* outFp, FILE* inpFp) 75 | { 76 | LZ4_streamDecode_t lz4StreamDecode_body; 77 | LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; 78 | 79 | char decBuf[2][BLOCK_BYTES]; 80 | int decBufIndex = 0; 81 | 82 | LZ4_setStreamDecode(lz4StreamDecode, NULL, 0); 83 | 84 | for(;;) { 85 | char cmpBuf[LZ4_COMPRESSBOUND(BLOCK_BYTES)]; 86 | int cmpBytes = 0; 87 | 88 | { 89 | const size_t readCount0 = read_int(inpFp, &cmpBytes); 90 | if(readCount0 != 1 || cmpBytes <= 0) { 91 | break; 92 | } 93 | 94 | const size_t readCount1 = read_bin(inpFp, cmpBuf, (size_t) cmpBytes); 95 | if(readCount1 != (size_t) cmpBytes) { 96 | break; 97 | } 98 | } 99 | 100 | { 101 | char* const decPtr = decBuf[decBufIndex]; 102 | const int decBytes = LZ4_decompress_safe_continue( 103 | lz4StreamDecode, cmpBuf, decPtr, cmpBytes, BLOCK_BYTES); 104 | if(decBytes <= 0) { 105 | break; 106 | } 107 | write_bin(outFp, decPtr, (size_t) decBytes); 108 | } 109 | 110 | decBufIndex = (decBufIndex + 1) % 2; 111 | } 112 | } 113 | 114 | 115 | int compare(FILE* fp0, FILE* fp1) 116 | { 117 | int result = 0; 118 | 119 | while(0 == result) { 120 | char b0[65536]; 121 | char b1[65536]; 122 | const size_t r0 = read_bin(fp0, b0, sizeof(b0)); 123 | const size_t r1 = read_bin(fp1, b1, sizeof(b1)); 124 | 125 | result = (int) r0 - (int) r1; 126 | 127 | if(0 == r0 || 0 == r1) { 128 | break; 129 | } 130 | if(0 == result) { 131 | result = memcmp(b0, b1, r0); 132 | } 133 | } 134 | 135 | return result; 136 | } 137 | 138 | 139 | int main(int argc, char* argv[]) 140 | { 141 | char inpFilename[256] = { 0 }; 142 | char lz4Filename[256] = { 0 }; 143 | char decFilename[256] = { 0 }; 144 | 145 | if(argc < 2) { 146 | printf("Please specify input filename\n"); 147 | return 0; 148 | } 149 | 150 | snprintf(inpFilename, 256, "%s", argv[1]); 151 | snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], BLOCK_BYTES); 152 | snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], BLOCK_BYTES); 153 | 154 | printf("inp = [%s]\n", inpFilename); 155 | printf("lz4 = [%s]\n", lz4Filename); 156 | printf("dec = [%s]\n", decFilename); 157 | 158 | // compress 159 | { 160 | FILE* inpFp = fopen(inpFilename, "rb"); 161 | FILE* outFp = fopen(lz4Filename, "wb"); 162 | 163 | printf("compress : %s -> %s\n", inpFilename, lz4Filename); 164 | test_compress(outFp, inpFp); 165 | printf("compress : done\n"); 166 | 167 | fclose(outFp); 168 | fclose(inpFp); 169 | } 170 | 171 | // decompress 172 | { 173 | FILE* inpFp = fopen(lz4Filename, "rb"); 174 | FILE* outFp = fopen(decFilename, "wb"); 175 | 176 | printf("decompress : %s -> %s\n", lz4Filename, decFilename); 177 | test_decompress(outFp, inpFp); 178 | printf("decompress : done\n"); 179 | 180 | fclose(outFp); 181 | fclose(inpFp); 182 | } 183 | 184 | // verify 185 | { 186 | FILE* inpFp = fopen(inpFilename, "rb"); 187 | FILE* decFp = fopen(decFilename, "rb"); 188 | 189 | printf("verify : %s <-> %s\n", inpFilename, decFilename); 190 | const int cmp = compare(inpFp, decFp); 191 | if(0 == cmp) { 192 | printf("verify : OK\n"); 193 | } else { 194 | printf("verify : NG\n"); 195 | } 196 | 197 | fclose(decFp); 198 | fclose(inpFp); 199 | } 200 | 201 | return 0; 202 | } 203 | -------------------------------------------------------------------------------- /examples/blockStreaming_ringBuffer.c: -------------------------------------------------------------------------------- 1 | /* LZ4 streaming API example : ring buffer 2 | * Based on sample code from Takayuki Matsuoka */ 3 | 4 | 5 | /************************************** 6 | * Compiler Options 7 | **************************************/ 8 | #if defined(_MSC_VER) && (_MSC_VER <= 1800) /* Visual Studio <= 2013 */ 9 | # define _CRT_SECURE_NO_WARNINGS 10 | # define snprintf sprintf_s 11 | #endif 12 | 13 | 14 | /************************************** 15 | * Includes 16 | **************************************/ 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "lz4.h" 22 | 23 | 24 | enum { 25 | MESSAGE_MAX_BYTES = 1024, 26 | RING_BUFFER_BYTES = 1024 * 8 + MESSAGE_MAX_BYTES, 27 | DECODE_RING_BUFFER = RING_BUFFER_BYTES + MESSAGE_MAX_BYTES /* Intentionally larger, to test unsynchronized ring buffers */ 28 | }; 29 | 30 | 31 | size_t write_int32(FILE* fp, int32_t i) { 32 | return fwrite(&i, sizeof(i), 1, fp); 33 | } 34 | 35 | size_t write_bin(FILE* fp, const void* array, int arrayBytes) { 36 | return fwrite(array, 1, arrayBytes, fp); 37 | } 38 | 39 | size_t read_int32(FILE* fp, int32_t* i) { 40 | return fread(i, sizeof(*i), 1, fp); 41 | } 42 | 43 | size_t read_bin(FILE* fp, void* array, int arrayBytes) { 44 | return fread(array, 1, arrayBytes, fp); 45 | } 46 | 47 | 48 | void test_compress(FILE* outFp, FILE* inpFp) 49 | { 50 | LZ4_stream_t lz4Stream_body = { { 0 } }; 51 | LZ4_stream_t* lz4Stream = &lz4Stream_body; 52 | 53 | static char inpBuf[RING_BUFFER_BYTES]; 54 | int inpOffset = 0; 55 | 56 | for(;;) { 57 | // Read random length ([1,MESSAGE_MAX_BYTES]) data to the ring buffer. 58 | char* const inpPtr = &inpBuf[inpOffset]; 59 | const int randomLength = (rand() % MESSAGE_MAX_BYTES) + 1; 60 | const int inpBytes = (int) read_bin(inpFp, inpPtr, randomLength); 61 | if (0 == inpBytes) break; 62 | 63 | { 64 | #define CMPBUFSIZE (LZ4_COMPRESSBOUND(MESSAGE_MAX_BYTES)) 65 | char cmpBuf[CMPBUFSIZE]; 66 | const int cmpBytes = LZ4_compress_fast_continue(lz4Stream, inpPtr, cmpBuf, inpBytes, CMPBUFSIZE, 0); 67 | if(cmpBytes <= 0) break; 68 | write_int32(outFp, cmpBytes); 69 | write_bin(outFp, cmpBuf, cmpBytes); 70 | 71 | inpOffset += inpBytes; 72 | 73 | // Wraparound the ringbuffer offset 74 | if(inpOffset >= RING_BUFFER_BYTES - MESSAGE_MAX_BYTES) inpOffset = 0; 75 | } 76 | } 77 | 78 | write_int32(outFp, 0); 79 | } 80 | 81 | 82 | void test_decompress(FILE* outFp, FILE* inpFp) 83 | { 84 | static char decBuf[DECODE_RING_BUFFER]; 85 | int decOffset = 0; 86 | LZ4_streamDecode_t lz4StreamDecode_body = { { 0 } }; 87 | LZ4_streamDecode_t* lz4StreamDecode = &lz4StreamDecode_body; 88 | 89 | for(;;) { 90 | int cmpBytes = 0; 91 | char cmpBuf[CMPBUFSIZE]; 92 | 93 | { const size_t r0 = read_int32(inpFp, &cmpBytes); 94 | if(r0 != 1 || cmpBytes <= 0) break; 95 | 96 | const size_t r1 = read_bin(inpFp, cmpBuf, cmpBytes); 97 | if(r1 != (size_t) cmpBytes) break; 98 | } 99 | 100 | { char* const decPtr = &decBuf[decOffset]; 101 | const int decBytes = LZ4_decompress_safe_continue( 102 | lz4StreamDecode, cmpBuf, decPtr, cmpBytes, MESSAGE_MAX_BYTES); 103 | if(decBytes <= 0) break; 104 | decOffset += decBytes; 105 | write_bin(outFp, decPtr, decBytes); 106 | 107 | // Wraparound the ringbuffer offset 108 | if(decOffset >= DECODE_RING_BUFFER - MESSAGE_MAX_BYTES) decOffset = 0; 109 | } 110 | } 111 | } 112 | 113 | 114 | int compare(FILE* f0, FILE* f1) 115 | { 116 | int result = 0; 117 | 118 | while (0 == result) { 119 | char b0[65536]; 120 | char b1[65536]; 121 | const size_t r0 = fread(b0, 1, sizeof(b0), f0); 122 | const size_t r1 = fread(b1, 1, sizeof(b1), f1); 123 | 124 | result = (int) r0 - (int) r1; 125 | 126 | if (0 == r0 || 0 == r1) break; 127 | 128 | if (0 == result) result = memcmp(b0, b1, r0); 129 | } 130 | 131 | return result; 132 | } 133 | 134 | 135 | int main(int argc, char** argv) 136 | { 137 | char inpFilename[256] = { 0 }; 138 | char lz4Filename[256] = { 0 }; 139 | char decFilename[256] = { 0 }; 140 | 141 | if (argc < 2) { 142 | printf("Please specify input filename\n"); 143 | return 0; 144 | } 145 | 146 | snprintf(inpFilename, 256, "%s", argv[1]); 147 | snprintf(lz4Filename, 256, "%s.lz4s-%d", argv[1], 0); 148 | snprintf(decFilename, 256, "%s.lz4s-%d.dec", argv[1], 0); 149 | 150 | printf("inp = [%s]\n", inpFilename); 151 | printf("lz4 = [%s]\n", lz4Filename); 152 | printf("dec = [%s]\n", decFilename); 153 | 154 | // compress 155 | { FILE* const inpFp = fopen(inpFilename, "rb"); 156 | FILE* const outFp = fopen(lz4Filename, "wb"); 157 | 158 | test_compress(outFp, inpFp); 159 | 160 | fclose(outFp); 161 | fclose(inpFp); 162 | } 163 | 164 | // decompress 165 | { FILE* const inpFp = fopen(lz4Filename, "rb"); 166 | FILE* const outFp = fopen(decFilename, "wb"); 167 | 168 | test_decompress(outFp, inpFp); 169 | 170 | fclose(outFp); 171 | fclose(inpFp); 172 | } 173 | 174 | // verify 175 | { FILE* const inpFp = fopen(inpFilename, "rb"); 176 | FILE* const decFp = fopen(decFilename, "rb"); 177 | 178 | const int cmp = compare(inpFp, decFp); 179 | if (0 == cmp) { 180 | printf("Verify : OK\n"); 181 | } else { 182 | printf("Verify : NG\n"); 183 | } 184 | 185 | fclose(decFp); 186 | fclose(inpFp); 187 | } 188 | 189 | return 0; 190 | } 191 | -------------------------------------------------------------------------------- /tests/datagencli.c: -------------------------------------------------------------------------------- 1 | /* 2 | datagencli.c 3 | compressible data command line generator 4 | Copyright (C) Yann Collet 2012-2020 5 | 6 | GPL v2 License 7 | 8 | This program is free software; you can redistribute it and/or modify 9 | it under the terms of the GNU General Public License as published by 10 | the Free Software Foundation; either version 2 of the License, or 11 | (at your option) any later version. 12 | 13 | This program is distributed in the hope that it will be useful, 14 | but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | GNU General Public License for more details. 17 | 18 | You should have received a copy of the GNU General Public License along 19 | with this program; if not, write to the Free Software Foundation, Inc., 20 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 21 | 22 | You can contact the author at : 23 | - LZ4 source repository : https://github.com/lz4/lz4 24 | - Public forum : https://groups.google.com/forum/#!forum/lz4c 25 | */ 26 | 27 | /************************************** 28 | * Includes 29 | **************************************/ 30 | #include "util.h" /* U32 */ 31 | #include /* fprintf, stderr */ 32 | #include "datagen.h" /* RDG_generate */ 33 | #include "lz4.h" /* LZ4_VERSION_STRING */ 34 | 35 | 36 | /************************************** 37 | * Compiler specific 38 | **************************************/ 39 | #ifdef _MSC_VER /* Visual Studio */ 40 | #define strtoull _strtoui64 /* https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/strtoui64-wcstoui64-strtoui64-l-wcstoui64-l */ 41 | #endif 42 | 43 | 44 | /************************************** 45 | * Constants 46 | **************************************/ 47 | #define KB *(1 <<10) 48 | #define MB *(1 <<20) 49 | #define GB *(1U<<30) 50 | 51 | #define SIZE_DEFAULT (64 KB) 52 | #define SEED_DEFAULT 0 53 | #define COMPRESSIBILITY_DEFAULT 50 54 | 55 | 56 | /************************************** 57 | * Macros 58 | **************************************/ 59 | #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) 60 | #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } 61 | static unsigned displayLevel = 2; 62 | 63 | 64 | /********************************************************* 65 | * Command line 66 | *********************************************************/ 67 | static int usage(char* programName) 68 | { 69 | DISPLAY( "Compressible data generator\n"); 70 | DISPLAY( "Usage :\n"); 71 | DISPLAY( " %s [size] [args]\n", programName); 72 | DISPLAY( "\n"); 73 | DISPLAY( "Arguments :\n"); 74 | DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); 75 | DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); 76 | DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", COMPRESSIBILITY_DEFAULT); 77 | DISPLAY( " -h : display help and exit\n"); 78 | DISPLAY( "Special values :\n"); 79 | DISPLAY( " -P0 : generate incompressible noise\n"); 80 | DISPLAY( " -P100 : generate sparse files\n"); 81 | return 0; 82 | } 83 | 84 | 85 | int main(int argc, char** argv) 86 | { 87 | int argNb; 88 | double proba = (double)COMPRESSIBILITY_DEFAULT / 100; 89 | double litProba = 0.0; 90 | U64 size = SIZE_DEFAULT; 91 | U32 seed = SEED_DEFAULT; 92 | char* programName; 93 | 94 | /* Check command line */ 95 | programName = argv[0]; 96 | for(argNb=1; argNb