├── .buckconfig ├── .buckversion ├── .cirrus.yml ├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── dependabot.yml └── workflows │ ├── android-ndk-build.yml │ ├── cmake-tests.yml │ ├── commit.yml │ ├── dev-long-tests.yml │ ├── dev-short-tests.yml │ ├── nightly.yml │ ├── publish-release-artifacts.yml │ ├── release_check.yml │ ├── scorecards.yml │ └── windows-artifacts.yml ├── .gitignore ├── CHANGELOG ├── CMakeLists.txt ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── COPYING ├── LICENSE ├── Makefile ├── Package.swift ├── README.md ├── SECURITY.md ├── TESTING.md ├── build ├── .gitignore ├── LICENSE ├── README.md ├── VS2008 │ ├── fullbench │ │ └── fullbench.vcproj │ ├── fuzzer │ │ └── fuzzer.vcproj │ ├── zstd.sln │ ├── zstd │ │ └── zstd.vcproj │ └── zstdlib │ │ └── zstdlib.vcproj ├── VS2010 │ ├── CompileAsCpp.props │ ├── datagen │ │ └── datagen.vcxproj │ ├── fullbench │ │ └── fullbench.vcxproj │ ├── fuzzer │ │ └── fuzzer.vcxproj │ ├── libzstd-dll │ │ ├── libzstd-dll.rc │ │ └── libzstd-dll.vcxproj │ ├── libzstd │ │ └── libzstd.vcxproj │ ├── zstd.sln │ └── zstd │ │ ├── zstd.rc │ │ └── zstd.vcxproj ├── VS_scripts │ ├── README.md │ ├── build.VS2010.cmd │ ├── build.VS2012.cmd │ ├── build.VS2013.cmd │ ├── build.VS2015.cmd │ ├── build.VS2017.cmd │ ├── build.VS2017Community.cmd │ ├── build.VS2017Enterprise.cmd │ ├── build.VS2017Professional.cmd │ ├── build.VSPreview.cmd │ └── build.generic.cmd ├── cmake │ ├── .gitignore │ ├── CMakeLists.txt │ ├── CMakeModules │ │ ├── AddZstdCompilationFlags.cmake │ │ ├── FindLibLZ4.cmake │ │ ├── GetZstdLibraryVersion.cmake │ │ ├── JoinPaths.cmake │ │ ├── ZstdBuild.cmake │ │ ├── ZstdDependencies.cmake │ │ ├── ZstdOptions.cmake │ │ ├── ZstdPackage.cmake │ │ └── ZstdVersion.cmake │ ├── README.md │ ├── contrib │ │ ├── CMakeLists.txt │ │ ├── gen_html │ │ │ └── CMakeLists.txt │ │ └── pzstd │ │ │ └── CMakeLists.txt │ ├── lib │ │ ├── .gitignore │ │ ├── CMakeLists.txt │ │ └── cmake_uninstall.cmake.in │ ├── programs │ │ ├── .gitignore │ │ └── CMakeLists.txt │ ├── tests │ │ ├── .gitignore │ │ └── CMakeLists.txt │ └── zstdConfig.cmake.in ├── meson │ ├── GetZstdLibraryVersion.py │ ├── InstallSymlink.py │ ├── README.md │ ├── contrib │ │ ├── gen_html │ │ │ └── meson.build │ │ ├── meson.build │ │ └── pzstd │ │ │ └── meson.build │ ├── lib │ │ └── meson.build │ ├── meson.build │ ├── meson_options.txt │ ├── programs │ │ └── meson.build │ └── tests │ │ ├── meson.build │ │ └── valgrindTest.py └── single_file_libs │ ├── .gitignore │ ├── README.md │ ├── build_decoder_test.sh │ ├── build_library_test.sh │ ├── combine.py │ ├── combine.sh │ ├── create_single_file_decoder.sh │ ├── create_single_file_library.sh │ ├── examples │ ├── README.md │ ├── emscripten.c │ ├── roundtrip.c │ ├── shell.html │ ├── simple.c │ ├── testcard-dxt1.inl │ ├── testcard-zstd.inl │ └── testcard.png │ ├── zstd-in.c │ └── zstddeclib-in.c ├── contrib ├── VS2005 │ ├── README.md │ ├── fullbench │ │ └── fullbench.vcproj │ ├── fuzzer │ │ └── fuzzer.vcproj │ ├── zstd.sln │ ├── zstd │ │ └── zstd.vcproj │ └── zstdlib │ │ └── zstdlib.vcproj ├── cleanTabs ├── diagnose_corruption │ ├── .gitignore │ ├── Makefile │ └── check_flipped_bits.c ├── docker │ ├── Dockerfile │ └── README.md ├── externalSequenceProducer │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── main.c │ ├── sequence_producer.c │ └── sequence_producer.h ├── freestanding_lib │ └── freestanding.py ├── gen_html │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── gen-zstd-manual.sh │ └── gen_html.cpp ├── largeNbDicts │ ├── .gitignore │ ├── Makefile │ ├── README.md │ └── largeNbDicts.c ├── linux-kernel │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── btrfs-benchmark.sh │ ├── btrfs-extract-benchmark.sh │ ├── decompress_sources.h │ ├── linux.mk │ ├── linux_zstd.h │ ├── mem.h │ ├── squashfs-benchmark.sh │ ├── test │ │ ├── Makefile │ │ ├── include │ │ │ └── linux │ │ │ │ ├── compiler.h │ │ │ │ ├── errno.h │ │ │ │ ├── kernel.h │ │ │ │ ├── limits.h │ │ │ │ ├── math64.h │ │ │ │ ├── module.h │ │ │ │ ├── printk.h │ │ │ │ ├── stddef.h │ │ │ │ ├── swab.h │ │ │ │ ├── types.h │ │ │ │ ├── unaligned.h │ │ │ │ └── xxhash.h │ │ ├── macro-test.sh │ │ ├── static_test.c │ │ └── test.c │ ├── zstd_common_module.c │ ├── zstd_compress_module.c │ ├── zstd_decompress_module.c │ └── zstd_deps.h ├── match_finders │ ├── README.md │ ├── zstd_edist.c │ └── zstd_edist.h ├── premake │ ├── premake4.lua │ └── zstd.lua ├── pzstd │ ├── .gitignore │ ├── BUCK │ ├── ErrorHolder.h │ ├── Logging.h │ ├── Makefile │ ├── Options.cpp │ ├── Options.h │ ├── Pzstd.cpp │ ├── Pzstd.h │ ├── README.md │ ├── SkippableFrame.cpp │ ├── SkippableFrame.h │ ├── images │ │ ├── Cspeed.png │ │ └── Dspeed.png │ ├── main.cpp │ ├── test │ │ ├── BUCK │ │ ├── OptionsTest.cpp │ │ ├── PzstdTest.cpp │ │ ├── RoundTrip.h │ │ └── RoundTripTest.cpp │ └── utils │ │ ├── BUCK │ │ ├── Buffer.h │ │ ├── FileSystem.h │ │ ├── Likely.h │ │ ├── Portability.h │ │ ├── Range.h │ │ ├── ResourcePool.h │ │ ├── ScopeGuard.h │ │ ├── ThreadPool.h │ │ ├── WorkQueue.h │ │ └── test │ │ ├── BUCK │ │ ├── BufferTest.cpp │ │ ├── RangeTest.cpp │ │ ├── ResourcePoolTest.cpp │ │ ├── ScopeGuardTest.cpp │ │ ├── ThreadPoolTest.cpp │ │ └── WorkQueueTest.cpp ├── recovery │ ├── Makefile │ └── recover_directory.c ├── seekable_format │ ├── README.md │ ├── examples │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── parallel_compression.c │ │ ├── parallel_processing.c │ │ ├── seekable_compression.c │ │ ├── seekable_decompression.c │ │ └── seekable_decompression_mem.c │ ├── tests │ │ ├── .gitignore │ │ ├── Makefile │ │ └── seekable_tests.c │ ├── zstd_seekable.h │ ├── zstd_seekable_compression_format.md │ ├── zstdseek_compress.c │ └── zstdseek_decompress.c ├── seqBench │ ├── Makefile │ └── seqBench.c └── snap │ └── snapcraft.yaml ├── doc ├── README.md ├── decompressor_errata.md ├── decompressor_permissive.md ├── educational_decoder │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── harness.c │ ├── zstd_decompress.c │ └── zstd_decompress.h ├── images │ ├── CSpeed2.png │ ├── DCspeed5.png │ ├── DSpeed3.png │ ├── cdict_v136.png │ ├── dict-cr.png │ ├── dict-cs.png │ ├── dict-ds.png │ ├── zstd_cdict_v1_3_5.png │ └── zstd_logo86.png ├── zstd_compression_format.md └── zstd_manual.html ├── examples ├── .gitignore ├── Makefile ├── README.md ├── common.h ├── dictionary_compression.c ├── dictionary_decompression.c ├── multiple_simple_compression.c ├── multiple_streaming_compression.c ├── simple_compression.c ├── simple_decompression.c ├── streaming_compression.c ├── streaming_compression_thread_pool.c ├── streaming_decompression.c └── streaming_memory_usage.c ├── lib ├── .gitignore ├── BUCK ├── Makefile ├── README.md ├── common │ ├── allocations.h │ ├── bits.h │ ├── bitstream.h │ ├── compiler.h │ ├── cpu.h │ ├── debug.c │ ├── debug.h │ ├── entropy_common.c │ ├── error_private.c │ ├── error_private.h │ ├── fse.h │ ├── fse_decompress.c │ ├── huf.h │ ├── mem.h │ ├── pool.c │ ├── pool.h │ ├── portability_macros.h │ ├── threading.c │ ├── threading.h │ ├── xxhash.c │ ├── xxhash.h │ ├── zstd_common.c │ ├── zstd_deps.h │ ├── zstd_internal.h │ └── zstd_trace.h ├── compress │ ├── clevels.h │ ├── fse_compress.c │ ├── hist.c │ ├── hist.h │ ├── huf_compress.c │ ├── zstd_compress.c │ ├── zstd_compress_internal.h │ ├── zstd_compress_literals.c │ ├── zstd_compress_literals.h │ ├── zstd_compress_sequences.c │ ├── zstd_compress_sequences.h │ ├── zstd_compress_superblock.c │ ├── zstd_compress_superblock.h │ ├── zstd_cwksp.h │ ├── zstd_double_fast.c │ ├── zstd_double_fast.h │ ├── zstd_fast.c │ ├── zstd_fast.h │ ├── zstd_lazy.c │ ├── zstd_lazy.h │ ├── zstd_ldm.c │ ├── zstd_ldm.h │ ├── zstd_ldm_geartab.h │ ├── zstd_opt.c │ ├── zstd_opt.h │ ├── zstd_preSplit.c │ ├── zstd_preSplit.h │ ├── zstdmt_compress.c │ └── zstdmt_compress.h ├── decompress │ ├── huf_decompress.c │ ├── huf_decompress_amd64.S │ ├── zstd_ddict.c │ ├── zstd_ddict.h │ ├── zstd_decompress.c │ ├── zstd_decompress_block.c │ ├── zstd_decompress_block.h │ └── zstd_decompress_internal.h ├── deprecated │ ├── zbuff.h │ ├── zbuff_common.c │ ├── zbuff_compress.c │ └── zbuff_decompress.c ├── dictBuilder │ ├── cover.c │ ├── cover.h │ ├── divsufsort.c │ ├── divsufsort.h │ ├── fastcover.c │ └── zdict.c ├── dll │ └── example │ │ ├── Makefile │ │ ├── README.md │ │ ├── build_package.bat │ │ ├── fullbench-dll.sln │ │ └── fullbench-dll.vcxproj ├── install_oses.mk ├── legacy │ ├── zstd_legacy.h │ ├── zstd_v01.c │ ├── zstd_v01.h │ ├── zstd_v02.c │ ├── zstd_v02.h │ ├── zstd_v03.c │ ├── zstd_v03.h │ ├── zstd_v04.c │ ├── zstd_v04.h │ ├── zstd_v05.c │ ├── zstd_v05.h │ ├── zstd_v06.c │ ├── zstd_v06.h │ ├── zstd_v07.c │ └── zstd_v07.h ├── libzstd.mk ├── libzstd.pc.in ├── module.modulemap ├── zdict.h ├── zstd.h └── zstd_errors.h ├── programs ├── .gitignore ├── BUCK ├── Makefile ├── README.md ├── benchfn.c ├── benchfn.h ├── benchzstd.c ├── benchzstd.h ├── datagen.c ├── datagen.h ├── dibio.c ├── dibio.h ├── fileio.c ├── fileio.h ├── fileio_asyncio.c ├── fileio_asyncio.h ├── fileio_common.h ├── fileio_types.h ├── lorem.c ├── lorem.h ├── platform.h ├── timefn.c ├── timefn.h ├── util.c ├── util.h ├── windres │ ├── verrsrc.h │ ├── zstd.rc │ ├── zstd32.res │ └── zstd64.res ├── zstd.1 ├── zstd.1.md ├── zstdcli.c ├── zstdcli_trace.c ├── zstdcli_trace.h ├── zstdgrep ├── zstdgrep.1 ├── zstdgrep.1.md ├── zstdless ├── zstdless.1 └── zstdless.1.md ├── tests ├── .gitignore ├── DEPRECATED-test-zstd-speed.py ├── Makefile ├── README.md ├── automated_benchmarking.py ├── checkTag.c ├── check_size.py ├── cli-tests │ ├── .gitignore │ ├── README.md │ ├── basic │ │ ├── args.sh │ │ ├── args.sh.exit │ │ ├── args.sh.stderr.glob │ │ ├── help.sh │ │ ├── help.sh.stdout.glob │ │ ├── memlimit.sh │ │ ├── memlimit.sh.stderr.exact │ │ ├── memlimit.sh.stdout.exact │ │ ├── output_dir.sh │ │ ├── output_dir.sh.stderr.exact │ │ ├── output_dir.sh.stdout.exact │ │ ├── version.sh │ │ └── version.sh.stdout.glob │ ├── bin │ │ ├── cmp_size │ │ ├── datagen │ │ ├── die │ │ ├── println │ │ ├── unzstd │ │ ├── zstd │ │ ├── zstdcat │ │ ├── zstdgrep │ │ └── zstdless │ ├── cltools │ │ ├── setup │ │ ├── zstdgrep.sh │ │ ├── zstdgrep.sh.exit │ │ ├── zstdgrep.sh.stderr.exact │ │ ├── zstdgrep.sh.stdout.glob │ │ ├── zstdless.sh │ │ ├── zstdless.sh.stderr.exact │ │ └── zstdless.sh.stdout.glob │ ├── common │ │ ├── format.sh │ │ ├── mtime.sh │ │ ├── permissions.sh │ │ └── platform.sh │ ├── compression │ │ ├── adapt.sh │ │ ├── basic.sh │ │ ├── compress-literals.sh │ │ ├── format.sh │ │ ├── golden.sh │ │ ├── gzip-compat.sh │ │ ├── levels.sh │ │ ├── levels.sh.stderr.exact │ │ ├── long-distance-matcher.sh │ │ ├── multi-threaded.sh │ │ ├── multi-threaded.sh.stderr.exact │ │ ├── multiple-files.sh │ │ ├── multiple-files.sh.stdout.exact │ │ ├── row-match-finder.sh │ │ ├── setup │ │ ├── stream-size.sh │ │ ├── verbose-wlog.sh │ │ ├── verbose-wlog.sh.stderr.glob │ │ ├── verbose-wlog.sh.stdout.glob │ │ ├── window-resize.sh │ │ ├── window-resize.sh.stderr.ignore │ │ └── window-resize.sh.stdout.glob │ ├── decompression │ │ ├── detectErrors.sh │ │ ├── golden.sh │ │ ├── pass-through.sh │ │ ├── pass-through.sh.stderr.exact │ │ └── pass-through.sh.stdout.exact │ ├── determinism │ │ ├── basic.sh │ │ ├── basic.sh.stderr.exact │ │ ├── basic.sh.stdout.exact │ │ ├── multithread.sh │ │ ├── multithread.sh.stderr.exact │ │ ├── multithread.sh.stdout.exact │ │ ├── reuse.sh │ │ ├── reuse.sh.stderr.exact │ │ ├── reuse.sh.stdout.exact │ │ ├── setup │ │ └── setup_once │ ├── dict-builder │ │ ├── empty-input.sh │ │ ├── empty-input.sh.stderr.exact │ │ ├── no-inputs.sh │ │ ├── no-inputs.sh.exit │ │ └── no-inputs.sh.stderr.exact │ ├── dictionaries │ │ ├── dictionary-mismatch.sh │ │ ├── dictionary-mismatch.sh.stderr.exact │ │ ├── golden.sh │ │ ├── setup │ │ └── setup_once │ ├── file-handling │ │ ├── directory-mirror.sh │ │ ├── directory-mirror.sh.stderr.exact │ │ └── directory-mirror.sh.stdout.exact │ ├── file-stat │ │ ├── compress-file-to-dir-without-write-perm.sh │ │ ├── compress-file-to-dir-without-write-perm.sh.stderr.exact │ │ ├── compress-file-to-file.sh │ │ ├── compress-file-to-file.sh.stderr.glob │ │ ├── compress-file-to-stdout.sh │ │ ├── compress-file-to-stdout.sh.stderr.exact │ │ ├── compress-stdin-to-file.sh │ │ ├── compress-stdin-to-file.sh.stderr.glob │ │ ├── compress-stdin-to-stdout.sh │ │ ├── compress-stdin-to-stdout.sh.stderr.exact │ │ ├── decompress-file-to-file.sh │ │ ├── decompress-file-to-file.sh.stderr.glob │ │ ├── decompress-file-to-stdout.sh │ │ ├── decompress-file-to-stdout.sh.stderr.exact │ │ ├── decompress-stdin-to-file.sh │ │ ├── decompress-stdin-to-file.sh.stderr.glob │ │ ├── decompress-stdin-to-stdout.sh │ │ └── decompress-stdin-to-stdout.sh.stderr.exact │ ├── progress │ │ ├── no-progress.sh │ │ ├── no-progress.sh.stderr.glob │ │ ├── progress.sh │ │ └── progress.sh.stderr.glob │ ├── run.py │ └── zstd-symlinks │ │ ├── setup │ │ ├── zstdcat.sh │ │ └── zstdcat.sh.stdout.exact ├── datagencli.c ├── decodecorpus.c ├── dict-files │ └── zero-weight-dict ├── external_matchfinder.c ├── external_matchfinder.h ├── fullbench.c ├── fuzz │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── block_decompress.c │ ├── block_round_trip.c │ ├── decompress_cross_format.c │ ├── decompress_dstSize_tooSmall.c │ ├── dictionary_decompress.c │ ├── dictionary_loader.c │ ├── dictionary_round_trip.c │ ├── dictionary_stream_round_trip.c │ ├── fse_read_ncount.c │ ├── fuzz.h │ ├── fuzz.py │ ├── fuzz_data_producer.c │ ├── fuzz_data_producer.h │ ├── fuzz_helpers.c │ ├── fuzz_helpers.h │ ├── fuzz_third_party_seq_prod.h │ ├── generate_sequences.c │ ├── huf_decompress.c │ ├── huf_round_trip.c │ ├── raw_dictionary_round_trip.c │ ├── regression_driver.c │ ├── seekable_roundtrip.c │ ├── seq_prod_fuzz_example │ │ ├── Makefile │ │ ├── README.md │ │ └── example_seq_prod.c │ ├── sequence_compression_api.c │ ├── simple_compress.c │ ├── simple_decompress.c │ ├── simple_round_trip.c │ ├── stream_decompress.c │ ├── stream_round_trip.c │ ├── zstd_frame_info.c │ ├── zstd_helpers.c │ └── zstd_helpers.h ├── fuzzer.c ├── golden-compression │ ├── PR-3517-block-splitter-corruption-test │ ├── http │ ├── huffman-compressed-larger │ └── large-literal-and-match-lengths ├── golden-decompression-errors │ ├── .gitignore │ ├── off0.bin.zst │ ├── truncated_huff_state.zst │ └── zeroSeq_extraneous.zst ├── golden-decompression │ ├── block-128k.zst │ ├── empty-block.zst │ ├── rle-first-block.zst │ └── zeroSeq_2B.zst ├── golden-dictionaries │ └── http-dict-missing-symbols ├── gzip │ ├── Makefile │ ├── gzip-env.sh │ ├── helin-segv.sh │ ├── help-version.sh │ ├── hufts-segv.gz │ ├── hufts.sh │ ├── init.cfg │ ├── init.sh │ ├── keep.sh │ ├── list.sh │ ├── memcpy-abuse.sh │ ├── mixed.sh │ ├── null-suffix-clobber.sh │ ├── stdin.sh │ ├── test-driver.sh │ ├── trailing-nul.sh │ ├── unpack-invalid.sh │ ├── z-suffix.sh │ ├── zdiff.sh │ ├── zgrep-context.sh │ ├── zgrep-f.sh │ ├── zgrep-signal.sh │ └── znew-k.sh ├── invalidDictionaries.c ├── largeDictionary.c ├── legacy.c ├── libzstd_builds.sh ├── longmatch.c ├── loremOut.c ├── loremOut.h ├── paramgrill.c ├── playTests.sh ├── poolTests.c ├── rateLimiter.py ├── regression │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── config.c │ ├── config.h │ ├── data.c │ ├── data.h │ ├── levels.h │ ├── method.c │ ├── method.h │ ├── result.c │ ├── result.h │ ├── results.csv │ └── test.c ├── roundTripCrash.c ├── seqgen.c ├── seqgen.h ├── test-license.py ├── test-variants.sh ├── test-zstd-versions.py ├── test_process_substitution.bash └── zstreamtest.c └── zlibWrapper ├── .gitignore ├── BUCK ├── Makefile ├── README.md ├── examples ├── example.c ├── example_original.c ├── fitblk.c ├── fitblk_original.c ├── minigzip.c └── zwrapbench.c ├── gzclose.c ├── gzcompatibility.h ├── gzguts.h ├── gzlib.c ├── gzread.c ├── gzwrite.c ├── zstd_zlibwrapper.c └── zstd_zlibwrapper.h /.buckconfig: -------------------------------------------------------------------------------- 1 | [cxx] 2 | cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4 3 | cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith 4 | cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4 5 | cxxflags = -std=c++11 -Wno-deprecated-declarations 6 | gtest_dep = //contrib/pzstd:gtest 7 | 8 | [httpserver] 9 | port = 0 10 | -------------------------------------------------------------------------------- /.buckversion: -------------------------------------------------------------------------------- 1 | c8dec2e8da52d483f6dd7c6cd2ad694e8e6fed2b 2 | -------------------------------------------------------------------------------- /.cirrus.yml: -------------------------------------------------------------------------------- 1 | task: 2 | name: FreeBSD (make check) 3 | freebsd_instance: 4 | matrix: 5 | image_family: freebsd-14-2 6 | install_script: pkg install -y gmake coreutils 7 | script: | 8 | MOREFLAGS="-Werror" gmake -j all 9 | gmake check 10 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set the default behavior 2 | * text eol=lf 3 | 4 | # Explicitly declare source files 5 | *.c text eol=lf 6 | *.h text eol=lf 7 | 8 | # Denote files that should not be modified. 9 | *.odt binary 10 | *.png binary 11 | 12 | # Visual Studio 13 | *.sln text eol=crlf 14 | *.vcxproj* text eol=crlf 15 | *.vcproj* text eol=crlf 16 | *.suo binary 17 | *.rc text eol=crlf 18 | 19 | # Windows 20 | *.bat text eol=crlf 21 | *.cmd text eol=crlf 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Downloads data '...' 16 | 2. Run '...' with flags '...' 17 | 3. Scroll up on the log to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots and charts** 24 | If applicable, add screenshots and charts to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. Mac] 28 | - Version [e.g. 22] 29 | - Compiler [e.g. gcc] 30 | - Flags [e.g. O2] 31 | - Other relevant hardware specs [e.g. Dual-core] 32 | - Build system [e.g. Makefile] 33 | 34 | **Additional context** 35 | Add any other context about the problem here. 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /.github/workflows/android-ndk-build.yml: -------------------------------------------------------------------------------- 1 | name: Android NDK Build 2 | 3 | on: 4 | pull_request: 5 | branches: [ dev, release, actionsTest ] 6 | push: 7 | branches: [ actionsTest, '*ndk*' ] 8 | 9 | permissions: read-all 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag=v5.0.0 17 | 18 | - name: Set up JDK 17 19 | uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 20 | with: 21 | java-version: '17' 22 | distribution: 'temurin' 23 | 24 | - name: Setup Android SDK 25 | uses: android-actions/setup-android@9fc6c4e9069bf8d3d10b2204b1fb8f6ef7065407 # v3.2.2 26 | 27 | - name: Install Android NDK 28 | run: | 29 | sdkmanager --install "ndk;27.0.12077973" 30 | echo "ANDROID_NDK_HOME=$ANDROID_SDK_ROOT/ndk/27.0.12077973" >> $GITHUB_ENV 31 | 32 | - name: Build with NDK 33 | run: | 34 | export PATH=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH 35 | make CC=aarch64-linux-android21-clang \ 36 | AR=llvm-ar \ 37 | RANLIB=llvm-ranlib \ 38 | STRIP=llvm-strip 39 | 40 | - name: Build with CMake and NDK 41 | run: | 42 | mkdir -p build-android 43 | cd build-android 44 | cmake --version 45 | cmake ../build/cmake \ 46 | -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake \ 47 | -DANDROID_ABI=arm64-v8a \ 48 | -DANDROID_PLATFORM=android-21 \ 49 | -DCMAKE_BUILD_TYPE=Release 50 | cmake --build . --parallel 51 | -------------------------------------------------------------------------------- /.github/workflows/nightly.yml: -------------------------------------------------------------------------------- 1 | name: facebook/zstd/nightly 2 | on: 3 | schedule: 4 | - cron: '0 0 * * *' 5 | push: 6 | branches: 7 | - release 8 | - dev 9 | - '*nightly*' 10 | permissions: read-all 11 | jobs: 12 | regression-test: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v5 16 | - name: Install dependencies 17 | run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev 18 | - name: Regression Test 19 | run: | 20 | make -C programs zstd 21 | make -C tests/regression test 22 | 23 | # Longer tests 24 | #- make -C tests test-zstd-nolegacy && make clean 25 | #- pyenv global 3.4.4; make -C tests versionsTest && make clean 26 | #- make zlibwrapper && make clean 27 | #- gcc -v; make -C tests test32 MOREFLAGS="-I/usr/include/x86_64-linux-gnu" && make clean 28 | #- make uasan && make clean 29 | #- make asan32 && make clean 30 | #- make -C tests test32 CC=clang MOREFLAGS="-g -fsanitize=address -I/usr/include/x86_64-linux-gnu" 31 | # Valgrind tests 32 | #- CFLAGS="-O1 -g" make -C zlibWrapper valgrindTest && make clean 33 | #- make -C tests valgrindTest && make clean 34 | # ARM, AArch64, PowerPC, PowerPC64 tests 35 | #- make ppctest && make clean 36 | #- make ppc64test && make clean 37 | #- make armtest && make clean 38 | #- make aarch64test && make clean 39 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | *.ko 4 | *.dSYM 5 | 6 | # Libraries 7 | *.lib 8 | *.a 9 | 10 | # Shared objects (inc. Windows DLLs) 11 | *.dll 12 | *.so 13 | *.so.* 14 | *.dylib 15 | *.framework 16 | *.xcframework 17 | 18 | # Executables 19 | /zstd 20 | zstdmt 21 | *.exe 22 | *.out 23 | *.app 24 | 25 | # Build artefacts 26 | contrib/linux-kernel/linux/ 27 | projects/ 28 | bin/ 29 | .buckd/ 30 | buck-out/ 31 | build-* 32 | *.gcda 33 | cmakebuild/ 34 | cmake-build/ 35 | 36 | # Test artefacts 37 | tmp* 38 | *.zst 39 | *.zstd 40 | dictionary. 41 | dictionary 42 | NUL 43 | install/ 44 | 45 | # IDE 46 | .clang_complete 47 | compile_flags.txt 48 | .clang-format 49 | 50 | # Other files 51 | .directory 52 | _codelite/ 53 | _zstdbench/ 54 | *.idea 55 | *.swp 56 | .DS_Store 57 | googletest/ 58 | *.d 59 | *.vscode 60 | *.code-workspace 61 | compile_commands.json 62 | .clangd 63 | perf.data 64 | perf.data.old 65 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | 3 | # Thin wrapper so `cmake -S .` behaves like `cmake -S build/cmake`. 4 | # Policy lives in build/cmake; keep parent project language-less. 5 | project(zstd-superbuild LANGUAGES NONE) 6 | 7 | if(CMAKE_SOURCE_DIR STREQUAL CMAKE_BINARY_DIR) 8 | message(FATAL_ERROR "In-source builds are not supported. Specify -B .") 9 | endif() 10 | 11 | add_subdirectory(build/cmake) 12 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | Facebook has adopted a Code of Conduct that we expect project participants to adhere to. 4 | Please read the [full text](https://code.fb.com/codeofconduct/) 5 | so that you can understand what actions will and will not be tolerated. 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD License 2 | 3 | For Zstandard software 4 | 5 | Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, 8 | are permitted provided that the following conditions are met: 9 | 10 | * Redistributions of source code must retain the above copyright notice, this 11 | list of conditions and the following disclaimer. 12 | 13 | * Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | 17 | * Neither the name Facebook, nor Meta, nor the names of its contributors may 18 | be used to endorse or promote products derived from this software without 19 | specific prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 25 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 26 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 28 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.0 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "zstd", 8 | platforms: [ 9 | .macOS(.v10_10), .iOS(.v9), .tvOS(.v9) 10 | ], 11 | products: [ 12 | // Products define the executables and libraries a package produces, and make them visible to other packages. 13 | .library( 14 | name: "libzstd", 15 | targets: [ "libzstd" ]) 16 | ], 17 | dependencies: [ 18 | // Dependencies declare other packages that this package depends on. 19 | // .package(url: /* package url */, from: "1.0.0"), 20 | ], 21 | targets: [ 22 | // Targets are the basic building blocks of a package. A target can define a module or a test suite. 23 | // Targets can depend on other targets in this package, and on products in packages this package depends on. 24 | .target( 25 | name: "libzstd", 26 | path: "lib", 27 | sources: [ "common", "compress", "decompress", "dictBuilder" ], 28 | publicHeadersPath: ".", 29 | cSettings: [ 30 | .headerSearchPath(".") 31 | ]) 32 | ], 33 | swiftLanguageVersions: [.v5], 34 | cLanguageStandard: .gnu11, 35 | cxxLanguageStandard: .gnucxx14 36 | ) 37 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting and Fixing Security Issues 2 | 3 | Please do not open GitHub issues or pull requests - this makes the problem immediately visible to everyone, including malicious actors. Security issues in this open source project can be safely reported via the Meta Bug Bounty program: 4 | 5 | https://www.facebook.com/whitehat 6 | 7 | Meta's security team will triage your report and determine whether or not is it eligible for a bounty under our program. 8 | 9 | # Receiving Vulnerability Notifications 10 | 11 | In the case that a significant security vulnerability is reported to us or discovered by us---without being publicly known---we will, at our discretion, notify high-profile, high-exposure users of Zstandard ahead of our public disclosure of the issue and associated fix. 12 | 13 | If you believe your project would benefit from inclusion in this list, please reach out to one of the maintainers. 14 | 15 | 16 | -------------------------------------------------------------------------------- /build/.gitignore: -------------------------------------------------------------------------------- 1 | # Visual C++ 2 | .vs/ 3 | *Copy 4 | *.db 5 | *.opensdf 6 | *.sdf 7 | *.suo 8 | *.user 9 | *.opendb 10 | 11 | VS2008/bin/ 12 | VS2010/bin/ 13 | VS2010/zwrapbench/ 14 | VS2012/ 15 | VS2013/ 16 | VS2015/ 17 | Studio* 18 | 19 | # CMake 20 | cmake/build/ 21 | CMakeCache.txt 22 | CMakeFiles 23 | CMakeScripts 24 | Testing 25 | Makefile 26 | cmake_install.cmake 27 | install_manifest.txt 28 | compile_commands.json 29 | CTestTestfile.cmake 30 | build 31 | lib 32 | !cmake/lib 33 | !meson/lib 34 | -------------------------------------------------------------------------------- /build/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/build/LICENSE -------------------------------------------------------------------------------- /build/VS2010/CompileAsCpp.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CompileAsCpp 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /build/VS2010/libzstd-dll/libzstd-dll.rc: -------------------------------------------------------------------------------- 1 | // Microsoft Visual C++ generated resource script. 2 | // 3 | 4 | #include "zstd.h" /* ZSTD_VERSION_STRING */ 5 | #define APSTUDIO_READONLY_SYMBOLS 6 | #include "verrsrc.h" 7 | #undef APSTUDIO_READONLY_SYMBOLS 8 | 9 | 10 | #if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) 11 | LANGUAGE 9, 1 12 | 13 | ///////////////////////////////////////////////////////////////////////////// 14 | // 15 | // Version 16 | // 17 | 18 | VS_VERSION_INFO VERSIONINFO 19 | FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 20 | PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 21 | FILEFLAGSMASK VS_FFI_FILEFLAGSMASK 22 | #ifdef _DEBUG 23 | FILEFLAGS VS_FF_DEBUG 24 | #else 25 | FILEFLAGS 0x0L 26 | #endif 27 | FILEOS VOS_NT_WINDOWS32 28 | FILETYPE VFT_DLL 29 | FILESUBTYPE VFT2_UNKNOWN 30 | BEGIN 31 | BLOCK "StringFileInfo" 32 | BEGIN 33 | BLOCK "040904B0" 34 | BEGIN 35 | VALUE "CompanyName", "Meta Platforms, Inc." 36 | VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm" 37 | VALUE "FileVersion", ZSTD_VERSION_STRING 38 | VALUE "InternalName", "libzstd.dll" 39 | VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates." 40 | VALUE "OriginalFilename", "libzstd.dll" 41 | VALUE "ProductName", "Zstandard" 42 | VALUE "ProductVersion", ZSTD_VERSION_STRING 43 | END 44 | END 45 | BLOCK "VarFileInfo" 46 | BEGIN 47 | VALUE "Translation", 0x0409, 1200 48 | END 49 | END 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /build/VS2010/zstd/zstd.rc: -------------------------------------------------------------------------------- 1 | // Microsoft Visual C++ generated resource script. 2 | // 3 | 4 | #include "zstd.h" /* ZSTD_VERSION_STRING */ 5 | #define APSTUDIO_READONLY_SYMBOLS 6 | #include "verrsrc.h" 7 | #undef APSTUDIO_READONLY_SYMBOLS 8 | 9 | 10 | #if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) 11 | LANGUAGE 9, 1 12 | 13 | ///////////////////////////////////////////////////////////////////////////// 14 | // 15 | // Version 16 | // 17 | 18 | VS_VERSION_INFO VERSIONINFO 19 | FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 20 | PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 21 | FILEFLAGSMASK VS_FFI_FILEFLAGSMASK 22 | #ifdef _DEBUG 23 | FILEFLAGS VS_FF_DEBUG 24 | #else 25 | FILEFLAGS 0x0L 26 | #endif 27 | FILEOS VOS_NT_WINDOWS32 28 | FILETYPE VFT_DLL 29 | FILESUBTYPE VFT2_UNKNOWN 30 | BEGIN 31 | BLOCK "StringFileInfo" 32 | BEGIN 33 | BLOCK "040904B0" 34 | BEGIN 35 | VALUE "CompanyName", "Meta Platforms, Inc." 36 | VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm" 37 | VALUE "FileVersion", ZSTD_VERSION_STRING 38 | VALUE "InternalName", "zstd.exe" 39 | VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates." 40 | VALUE "OriginalFilename", "zstd.exe" 41 | VALUE "ProductName", "Zstandard" 42 | VALUE "ProductVersion", ZSTD_VERSION_STRING 43 | END 44 | END 45 | BLOCK "VarFileInfo" 46 | BEGIN 47 | VALUE "Translation", 0x0409, 1200 48 | END 49 | END 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /build/VS_scripts/build.VS2010.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2010 Win32 Release v100 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2010 x64 Release v100 -------------------------------------------------------------------------------- /build/VS_scripts/build.VS2012.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2012 Win32 Release v110 5 | rem build 64-bit 6 | call "%~p0%build.generic.cmd" VS2012 x64 Release v110 -------------------------------------------------------------------------------- /build/VS_scripts/build.VS2013.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2013 Win32 Release v120 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2013 x64 Release v120 -------------------------------------------------------------------------------- /build/VS_scripts/build.VS2015.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2015 Win32 Release v140 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2015 x64 Release v140 -------------------------------------------------------------------------------- /build/VS_scripts/build.VS2017.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2017 Win32 Release v141 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2017 x64 Release v141 -------------------------------------------------------------------------------- /build/VS_scripts/build.VS2017Community.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2017Community Win32 Release v141 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2017Community x64 Release v141 -------------------------------------------------------------------------------- /build/VS_scripts/build.VS2017Enterprise.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2017Enterprise Win32 Release v141 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2017Enterprise x64 Release v141 -------------------------------------------------------------------------------- /build/VS_scripts/build.VS2017Professional.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2017Professional Win32 Release v141 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2017Professional x64 Release v141 -------------------------------------------------------------------------------- /build/VS_scripts/build.VSPreview.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" preview Win32 Release v143 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" preview x64 Release v143 -------------------------------------------------------------------------------- /build/cmake/.gitignore: -------------------------------------------------------------------------------- 1 | # cmake working directory 2 | cmakeBuild 3 | 4 | # cmake artefacts 5 | CMakeCache.txt 6 | CMakeFiles 7 | Makefile 8 | cmake_install.cmake 9 | cmake_uninstall.cmake 10 | *.1 11 | -------------------------------------------------------------------------------- /build/cmake/CMakeModules/GetZstdLibraryVersion.cmake: -------------------------------------------------------------------------------- 1 | function(GetZstdLibraryVersion _header _major _minor _patch) 2 | # Read file content 3 | file(READ ${_header} CONTENT) 4 | 5 | string(REGEX MATCH ".*define ZSTD_VERSION_MAJOR *([0-9]+).*define ZSTD_VERSION_MINOR *([0-9]+).*define ZSTD_VERSION_RELEASE *([0-9]+)" VERSION_REGEX "${CONTENT}") 6 | set(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE) 7 | set(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE) 8 | set(${_patch} ${CMAKE_MATCH_3} PARENT_SCOPE) 9 | endfunction() 10 | 11 | -------------------------------------------------------------------------------- /build/cmake/CMakeModules/JoinPaths.cmake: -------------------------------------------------------------------------------- 1 | # This module provides function for joining paths 2 | # known from most languages 3 | # 4 | # SPDX-License-Identifier: (MIT OR CC0-1.0) 5 | # Copyright 2020 Jan Tojnar 6 | # https://github.com/jtojnar/cmake-snips 7 | # 8 | # Modelled after Python’s os.path.join 9 | # https://docs.python.org/3.7/library/os.path.html#os.path.join 10 | # Windows not supported 11 | function(join_paths joined_path first_path_segment) 12 | set(temp_path "${first_path_segment}") 13 | foreach(current_segment IN LISTS ARGN) 14 | if(NOT ("${current_segment}" STREQUAL "")) 15 | if(IS_ABSOLUTE "${current_segment}") 16 | set(temp_path "${current_segment}") 17 | else() 18 | set(temp_path "${temp_path}/${current_segment}") 19 | endif() 20 | endif() 21 | endforeach() 22 | set(${joined_path} "${temp_path}" PARENT_SCOPE) 23 | endfunction() 24 | -------------------------------------------------------------------------------- /build/cmake/CMakeModules/ZstdBuild.cmake: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # ZSTD Build Targets Configuration 3 | # ################################################################ 4 | 5 | # Always build the library first (this defines ZSTD_BUILD_STATIC/SHARED options) 6 | add_subdirectory(lib) 7 | 8 | # Validate build configuration after lib options are defined 9 | if(ZSTD_BUILD_PROGRAMS) 10 | if(NOT ZSTD_BUILD_STATIC AND NOT ZSTD_PROGRAMS_LINK_SHARED) 11 | message(SEND_ERROR "Static library required to build zstd CLI programs") 12 | elseif(NOT ZSTD_BUILD_SHARED AND ZSTD_PROGRAMS_LINK_SHARED) 13 | message(SEND_ERROR "Shared library required to build zstd CLI programs") 14 | endif() 15 | endif() 16 | 17 | if(ZSTD_BUILD_TESTS AND NOT ZSTD_BUILD_STATIC) 18 | message(SEND_ERROR "Static library required to build test suite") 19 | endif() 20 | 21 | # Add programs if requested 22 | if(ZSTD_BUILD_PROGRAMS) 23 | add_subdirectory(programs) 24 | endif() 25 | 26 | # Add tests if requested 27 | if(ZSTD_BUILD_TESTS) 28 | enable_testing() 29 | add_subdirectory(tests) 30 | endif() 31 | 32 | # Add contrib utilities if requested 33 | if(ZSTD_BUILD_CONTRIB) 34 | add_subdirectory(contrib) 35 | endif() 36 | 37 | # Clean-all target for thorough cleanup 38 | add_custom_target(clean-all 39 | COMMAND ${CMAKE_BUILD_TOOL} clean 40 | COMMAND ${CMAKE_COMMAND} -E remove_directory ${CMAKE_BINARY_DIR}/ 41 | COMMENT "Performing complete clean including build directory" 42 | ) 43 | -------------------------------------------------------------------------------- /build/cmake/CMakeModules/ZstdDependencies.cmake: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # ZSTD Dependencies Configuration 3 | # ################################################################ 4 | 5 | # Function to handle HP-UX thread configuration 6 | function(setup_hpux_threads) 7 | find_package(Threads) 8 | if(NOT Threads_FOUND) 9 | set(CMAKE_USE_PTHREADS_INIT 1 PARENT_SCOPE) 10 | set(CMAKE_THREAD_LIBS_INIT -lpthread PARENT_SCOPE) 11 | set(CMAKE_HAVE_THREADS_LIBRARY 1 PARENT_SCOPE) 12 | set(Threads_FOUND TRUE PARENT_SCOPE) 13 | endif() 14 | endfunction() 15 | 16 | # Configure threading support 17 | if(ZSTD_MULTITHREAD_SUPPORT AND UNIX) 18 | if(CMAKE_SYSTEM_NAME MATCHES "HP-UX") 19 | setup_hpux_threads() 20 | else() 21 | set(THREADS_PREFER_PTHREAD_FLAG ON) 22 | find_package(Threads REQUIRED) 23 | endif() 24 | 25 | if(CMAKE_USE_PTHREADS_INIT) 26 | set(THREADS_LIBS "${CMAKE_THREAD_LIBS_INIT}") 27 | else() 28 | message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads") 29 | endif() 30 | endif() 31 | -------------------------------------------------------------------------------- /build/cmake/CMakeModules/ZstdPackage.cmake: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # ZSTD Package Configuration 3 | # ################################################################ 4 | 5 | include(CMakePackageConfigHelpers) 6 | 7 | # Generate version file 8 | write_basic_package_version_file( 9 | "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake" 10 | VERSION ${zstd_VERSION} 11 | COMPATIBILITY SameMajorVersion 12 | ) 13 | 14 | # Export targets for build directory 15 | export(EXPORT zstdExports 16 | FILE "${CMAKE_CURRENT_BINARY_DIR}/zstdTargets.cmake" 17 | NAMESPACE zstd:: 18 | ) 19 | 20 | # Configure package for installation 21 | set(ConfigPackageLocation ${CMAKE_INSTALL_LIBDIR}/cmake/zstd) 22 | 23 | # Install exported targets 24 | install(EXPORT zstdExports 25 | FILE zstdTargets.cmake 26 | NAMESPACE zstd:: 27 | DESTINATION ${ConfigPackageLocation} 28 | ) 29 | 30 | # Configure and install package config file 31 | configure_package_config_file( 32 | zstdConfig.cmake.in 33 | "${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake" 34 | INSTALL_DESTINATION ${ConfigPackageLocation} 35 | ) 36 | 37 | # Install config files 38 | install(FILES 39 | "${CMAKE_CURRENT_BINARY_DIR}/zstdConfig.cmake" 40 | "${CMAKE_CURRENT_BINARY_DIR}/zstdConfigVersion.cmake" 41 | DESTINATION ${ConfigPackageLocation} 42 | ) 43 | -------------------------------------------------------------------------------- /build/cmake/CMakeModules/ZstdVersion.cmake: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # ZSTD Version Configuration 3 | # ################################################################ 4 | 5 | # Setup CMake policy version 6 | set(ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION "3") 7 | set(ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION "13") 8 | 9 | # Determine appropriate policy version 10 | if("${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}" EQUAL "${CMAKE_MAJOR_VERSION}" AND 11 | "${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}" GREATER "${CMAKE_MINOR_VERSION}") 12 | set(ZSTD_CMAKE_POLICY_VERSION "${CMAKE_VERSION}") 13 | else() 14 | set(ZSTD_CMAKE_POLICY_VERSION "${ZSTD_MAX_VALIDATED_CMAKE_MAJOR_VERSION}.${ZSTD_MAX_VALIDATED_CMAKE_MINOR_VERSION}.0") 15 | endif() 16 | 17 | cmake_policy(VERSION ${ZSTD_CMAKE_POLICY_VERSION}) 18 | 19 | # Parse version from header file 20 | include(GetZstdLibraryVersion) 21 | GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h zstd_VERSION_MAJOR zstd_VERSION_MINOR zstd_VERSION_PATCH) 22 | 23 | # Set version variables 24 | set(ZSTD_SHORT_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}") 25 | set(ZSTD_FULL_VERSION "${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}") 26 | 27 | # Project metadata 28 | set(zstd_HOMEPAGE_URL "https://facebook.github.io/zstd") 29 | set(zstd_DESCRIPTION "Zstandard is a real-time compression algorithm, providing high compression ratios.") 30 | 31 | message(STATUS "ZSTD VERSION: ${zstd_VERSION_MAJOR}.${zstd_VERSION_MINOR}.${zstd_VERSION_PATCH}") 32 | -------------------------------------------------------------------------------- /build/cmake/contrib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | project(contrib) 11 | 12 | add_subdirectory(pzstd) 13 | add_subdirectory(gen_html) 14 | -------------------------------------------------------------------------------- /build/cmake/contrib/gen_html/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | project(gen_html) 11 | include(GetZstdLibraryVersion) 12 | 13 | set(CMAKE_INCLUDE_CURRENT_DIR TRUE) 14 | 15 | # Define programs directory, where sources and header files are located 16 | set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib) 17 | set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs) 18 | set(GENHTML_DIR ${ZSTD_SOURCE_DIR}/contrib/gen_html) 19 | set(GENHTML_BINARY ${PROJECT_BINARY_DIR}/gen_html${CMAKE_EXECUTABLE_SUFFIX}) 20 | include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHTML_DIR}) 21 | 22 | add_executable(gen_html ${GENHTML_DIR}/gen_html.cpp) 23 | 24 | GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h VMAJOR VMINOR VRELEASE) 25 | set(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}") 26 | add_custom_target(zstd_manual.html ALL 27 | ${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html" 28 | DEPENDS gen_html COMMENT "Update zstd manual") 29 | 30 | install(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}") 31 | -------------------------------------------------------------------------------- /build/cmake/lib/.gitignore: -------------------------------------------------------------------------------- 1 | # cmake build artefact 2 | libzstd.pc 3 | -------------------------------------------------------------------------------- /build/cmake/lib/cmake_uninstall.cmake.in: -------------------------------------------------------------------------------- 1 | 2 | if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt") 3 | message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt") 4 | endif() 5 | 6 | file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files) 7 | string(REGEX REPLACE "\n" ";" files "${files}") 8 | foreach(file ${files}) 9 | message(STATUS "Uninstalling $ENV{DESTDIR}${file}") 10 | if(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") 11 | exec_program( 12 | "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" 13 | OUTPUT_VARIABLE rm_out 14 | RETURN_VALUE rm_retval 15 | ) 16 | if(NOT "${rm_retval}" STREQUAL 0) 17 | message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}") 18 | endif() 19 | else() 20 | message(STATUS "File $ENV{DESTDIR}${file} does not exist.") 21 | endif() 22 | endforeach() 23 | -------------------------------------------------------------------------------- /build/cmake/programs/.gitignore: -------------------------------------------------------------------------------- 1 | # produced by make 2 | zstd 3 | zstd-frugal 4 | unzstd 5 | zstdcat 6 | -------------------------------------------------------------------------------- /build/cmake/tests/.gitignore: -------------------------------------------------------------------------------- 1 | # produced by make 2 | datagen 3 | fullbench 4 | fuzzer 5 | paramgrill 6 | 7 | -------------------------------------------------------------------------------- /build/cmake/zstdConfig.cmake.in: -------------------------------------------------------------------------------- 1 | @PACKAGE_INIT@ 2 | 3 | include(CMakeFindDependencyMacro) 4 | if(@ZSTD_MULTITHREAD_SUPPORT@ AND "@UNIX@") 5 | find_dependency(Threads) 6 | endif() 7 | 8 | include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets.cmake") 9 | 10 | check_required_components("zstd") 11 | -------------------------------------------------------------------------------- /build/meson/GetZstdLibraryVersion.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # ############################################################################# 3 | # Copyright (c) 2018-present lzutao 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | import re 11 | 12 | 13 | def find_version_tuple(filepath): 14 | version_file_data = None 15 | with open(filepath) as fd: 16 | version_file_data = fd.read() 17 | 18 | patterns = r"""#\s*define\s+ZSTD_VERSION_MAJOR\s+([0-9]+) 19 | #\s*define\s+ZSTD_VERSION_MINOR\s+([0-9]+) 20 | #\s*define\s+ZSTD_VERSION_RELEASE\s+([0-9]+) 21 | """ 22 | regex = re.compile(patterns, re.MULTILINE) 23 | version_match = regex.search(version_file_data) 24 | if version_match: 25 | return version_match.groups() 26 | raise Exception("Unable to find version string") 27 | 28 | 29 | def main(): 30 | import argparse 31 | parser = argparse.ArgumentParser(description='Print zstd version from lib/zstd.h') 32 | parser.add_argument('file', help='path to lib/zstd.h') 33 | args = parser.parse_args() 34 | version_tuple = find_version_tuple(args.file) 35 | print('.'.join(version_tuple)) 36 | 37 | 38 | if __name__ == '__main__': 39 | main() 40 | -------------------------------------------------------------------------------- /build/meson/README.md: -------------------------------------------------------------------------------- 1 | Meson build system for zstandard 2 | ================================ 3 | 4 | Meson is a build system designed to optimize programmer productivity. 5 | It aims to do this by providing simple, out-of-the-box support for 6 | modern software development tools and practices, such as unit tests, 7 | coverage reports, Valgrind, CCache and the like. 8 | 9 | This Meson build system is provided with no guarantee and maintained 10 | by Dima Krasner \. 11 | 12 | It outputs one `libzstd`, either shared or static, depending on 13 | `default_library` option. 14 | 15 | ## How to build 16 | 17 | `cd` to this meson directory (`build/meson`) 18 | 19 | ```sh 20 | meson setup -Dbin_programs=true -Dbin_contrib=true builddir 21 | cd builddir 22 | ninja # to build 23 | ninja install # to install 24 | ``` 25 | 26 | You might want to install it in staging directory: 27 | 28 | ```sh 29 | DESTDIR=./staging ninja install 30 | ``` 31 | 32 | To configure build options, use: 33 | 34 | ```sh 35 | meson configure 36 | ``` 37 | 38 | See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html). 39 | -------------------------------------------------------------------------------- /build/meson/contrib/gen_html/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ############################################################################# 9 | 10 | zstd_rootdir = '../../../..' 11 | 12 | gen_html_includes = include_directories(join_paths(zstd_rootdir, 'programs'), 13 | join_paths(zstd_rootdir, 'lib'), 14 | join_paths(zstd_rootdir, 'lib/common'), 15 | join_paths(zstd_rootdir, 'contrib/gen_html')) 16 | 17 | gen_html = executable('gen_html', 18 | join_paths(zstd_rootdir, 'contrib/gen_html/gen_html.cpp'), 19 | include_directories: gen_html_includes, 20 | native: true, 21 | install: false) 22 | 23 | # Update zstd manual 24 | zstd_manual_html = custom_target('zstd_manual.html', 25 | output : 'zstd_manual.html', 26 | command : [gen_html, 27 | zstd_version, 28 | join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h'), 29 | '@OUTPUT@'], 30 | install : false) 31 | -------------------------------------------------------------------------------- /build/meson/contrib/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present Dima Krasner 3 | # lzutao 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | 11 | subdir('pzstd') 12 | subdir('gen_html') 13 | -------------------------------------------------------------------------------- /build/meson/contrib/pzstd/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ############################################################################# 9 | 10 | zstd_rootdir = '../../../..' 11 | 12 | pzstd_includes = include_directories(join_paths(zstd_rootdir, 'programs'), 13 | join_paths(zstd_rootdir, 'contrib/pzstd')) 14 | pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'), 15 | join_paths(zstd_rootdir, 'contrib/pzstd/main.cpp'), 16 | join_paths(zstd_rootdir, 'contrib/pzstd/Options.cpp'), 17 | join_paths(zstd_rootdir, 'contrib/pzstd/Pzstd.cpp'), 18 | join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')] 19 | pzstd = executable('pzstd', 20 | pzstd_sources, 21 | cpp_args: pzstd_warning_flags, 22 | include_directories: pzstd_includes, 23 | dependencies: [ libzstd_internal_dep, thread_dep ], 24 | override_options: ['b_ndebug=true'], 25 | install: true) 26 | -------------------------------------------------------------------------------- /build/single_file_libs/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # build artifacts 3 | zstddeclib.c 4 | zstdenclib.c 5 | zstd.c 6 | zstd.h 7 | zstd_errors.h 8 | 9 | # test artifacts 10 | temp* 11 | -------------------------------------------------------------------------------- /build/single_file_libs/create_single_file_decoder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Where to find the sources 4 | ZSTD_SRC_ROOT="../../lib" 5 | 6 | # Amalgamate the sources 7 | echo "Amalgamating files..." 8 | # Using the faster Python script if we have 3.8 or higher 9 | if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then 10 | ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c 11 | else 12 | ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c 13 | fi 14 | # Did combining work? 15 | if [ $? -ne 0 ]; then 16 | echo "Combine script: FAILED" 17 | exit 1 18 | fi 19 | echo "Combine script: PASSED" 20 | -------------------------------------------------------------------------------- /build/single_file_libs/create_single_file_library.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Where to find the sources 4 | ZSTD_SRC_ROOT="../../lib" 5 | 6 | # Amalgamate the sources 7 | echo "Amalgamating files..." 8 | # Using the faster Python script if we have 3.8 or higher 9 | if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then 10 | ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c 11 | else 12 | ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c 13 | fi 14 | # Did combining work? 15 | if [ $? -ne 0 ]; then 16 | echo "Combine script: FAILED" 17 | exit 1 18 | fi 19 | echo "Combine script: PASSED" 20 | -------------------------------------------------------------------------------- /build/single_file_libs/examples/README.md: -------------------------------------------------------------------------------- 1 | # Single File ZStandard Examples 2 | 3 | The examples `#include` the generated `zstddeclib.c` directly but work equally as well when including `zstd.h` and compiling the amalgamated source separately. 4 | 5 | `simple.c` is the most basic example of decompressing content and verifying the result. 6 | 7 | `emscripten.c` is a bare-bones [Emscripten](https://github.com/emscripten-core/emscripten) compiled WebGL demo using Zstd to further compress a DXT1 texture (see the [original PNG image](testcard.png)). The 256x256 texture would normally be 32kB, but even when bundled with the Zstd decompressor the resulting WebAssembly weighs in at 41kB (`shell.html` is a support file to run the Wasm). 8 | 9 | `roundtrip.c` is an example to use with the optional [amalgamated library](../create_single_file_library.sh) showing compression the decompression. 10 | 11 | The example files in this directory are released under a [Creative Commons Zero license](https://creativecommons.org/publicdomain/zero/1.0/) (or Public Domain, whichever is applicable in your jurisdiction). 12 | -------------------------------------------------------------------------------- /build/single_file_libs/examples/shell.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Emscripten Shell 9 | 13 | 14 | 15 | 16 | 29 | {{{ SCRIPT }}} 30 | 31 | 32 | -------------------------------------------------------------------------------- /build/single_file_libs/examples/testcard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/build/single_file_libs/examples/testcard.png -------------------------------------------------------------------------------- /contrib/VS2005/README.md: -------------------------------------------------------------------------------- 1 | ## Project Support Notice 2 | 3 | The VS2005 Project directory has been moved to the contrib directory in order to indicate that it will no longer be supported. 4 | -------------------------------------------------------------------------------- /contrib/cleanTabs: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | sed -i '' $'s/\t/ /g' ../lib/**/*.{h,c} ../programs/*.{h,c} ../tests/*.c ./**/*.{h,cpp} ../examples/*.c ../zlibWrapper/*.{h,c} 3 | -------------------------------------------------------------------------------- /contrib/diagnose_corruption/.gitignore: -------------------------------------------------------------------------------- 1 | check_flipped_bits 2 | -------------------------------------------------------------------------------- /contrib/diagnose_corruption/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | .PHONY: all 11 | all: check_flipped_bits 12 | 13 | ZSTDLIBDIR ?= ../../lib 14 | 15 | CFLAGS ?= -O3 16 | CFLAGS += -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZSTDLIBDIR)/compress \ 17 | -I$(ZSTDLIBDIR)/decompress 18 | CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ 19 | -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ 20 | -Wstrict-prototypes -Wundef \ 21 | -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ 22 | -Wredundant-decls -Wmissing-prototypes 23 | CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) 24 | FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) 25 | 26 | .PHONY: $(ZSTDLIBDIR)/libzstd.a 27 | $(ZSTDLIBDIR)/libzstd.a: 28 | $(MAKE) -C $(ZSTDLIBDIR) libzstd.a 29 | 30 | check_flipped_bits: check_flipped_bits.c $(ZSTDLIBDIR)/libzstd.a 31 | $(CC) $(FLAGS) $< -o $@$(EXT) $(ZSTDLIBDIR)/libzstd.a 32 | 33 | .PHONY: clean 34 | clean: 35 | rm -f check_flipped_bits 36 | -------------------------------------------------------------------------------- /contrib/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Dockerfile 2 | # First image to build the binary 3 | FROM alpine@sha256:69665d02cb32192e52e07644d76bc6f25abeb5410edc1c7a81a10ba3f0efb90a as builder 4 | 5 | RUN apk --no-cache add make gcc libc-dev 6 | COPY . /src 7 | RUN mkdir /pkg && cd /src && make && make DESTDIR=/pkg install 8 | 9 | # Second minimal image to only keep the built binary 10 | FROM alpine@sha256:69665d02cb32192e52e07644d76bc6f25abeb5410edc1c7a81a10ba3f0efb90a 11 | 12 | # Copy the built files 13 | COPY --from=builder /pkg / 14 | 15 | # Copy the license as well 16 | RUN mkdir -p /usr/local/share/licenses/zstd 17 | COPY --from=builder /src/LICENSE /usr/local/share/licences/zstd/ 18 | 19 | # Just run `zstd` if no other command is given 20 | CMD ["/usr/local/bin/zstd"] 21 | -------------------------------------------------------------------------------- /contrib/docker/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Requirement 3 | 4 | The `Dockerfile` script requires a version of `docker` >= 17.05 5 | 6 | ## Installing docker 7 | 8 | The official docker install docs use a ppa with a modern version available: 9 | https://docs.docker.com/install/linux/docker-ce/ubuntu/ 10 | 11 | ## How to run 12 | 13 | `docker build -t zstd .` 14 | 15 | ## test 16 | 17 | ``` 18 | echo foo | docker run -i --rm zstd | docker run -i --rm zstd zstdcat 19 | foo 20 | ``` 21 | -------------------------------------------------------------------------------- /contrib/externalSequenceProducer/.gitignore: -------------------------------------------------------------------------------- 1 | # build artifacts 2 | externalSequenceProducer 3 | -------------------------------------------------------------------------------- /contrib/externalSequenceProducer/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Yann Collet, Meta Platforms, Inc. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | PROGDIR = ../../programs 11 | LIBDIR = ../../lib 12 | 13 | LIBZSTD = $(LIBDIR)/libzstd.a 14 | 15 | CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/compress -I$(LIBDIR)/common 16 | 17 | CFLAGS ?= -O3 18 | CFLAGS += -std=gnu99 19 | DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ 20 | -Wstrict-aliasing=1 -Wswitch-enum \ 21 | -Wstrict-prototypes -Wundef -Wpointer-arith \ 22 | -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ 23 | -Wredundant-decls 24 | CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) 25 | 26 | default: externalSequenceProducer 27 | 28 | all: externalSequenceProducer 29 | 30 | externalSequenceProducer: sequence_producer.c main.c $(LIBZSTD) 31 | $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ 32 | 33 | .PHONY: $(LIBZSTD) 34 | $(LIBZSTD): 35 | $(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)" 36 | 37 | clean: 38 | $(RM) *.o 39 | $(MAKE) -C $(LIBDIR) clean > /dev/null 40 | $(RM) externalSequenceProducer 41 | -------------------------------------------------------------------------------- /contrib/externalSequenceProducer/README.md: -------------------------------------------------------------------------------- 1 | externalSequenceProducer 2 | ===================== 3 | 4 | `externalSequenceProducer` is a test tool for the Block-Level Sequence Producer API. 5 | It demonstrates how to use the API to perform a simple round-trip test. 6 | 7 | A sample sequence producer is provided in sequence_producer.c, but the user can swap 8 | this out with a different one if desired. The sample sequence producer implements 9 | LZ parsing with a 1KB hashtable. Dictionary-based parsing is not currently supported. 10 | 11 | Command line : 12 | ``` 13 | externalSequenceProducer filename 14 | ``` 15 | -------------------------------------------------------------------------------- /contrib/externalSequenceProducer/sequence_producer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Yann Collet, Meta Platforms, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef MATCHFINDER_H 12 | #define MATCHFINDER_H 13 | 14 | #define ZSTD_STATIC_LINKING_ONLY 15 | #include "zstd.h" 16 | 17 | size_t simpleSequenceProducer( 18 | void* sequenceProducerState, 19 | ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, 20 | const void* src, size_t srcSize, 21 | const void* dict, size_t dictSize, 22 | int compressionLevel, 23 | size_t windowSize 24 | ); 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /contrib/gen_html/.gitignore: -------------------------------------------------------------------------------- 1 | # make artefact 2 | gen_html 3 | zstd_manual.html 4 | -------------------------------------------------------------------------------- /contrib/gen_html/README.md: -------------------------------------------------------------------------------- 1 | gen_html - a program for automatic generation of zstd manual 2 | ============================================================ 3 | 4 | #### Introduction 5 | 6 | This simple C++ program generates a single-page HTML manual from `zstd.h`. 7 | 8 | The format of recognized comment blocks is following: 9 | - comments of type `/*!` mean: this is a function declaration; switch comments with declarations 10 | - comments of type `/**` and `/*-` mean: this is a comment; use a `

` header for the first line 11 | - comments of type `/*=` and `/**=` mean: use a `

` header and show also all functions until first empty line 12 | - comments of type `/*X` where `X` is different from above-mentioned are ignored 13 | 14 | Moreover: 15 | - `ZSTDLIB_API` is removed to improve readability 16 | - `typedef` are detected and included even if uncommented 17 | - comments of type `/**<` and `/*!<` are detected and only function declaration is highlighted (bold) 18 | 19 | 20 | #### Usage 21 | 22 | The program requires 3 parameters: 23 | ``` 24 | gen_html [zstd_version] [input_file] [output_html] 25 | ``` 26 | 27 | To compile program and generate zstd manual we have used: 28 | ``` 29 | make 30 | ./gen_html.exe 1.1.1 ../../lib/zstd.h zstd_manual.html 31 | ``` 32 | -------------------------------------------------------------------------------- /contrib/gen_html/gen-zstd-manual.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | LIBVER_MAJOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h` 4 | LIBVER_MINOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h` 5 | LIBVER_PATCH_SCRIPT=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h` 6 | LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT 7 | 8 | echo ZSTD_VERSION=$LIBVER_SCRIPT 9 | ./gen_html $LIBVER_SCRIPT ../../lib/zstd.h ./zstd_manual.html 10 | -------------------------------------------------------------------------------- /contrib/largeNbDicts/.gitignore: -------------------------------------------------------------------------------- 1 | # build artifacts 2 | largeNbDicts 3 | -------------------------------------------------------------------------------- /contrib/largeNbDicts/README.md: -------------------------------------------------------------------------------- 1 | largeNbDicts 2 | ===================== 3 | 4 | `largeNbDicts` is a benchmark test tool 5 | dedicated to the specific scenario of 6 | dictionary decompression using a very large number of dictionaries. 7 | When dictionaries are constantly changing, they are always "cold", 8 | suffering from increased latency due to cache misses. 9 | 10 | The tool is created in a bid to investigate performance for this scenario, 11 | and experiment mitigation techniques. 12 | 13 | Command line : 14 | ``` 15 | largeNbDicts [Options] filename(s) 16 | 17 | Options : 18 | -z : benchmark compression (default) 19 | -d : benchmark decompression 20 | -r : recursively load all files in subdirectories (default: off) 21 | -B# : split input into blocks of size # (default: no split) 22 | -# : use compression level # (default: 3) 23 | -D # : use # as a dictionary (default: create one) 24 | -i# : nb benchmark rounds (default: 6) 25 | --nbBlocks=#: use # blocks for bench (default: one per file) 26 | --nbDicts=# : create # dictionaries for bench (default: one per block) 27 | -h : help (this text) 28 | 29 | Advanced Options (see zstd.h for documentation) : 30 | --dedicated-dict-search 31 | --dict-content-type=# 32 | --dict-attach-pref=# 33 | ``` 34 | -------------------------------------------------------------------------------- /contrib/linux-kernel/.gitignore: -------------------------------------------------------------------------------- 1 | !lib/zstd 2 | !lib/zstd/* 3 | *.o 4 | *.a 5 | -------------------------------------------------------------------------------- /contrib/linux-kernel/README.md: -------------------------------------------------------------------------------- 1 | # Zstd in the Linux Kernel 2 | 3 | This directory contains the scripts needed to transform upstream zstd into the version imported into the kernel. All the transforms are automated and tested by our continuous integration. 4 | 5 | ## Upgrading Zstd in the Linux Kernel 6 | 7 | 1. `cd` into this directory. 8 | 2. Run `make libzstd` and read the output. Make sure that all the diffs printed and changes made by the script are correct. 9 | 3. Run `make test` and ensure that it passes. 10 | 4. Import zstd into the Linux Kernel `make import LINUX=/path/to/linux/repo` 11 | 5. Inspect the diff for sanity. 12 | 6. Check the Linux Kernel history for zstd. If any patches were made to the kernel version of zstd, but not to upstream zstd, then port them upstream if necessary. 13 | 7. Test the diff. Benchmark if necessary. Make sure to test multiple architectures: At least x86, i386, and arm. 14 | 8. Submit the patch to the LKML. 15 | -------------------------------------------------------------------------------- /contrib/linux-kernel/decompress_sources.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */ 2 | /* 3 | * Copyright (c) Meta Platforms, Inc. and affiliates. 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | /* 13 | * This file includes every .c file needed for decompression. 14 | * It is used by lib/decompress_unzstd.c to include the decompression 15 | * source into the translation-unit, so it can be used for kernel 16 | * decompression. 17 | */ 18 | 19 | /* 20 | * Disable the ASM Huffman implementation because we need to 21 | * include all the sources. 22 | */ 23 | #define ZSTD_DISABLE_ASM 1 24 | 25 | #include "common/debug.c" 26 | #include "common/entropy_common.c" 27 | #include "common/error_private.c" 28 | #include "common/fse_decompress.c" 29 | #include "common/zstd_common.c" 30 | #include "decompress/huf_decompress.c" 31 | #include "decompress/zstd_ddict.c" 32 | #include "decompress/zstd_decompress.c" 33 | #include "decompress/zstd_decompress_block.c" 34 | #include "zstd_decompress_module.c" 35 | -------------------------------------------------------------------------------- /contrib/linux-kernel/linux.mk: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause 2 | # ################################################################ 3 | # Copyright (c) Meta Platforms, Inc. and affiliates. 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # You may select, at your option, one of the above-listed licenses. 10 | # ################################################################ 11 | obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o 12 | obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o 13 | obj-$(CONFIG_ZSTD_COMMON) += zstd_common.o 14 | 15 | zstd_compress-y := \ 16 | zstd_compress_module.o \ 17 | compress/fse_compress.o \ 18 | compress/hist.o \ 19 | compress/huf_compress.o \ 20 | compress/zstd_compress.o \ 21 | compress/zstd_compress_literals.o \ 22 | compress/zstd_compress_sequences.o \ 23 | compress/zstd_compress_superblock.o \ 24 | compress/zstd_double_fast.o \ 25 | compress/zstd_fast.o \ 26 | compress/zstd_lazy.o \ 27 | compress/zstd_ldm.o \ 28 | compress/zstd_opt.o \ 29 | compress/zstd_preSplit.o \ 30 | 31 | zstd_decompress-y := \ 32 | zstd_decompress_module.o \ 33 | decompress/huf_decompress.o \ 34 | decompress/zstd_ddict.o \ 35 | decompress/zstd_decompress.o \ 36 | decompress/zstd_decompress_block.o \ 37 | 38 | zstd_common-y := \ 39 | zstd_common_module.o \ 40 | common/debug.o \ 41 | common/entropy_common.o \ 42 | common/error_private.o \ 43 | common/fse_decompress.o \ 44 | common/zstd_common.o \ 45 | -------------------------------------------------------------------------------- /contrib/linux-kernel/squashfs-benchmark.sh: -------------------------------------------------------------------------------- 1 | # !/bin/sh 2 | set -e 3 | 4 | # Benchmarks run on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. 5 | # The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor and 6 | # 16 GB of RAM and an SSD. 7 | 8 | # $BENCHMARK_DIR is generated with the following commands, from the Ubuntu image 9 | # ubuntu-16.10-desktop-amd64.iso. 10 | # > mkdir mnt 11 | # > sudo mount -o loop ubuntu-16.10-desktop-amd64.iso mnt 12 | # > cp mnt/casper/filesystem.squashfs . 13 | # > sudo unsquashfs filesystem.squashfs 14 | 15 | # $HOME is on a ext4 filesystem 16 | BENCHMARK_DIR="$HOME/squashfs-root/" 17 | BENCHMARK_FS="$HOME/filesystem.squashfs" 18 | 19 | # Normalize the environment 20 | sudo rm -f $BENCHMARK_FS 2> /dev/null > /dev/null || true 21 | sudo umount /mnt/squashfs 2> /dev/null > /dev/null || true 22 | 23 | # Run the benchmark 24 | echo "Compression" 25 | echo "sudo mksquashfs $BENCHMARK_DIR $BENCHMARK_FS $@" 26 | time sudo mksquashfs $BENCHMARK_DIR $BENCHMARK_FS $@ 2> /dev/null > /dev/null 27 | 28 | echo "Approximate compression ratio" 29 | printf "%d / %d\n" \ 30 | $(sudo du -sx --block-size=1 $BENCHMARK_DIR | cut -f1) \ 31 | $(sudo du -sx --block-size=1 $BENCHMARK_FS | cut -f1); 32 | 33 | # Mount the filesystem 34 | sudo mount -t squashfs $BENCHMARK_FS /mnt/squashfs 35 | 36 | echo "Decompression" 37 | time sudo tar -c /mnt/squashfs 2> /dev/null | wc -c > /dev/null 38 | 39 | sudo umount /mnt/squashfs 40 | -------------------------------------------------------------------------------- /contrib/linux-kernel/test/include/linux/compiler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_COMPILER_H 11 | #define LINUX_COMPILER_H 12 | 13 | #ifndef inline 14 | #define inline __inline __attribute__((unused)) 15 | #endif 16 | 17 | #ifndef noinline 18 | #define noinline __attribute__((noinline)) 19 | #endif 20 | 21 | #define fallthrough __attribute__((__fallthrough__)) 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /contrib/linux-kernel/test/include/linux/errno.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_ERRNO_H 11 | #define LINUX_ERRNO_H 12 | 13 | #define EINVAL 22 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /contrib/linux-kernel/test/include/linux/kernel.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_KERNEL_H 11 | #define LINUX_KERNEL_H 12 | 13 | #define WARN_ON(x) 14 | 15 | #define PTR_ALIGN(p, a) (typeof(p))ALIGN((unsigned long long)(p), (a)) 16 | #define ALIGN(x, a) ALIGN_MASK((x), (a) - 1) 17 | #define ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /contrib/linux-kernel/test/include/linux/limits.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_LIMITS_H 11 | #define LINUX_LIMITS_H 12 | 13 | #include 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /contrib/linux-kernel/test/include/linux/math64.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_MATH64_H 11 | #define LINUX_MATH64_H 12 | 13 | #define div_u64(dividend, divisor) ((dividend) / (divisor)) 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /contrib/linux-kernel/test/include/linux/module.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_MODULE_H 11 | #define LINUX_MODULE_H 12 | 13 | #define EXPORT_SYMBOL(symbol) \ 14 | void* __##symbol = symbol 15 | #define EXPORT_SYMBOL_GPL(symbol) \ 16 | void* __##symbol = symbol 17 | #define MODULE_LICENSE(license) 18 | #define MODULE_DESCRIPTION(description) 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /contrib/linux-kernel/test/include/linux/printk.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_PRINTK_H 11 | #define LINUX_PRINTK_H 12 | 13 | #define pr_debug(...) 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /contrib/linux-kernel/test/include/linux/stddef.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_STDDEF_H 11 | #define LINUX_STDDEF_H 12 | 13 | #include 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /contrib/linux-kernel/test/include/linux/swab.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_SWAB_H 11 | #define LINUX_SWAB_H 12 | 13 | #define swab32(x) __builtin_bswap32((x)) 14 | #define swab64(x) __builtin_bswap64((x)) 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /contrib/linux-kernel/test/include/linux/types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_TYPES_H 11 | #define LINUX_TYPES_H 12 | 13 | #include 14 | #include 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /contrib/linux-kernel/test/macro-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -e 4 | 5 | SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) 6 | INCLUDE_DIR="$SCRIPT_DIR/../linux/include" 7 | LIB_DIR="$SCRIPT_DIR/../linux/lib" 8 | 9 | 10 | print() { 11 | printf '%b' "${*}" 12 | } 13 | 14 | println() { 15 | printf '%b\n' "${*}" 16 | } 17 | 18 | die() { 19 | println "$@" 1>&2 20 | exit 1 21 | } 22 | 23 | test_not_present() { 24 | print "Testing that '$1' is not present... " 25 | grep -r $1 "$INCLUDE_DIR" "$LIB_DIR" && die "Fail!" 26 | println "Okay" 27 | } 28 | 29 | println "This test checks that the macro removal process worked as expected" 30 | println "If this test fails, then freestanding.py wasn't able to remove one of these" 31 | println "macros from the source code completely. You'll either need to rewrite the check" 32 | println "or improve freestanding.py." 33 | println "" 34 | 35 | test_not_present "ZSTD_NO_INTRINSICS" 36 | test_not_present "ZSTD_NO_UNUSED_FUNCTIONS" 37 | test_not_present "ZSTD_LEGACY_SUPPORT" 38 | test_not_present "STATIC_BMI2" 39 | test_not_present "ZSTD_DLL_EXPORT" 40 | test_not_present "ZSTD_DLL_IMPORT" 41 | test_not_present "__ICCARM__" 42 | test_not_present "_MSC_VER" 43 | test_not_present "_WIN32" 44 | test_not_present "__linux__" 45 | -------------------------------------------------------------------------------- /contrib/linux-kernel/zstd_common_module.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause 2 | /* 3 | * Copyright (c) Meta Platforms, Inc. and affiliates. 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include 13 | 14 | #include "common/huf.h" 15 | #include "common/fse.h" 16 | #include "common/zstd_internal.h" 17 | 18 | // Export symbols shared by compress and decompress into a common module 19 | 20 | #undef ZSTD_isError /* defined within zstd_internal.h */ 21 | EXPORT_SYMBOL_GPL(FSE_readNCount); 22 | EXPORT_SYMBOL_GPL(HUF_readStats); 23 | EXPORT_SYMBOL_GPL(HUF_readStats_wksp); 24 | EXPORT_SYMBOL_GPL(ZSTD_isError); 25 | EXPORT_SYMBOL_GPL(ZSTD_getErrorName); 26 | EXPORT_SYMBOL_GPL(ZSTD_getErrorCode); 27 | 28 | MODULE_LICENSE("Dual BSD/GPL"); 29 | MODULE_DESCRIPTION("Zstd Common"); 30 | -------------------------------------------------------------------------------- /contrib/premake/premake4.lua: -------------------------------------------------------------------------------- 1 | -- Include zstd.lua in your GENie or premake4 file, which exposes a project_zstd function 2 | dofile('zstd.lua') 3 | 4 | solution 'example' 5 | configurations { 'Debug', 'Release' } 6 | project_zstd('../../lib/') 7 | -------------------------------------------------------------------------------- /contrib/pzstd/.gitignore: -------------------------------------------------------------------------------- 1 | # compilation result 2 | pzstd 3 | -------------------------------------------------------------------------------- /contrib/pzstd/ErrorHolder.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | #pragma once 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | namespace pzstd { 17 | 18 | // Coordinates graceful shutdown of the pzstd pipeline 19 | class ErrorHolder { 20 | std::atomic error_; 21 | std::string message_; 22 | 23 | public: 24 | ErrorHolder() : error_(false) {} 25 | 26 | bool hasError() noexcept { 27 | return error_.load(); 28 | } 29 | 30 | void setError(std::string message) noexcept { 31 | // Given multiple possibly concurrent calls, exactly one will ever succeed. 32 | bool expected = false; 33 | if (error_.compare_exchange_strong(expected, true)) { 34 | message_ = std::move(message); 35 | } 36 | } 37 | 38 | bool check(bool predicate, std::string message) noexcept { 39 | if (!predicate) { 40 | setError(std::move(message)); 41 | } 42 | return !hasError(); 43 | } 44 | 45 | std::string getError() noexcept { 46 | error_.store(false); 47 | return std::move(message_); 48 | } 49 | 50 | ~ErrorHolder() { 51 | assert(!hasError()); 52 | } 53 | }; 54 | } 55 | -------------------------------------------------------------------------------- /contrib/pzstd/SkippableFrame.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | #include "SkippableFrame.h" 10 | #include "mem.h" 11 | #include "utils/Range.h" 12 | 13 | #include 14 | 15 | using namespace pzstd; 16 | 17 | SkippableFrame::SkippableFrame(std::uint32_t size) : frameSize_(size) { 18 | MEM_writeLE32(data_.data(), kSkippableFrameMagicNumber); 19 | MEM_writeLE32(data_.data() + 4, kFrameContentsSize); 20 | MEM_writeLE32(data_.data() + 8, frameSize_); 21 | } 22 | 23 | /* static */ std::size_t SkippableFrame::tryRead(ByteRange bytes) { 24 | if (bytes.size() < SkippableFrame::kSize || 25 | MEM_readLE32(bytes.begin()) != kSkippableFrameMagicNumber || 26 | MEM_readLE32(bytes.begin() + 4) != kFrameContentsSize) { 27 | return 0; 28 | } 29 | return MEM_readLE32(bytes.begin() + 8); 30 | } 31 | -------------------------------------------------------------------------------- /contrib/pzstd/images/Cspeed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/contrib/pzstd/images/Cspeed.png -------------------------------------------------------------------------------- /contrib/pzstd/images/Dspeed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/contrib/pzstd/images/Dspeed.png -------------------------------------------------------------------------------- /contrib/pzstd/main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | #include "ErrorHolder.h" 10 | #include "Options.h" 11 | #include "Pzstd.h" 12 | 13 | using namespace pzstd; 14 | 15 | int main(int argc, const char** argv) { 16 | Options options; 17 | switch (options.parse(argc, argv)) { 18 | case Options::Status::Failure: 19 | return 1; 20 | case Options::Status::Message: 21 | return 0; 22 | default: 23 | break; 24 | } 25 | 26 | return pzstdMain(options); 27 | } 28 | -------------------------------------------------------------------------------- /contrib/pzstd/test/BUCK: -------------------------------------------------------------------------------- 1 | cxx_test( 2 | name='options_test', 3 | srcs=['OptionsTest.cpp'], 4 | deps=['//contrib/pzstd:options'], 5 | ) 6 | 7 | cxx_test( 8 | name='pzstd_test', 9 | srcs=['PzstdTest.cpp'], 10 | deps=[ 11 | ':round_trip', 12 | '//contrib/pzstd:libpzstd', 13 | '//contrib/pzstd/utils:scope_guard', 14 | '//programs:datagen', 15 | ], 16 | ) 17 | 18 | cxx_binary( 19 | name='round_trip_test', 20 | srcs=['RoundTripTest.cpp'], 21 | deps=[ 22 | ':round_trip', 23 | '//contrib/pzstd/utils:scope_guard', 24 | '//programs:datagen', 25 | ] 26 | ) 27 | 28 | cxx_library( 29 | name='round_trip', 30 | header_namespace='test', 31 | exported_headers=['RoundTrip.h'], 32 | deps=[ 33 | '//contrib/pzstd:libpzstd', 34 | '//contrib/pzstd:options', 35 | '//contrib/pzstd/utils:scope_guard', 36 | ] 37 | ) 38 | -------------------------------------------------------------------------------- /contrib/pzstd/utils/Likely.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | 10 | /** 11 | * Compiler hints to indicate the fast path of an "if" branch: whether 12 | * the if condition is likely to be true or false. 13 | * 14 | * @author Tudor Bosman (tudorb@fb.com) 15 | */ 16 | 17 | #pragma once 18 | 19 | #undef LIKELY 20 | #undef UNLIKELY 21 | 22 | #if defined(__GNUC__) && __GNUC__ >= 4 23 | #define LIKELY(x) (__builtin_expect((x), 1)) 24 | #define UNLIKELY(x) (__builtin_expect((x), 0)) 25 | #else 26 | #define LIKELY(x) (x) 27 | #define UNLIKELY(x) (x) 28 | #endif 29 | -------------------------------------------------------------------------------- /contrib/pzstd/utils/Portability.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | 10 | #pragma once 11 | 12 | #include 13 | 14 | // Required for windows, which defines min/max, but we want the std:: version. 15 | #undef min 16 | #undef max 17 | -------------------------------------------------------------------------------- /contrib/pzstd/utils/ScopeGuard.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | #pragma once 10 | 11 | #include 12 | 13 | namespace pzstd { 14 | 15 | /** 16 | * Dismissable scope guard. 17 | * `Function` must be callable and take no parameters. 18 | * Unless `dismiss()` is called, the callable is executed upon destruction of 19 | * `ScopeGuard`. 20 | * 21 | * Example: 22 | * 23 | * auto guard = makeScopeGuard([&] { cleanup(); }); 24 | */ 25 | template 26 | class ScopeGuard { 27 | Function function; 28 | bool dismissed; 29 | 30 | public: 31 | explicit ScopeGuard(Function&& function) 32 | : function(std::move(function)), dismissed(false) {} 33 | 34 | void dismiss() { 35 | dismissed = true; 36 | } 37 | 38 | ~ScopeGuard() noexcept { 39 | if (!dismissed) { 40 | function(); 41 | } 42 | } 43 | }; 44 | 45 | /// Creates a scope guard from `function`. 46 | template 47 | ScopeGuard makeScopeGuard(Function&& function) { 48 | return ScopeGuard(std::forward(function)); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /contrib/pzstd/utils/test/BUCK: -------------------------------------------------------------------------------- 1 | cxx_test( 2 | name='buffer_test', 3 | srcs=['BufferTest.cpp'], 4 | deps=['//contrib/pzstd/utils:buffer'], 5 | ) 6 | 7 | cxx_test( 8 | name='range_test', 9 | srcs=['RangeTest.cpp'], 10 | deps=['//contrib/pzstd/utils:range'], 11 | ) 12 | 13 | cxx_test( 14 | name='resource_pool_test', 15 | srcs=['ResourcePoolTest.cpp'], 16 | deps=['//contrib/pzstd/utils:resource_pool'], 17 | ) 18 | 19 | cxx_test( 20 | name='scope_guard_test', 21 | srcs=['ScopeGuardTest.cpp'], 22 | deps=['//contrib/pzstd/utils:scope_guard'], 23 | ) 24 | 25 | cxx_test( 26 | name='thread_pool_test', 27 | srcs=['ThreadPoolTest.cpp'], 28 | deps=['//contrib/pzstd/utils:thread_pool'], 29 | ) 30 | 31 | cxx_test( 32 | name='work_queue_test', 33 | srcs=['RangeTest.cpp'], 34 | deps=['//contrib/pzstd/utils:work_queue'], 35 | ) 36 | -------------------------------------------------------------------------------- /contrib/pzstd/utils/test/ScopeGuardTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | #include "utils/ScopeGuard.h" 10 | 11 | #include 12 | 13 | using namespace pzstd; 14 | 15 | TEST(ScopeGuard, Dismiss) { 16 | { 17 | auto guard = makeScopeGuard([&] { EXPECT_TRUE(false); }); 18 | guard.dismiss(); 19 | } 20 | } 21 | 22 | TEST(ScopeGuard, Executes) { 23 | bool executed = false; 24 | { 25 | auto guard = makeScopeGuard([&] { executed = true; }); 26 | } 27 | EXPECT_TRUE(executed); 28 | } 29 | -------------------------------------------------------------------------------- /contrib/recovery/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | .PHONY: all 11 | all: recover_directory 12 | 13 | ZSTDLIBDIR ?= ../../lib 14 | PROGRAMDIR ?= ../../programs 15 | 16 | CFLAGS ?= -O3 17 | CFLAGS += -I$(ZSTDLIBDIR) -I$(PROGRAMDIR) 18 | CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ 19 | -Wstrict-aliasing=1 -Wswitch-enum \ 20 | -Wstrict-prototypes -Wundef \ 21 | -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ 22 | -Wredundant-decls -Wmissing-prototypes 23 | CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) 24 | FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) 25 | 26 | .PHONY: $(ZSTDLIBDIR)/libzstd.a 27 | $(ZSTDLIBDIR)/libzstd.a: 28 | $(MAKE) -C $(ZSTDLIBDIR) libzstd.a 29 | 30 | recover_directory: recover_directory.c $(ZSTDLIBDIR)/libzstd.a $(PROGRAMDIR)/util.c 31 | $(CC) $(FLAGS) $^ -o $@$(EXT) 32 | 33 | .PHONY: clean 34 | clean: 35 | rm -f recover_directory 36 | -------------------------------------------------------------------------------- /contrib/seekable_format/examples/.gitignore: -------------------------------------------------------------------------------- 1 | seekable_compression 2 | seekable_decompression 3 | seekable_decompression_mem 4 | parallel_processing 5 | parallel_compression 6 | -------------------------------------------------------------------------------- /contrib/seekable_format/tests/.gitignore: -------------------------------------------------------------------------------- 1 | seekable_tests 2 | data.txt 3 | -------------------------------------------------------------------------------- /contrib/snap/snapcraft.yaml: -------------------------------------------------------------------------------- 1 | name: zstd 2 | version: git 3 | summary: Zstandard - Fast real-time compression algorithm 4 | description: | 5 | Zstandard, or zstd as short version, is a fast lossless compression 6 | algorithm, targeting real-time compression scenarios at zlib-level and better 7 | compression ratios. It's backed by a very fast entropy stage, provided by 8 | Huff0 and FSE library 9 | 10 | grade: devel # must be 'stable' to release into candidate/stable channels 11 | confinement: devmode # use 'strict' once you have the right plugs and slots 12 | 13 | apps: 14 | zstd: 15 | command: usr/local/bin/zstd 16 | plugs: [home, removable-media] 17 | zstdgrep: 18 | command: usr/local/bin/zstdgrep 19 | plugs: [home, removable-media] 20 | zstdless: 21 | command: usr/local/bin/zstdless 22 | plugs: [home, removable-media] 23 | 24 | parts: 25 | zstd: 26 | source: . 27 | plugin: make 28 | build-packages: [g++] 29 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | Zstandard Documentation 2 | ======================= 3 | 4 | This directory contains material defining the Zstandard format, 5 | as well as detailed instructions to use `zstd` library. 6 | 7 | __`zstd_manual.html`__ : Documentation of `zstd.h` API, in html format. 8 | Unfortunately, Github doesn't display `html` files in parsed format, just as source code. 9 | For a readable display of html API documentation of latest release, 10 | use this link: [https://raw.githack.com/facebook/zstd/release/doc/zstd_manual.html](https://raw.githack.com/facebook/zstd/release/doc/zstd_manual.html) . 11 | 12 | __`zstd_compression_format.md`__ : This document defines the Zstandard compression format. 13 | Compliant decoders must adhere to this document, 14 | and compliant encoders must generate data that follows it. 15 | 16 | Should you look for resources to develop your own port of Zstandard algorithm, 17 | you may find the following resources useful : 18 | 19 | __`educational_decoder`__ : This directory contains an implementation of a Zstandard decoder, 20 | compliant with the Zstandard compression format. 21 | It can be used, for example, to better understand the format, 22 | or as the basis for a separate implementation of Zstandard decoder. 23 | 24 | [__`decode_corpus`__](https://github.com/facebook/zstd/tree/dev/tests#decodecorpus---tool-to-generate-zstandard-frames-for-decoder-testing) : 25 | This tool, stored in `/tests` directory, is able to generate random valid frames, 26 | which is useful if you wish to test your decoder and verify it fully supports the specification. 27 | -------------------------------------------------------------------------------- /doc/educational_decoder/.gitignore: -------------------------------------------------------------------------------- 1 | # Build artifacts 2 | harness 3 | -------------------------------------------------------------------------------- /doc/images/CSpeed2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/doc/images/CSpeed2.png -------------------------------------------------------------------------------- /doc/images/DCspeed5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/doc/images/DCspeed5.png -------------------------------------------------------------------------------- /doc/images/DSpeed3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/doc/images/DSpeed3.png -------------------------------------------------------------------------------- /doc/images/cdict_v136.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/doc/images/cdict_v136.png -------------------------------------------------------------------------------- /doc/images/dict-cr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/doc/images/dict-cr.png -------------------------------------------------------------------------------- /doc/images/dict-cs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/doc/images/dict-cs.png -------------------------------------------------------------------------------- /doc/images/dict-ds.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/doc/images/dict-ds.png -------------------------------------------------------------------------------- /doc/images/zstd_cdict_v1_3_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/doc/images/zstd_cdict_v1_3_5.png -------------------------------------------------------------------------------- /doc/images/zstd_logo86.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/doc/images/zstd_logo86.png -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | #build 2 | simple_compression 3 | simple_decompression 4 | multiple_simple_compression 5 | dictionary_compression 6 | dictionary_decompression 7 | streaming_compression 8 | streaming_decompression 9 | multiple_streaming_compression 10 | streaming_memory_usage 11 | 12 | #test artefact 13 | tmp* 14 | test* 15 | *.zst 16 | -------------------------------------------------------------------------------- /lib/.gitignore: -------------------------------------------------------------------------------- 1 | # make install artefact 2 | libzstd.pc 3 | libzstd-nomt 4 | -------------------------------------------------------------------------------- /lib/common/debug.c: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * debug 3 | * Part of FSE library 4 | * Copyright (c) Meta Platforms, Inc. and affiliates. 5 | * 6 | * You can contact the author at : 7 | * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 8 | * 9 | * This source code is licensed under both the BSD-style license (found in the 10 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 11 | * in the COPYING file in the root directory of this source tree). 12 | * You may select, at your option, one of the above-listed licenses. 13 | ****************************************************************** */ 14 | 15 | 16 | /* 17 | * This module only hosts one global variable 18 | * which can be used to dynamically influence the verbosity of traces, 19 | * such as DEBUGLOG and RAWLOG 20 | */ 21 | 22 | #include "debug.h" 23 | 24 | #if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2) 25 | /* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a 26 | * translation unit is empty. So remove this from Linux kernel builds, but 27 | * otherwise just leave it in. 28 | */ 29 | int g_debuglevel = DEBUGLEVEL; 30 | #endif 31 | -------------------------------------------------------------------------------- /lib/common/xxhash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xxHash - Extremely Fast Hash algorithm 3 | * Copyright (c) Yann Collet - Meta Platforms, Inc 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* 12 | * xxhash.c instantiates functions defined in xxhash.h 13 | */ 14 | 15 | #define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ 16 | #define XXH_IMPLEMENTATION /* access definitions */ 17 | 18 | #include "xxhash.h" 19 | -------------------------------------------------------------------------------- /lib/compress/zstd_compress_superblock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_COMPRESS_ADVANCED_H 12 | #define ZSTD_COMPRESS_ADVANCED_H 13 | 14 | /*-************************************* 15 | * Dependencies 16 | ***************************************/ 17 | 18 | #include "../zstd.h" /* ZSTD_CCtx */ 19 | 20 | /*-************************************* 21 | * Target Compressed Block Size 22 | ***************************************/ 23 | 24 | /* ZSTD_compressSuperBlock() : 25 | * Used to compress a super block when targetCBlockSize is being used. 26 | * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */ 27 | size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, 28 | void* dst, size_t dstCapacity, 29 | void const* src, size_t srcSize, 30 | unsigned lastBlock); 31 | 32 | #endif /* ZSTD_COMPRESS_ADVANCED_H */ 33 | -------------------------------------------------------------------------------- /lib/compress/zstd_fast.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_FAST_H 12 | #define ZSTD_FAST_H 13 | 14 | #include "../common/mem.h" /* U32 */ 15 | #include "zstd_compress_internal.h" 16 | 17 | void ZSTD_fillHashTable(ZSTD_MatchState_t* ms, 18 | void const* end, ZSTD_dictTableLoadMethod_e dtlm, 19 | ZSTD_tableFillPurpose_e tfp); 20 | size_t ZSTD_compressBlock_fast( 21 | ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 22 | void const* src, size_t srcSize); 23 | size_t ZSTD_compressBlock_fast_dictMatchState( 24 | ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 25 | void const* src, size_t srcSize); 26 | size_t ZSTD_compressBlock_fast_extDict( 27 | ZSTD_MatchState_t* ms, SeqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 28 | void const* src, size_t srcSize); 29 | 30 | #endif /* ZSTD_FAST_H */ 31 | -------------------------------------------------------------------------------- /lib/compress/zstd_preSplit.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_PRESPLIT_H 12 | #define ZSTD_PRESPLIT_H 13 | 14 | #include /* size_t */ 15 | 16 | #define ZSTD_SLIPBLOCK_WORKSPACESIZE 8208 17 | 18 | /* ZSTD_splitBlock(): 19 | * @level must be a value between 0 and 4. 20 | * higher levels spend more energy to detect block boundaries. 21 | * @workspace must be aligned for size_t. 22 | * @wkspSize must be at least >= ZSTD_SLIPBLOCK_WORKSPACESIZE 23 | * note: 24 | * For the time being, this function only accepts full 128 KB blocks. 25 | * Therefore, @blockSize must be == 128 KB. 26 | * While this could be extended to smaller sizes in the future, 27 | * it is not yet clear if this would be useful. TBD. 28 | */ 29 | size_t ZSTD_splitBlock(const void* blockStart, size_t blockSize, 30 | int level, 31 | void* workspace, size_t wkspSize); 32 | 33 | #endif /* ZSTD_PRESPLIT_H */ 34 | -------------------------------------------------------------------------------- /lib/decompress/zstd_ddict.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | 12 | #ifndef ZSTD_DDICT_H 13 | #define ZSTD_DDICT_H 14 | 15 | /*-******************************************************* 16 | * Dependencies 17 | *********************************************************/ 18 | #include "../common/zstd_deps.h" /* size_t */ 19 | #include "../zstd.h" /* ZSTD_DDict, and several public functions */ 20 | 21 | 22 | /*-******************************************************* 23 | * Interface 24 | *********************************************************/ 25 | 26 | /* note: several prototypes are already published in `zstd.h` : 27 | * ZSTD_createDDict() 28 | * ZSTD_createDDict_byReference() 29 | * ZSTD_createDDict_advanced() 30 | * ZSTD_freeDDict() 31 | * ZSTD_initStaticDDict() 32 | * ZSTD_sizeof_DDict() 33 | * ZSTD_estimateDDictSize() 34 | * ZSTD_getDictID_fromDict() 35 | */ 36 | 37 | const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); 38 | size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); 39 | 40 | void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); 41 | 42 | 43 | 44 | #endif /* ZSTD_DDICT_H */ 45 | -------------------------------------------------------------------------------- /lib/deprecated/zbuff_common.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /*-************************************* 12 | * Dependencies 13 | ***************************************/ 14 | #include "../common/error_private.h" 15 | #include "zbuff.h" 16 | 17 | /*-**************************************** 18 | * ZBUFF Error Management (deprecated) 19 | ******************************************/ 20 | 21 | /*! ZBUFF_isError() : 22 | * tells if a return value is an error code */ 23 | unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } 24 | /*! ZBUFF_getErrorName() : 25 | * provides error code string from function result (useful for debugging) */ 26 | const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } 27 | -------------------------------------------------------------------------------- /lib/dll/example/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # You may select, at your option, one of the above-listed licenses. 9 | # ################################################################ 10 | 11 | VOID := /dev/null 12 | ZSTDDIR := ../include 13 | LIBDIR := ../static 14 | DLLDIR := ../dll 15 | 16 | CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make 17 | CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ 18 | -Wdeclaration-after-statement -Wstrict-prototypes \ 19 | -Wpointer-arith -Wstrict-aliasing=1 20 | CFLAGS += $(MOREFLAGS) 21 | CPPFLAGS:= -I$(ZSTDDIR) -DXXH_NAMESPACE=ZSTD_ 22 | FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) 23 | 24 | 25 | # Define *.exe as extension for Windows systems 26 | ifneq (,$(filter Windows%,$(OS))) 27 | EXT =.exe 28 | else 29 | EXT = 30 | endif 31 | 32 | .PHONY: default fullbench-dll fullbench-lib 33 | 34 | 35 | default: all 36 | 37 | all: fullbench-dll fullbench-lib 38 | 39 | 40 | fullbench-lib: fullbench.c datagen.c 41 | $(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/libzstd_static.lib 42 | 43 | fullbench-dll: fullbench.c datagen.c 44 | $(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(DLLDIR)/libzstd.dll 45 | 46 | clean: 47 | @$(RM) fullbench-dll$(EXT) fullbench-lib$(EXT) \ 48 | @echo Cleaning completed 49 | -------------------------------------------------------------------------------- /lib/dll/example/fullbench-dll.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio Express 2012 for Windows Desktop 3 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}" 4 | EndProject 5 | Global 6 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 7 | Debug|Win32 = Debug|Win32 8 | Debug|x64 = Debug|x64 9 | Release|Win32 = Release|Win32 10 | Release|x64 = Release|x64 11 | EndGlobalSection 12 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 13 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32 14 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32 15 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64 16 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64 17 | {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32 18 | {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32 19 | {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64 20 | {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64 21 | EndGlobalSection 22 | GlobalSection(SolutionProperties) = preSolution 23 | HideSolutionNode = FALSE 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /lib/install_oses.mk: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # You may select, at your option, one of the above-listed licenses. 9 | # ################################################################ 10 | 11 | # This included Makefile provides the following variables : 12 | # UNAME, INSTALL_OS_LIST 13 | 14 | UNAME := $(shell sh -c 'MSYSTEM="MSYS" uname') 15 | 16 | # List of OSes for which target install is supported 17 | INSTALL_OS_LIST ?= Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku AIX MSYS_NT% CYGWIN_NT% 18 | -------------------------------------------------------------------------------- /lib/libzstd.pc.in: -------------------------------------------------------------------------------- 1 | # ZSTD - standard compression algorithm 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) 4 | 5 | prefix=@PREFIX@ 6 | exec_prefix=@EXEC_PREFIX@ 7 | includedir=@INCLUDEDIR@ 8 | libdir=@LIBDIR@ 9 | 10 | Name: zstd 11 | Description: fast lossless compression algorithm library 12 | URL: https://facebook.github.io/zstd/ 13 | Version: @VERSION@ 14 | Libs: -L${libdir} -lzstd @LIBS_MT@ 15 | Libs.private: @LIBS_PRIVATE@ 16 | Cflags: -I${includedir} @LIBS_MT@ 17 | License: BSD-3-Clause OR GPL-2.0-only 18 | -------------------------------------------------------------------------------- /lib/module.modulemap: -------------------------------------------------------------------------------- 1 | module libzstd [extern_c] { 2 | header "zstd.h" 3 | export * 4 | config_macros [exhaustive] \ 5 | /* zstd.h */ \ 6 | ZSTD_STATIC_LINKING_ONLY, \ 7 | ZSTDLIB_VISIBILITY, \ 8 | ZSTDLIB_VISIBLE, \ 9 | ZSTDLIB_HIDDEN, \ 10 | ZSTD_DLL_EXPORT, \ 11 | ZSTDLIB_STATIC_API, \ 12 | ZSTD_DISABLE_DEPRECATE_WARNINGS, \ 13 | ZSTD_CLEVEL_DEFAULT, \ 14 | /* zdict.h */ \ 15 | ZDICT_STATIC_LINKING_ONLY, \ 16 | ZDICTLIB_VISIBLE, \ 17 | ZDICTLIB_HIDDEN, \ 18 | ZDICTLIB_VISIBILITY, \ 19 | ZDICTLIB_STATIC_API, \ 20 | ZDICT_DISABLE_DEPRECATE_WARNINGS, \ 21 | /* zstd_errors.h */ \ 22 | ZSTDERRORLIB_VISIBLE, \ 23 | ZSTDERRORLIB_HIDDEN, \ 24 | ZSTDERRORLIB_VISIBILITY 25 | 26 | module dictbuilder [extern_c] { 27 | header "zdict.h" 28 | export * 29 | } 30 | 31 | module errors [extern_c] { 32 | header "zstd_errors.h" 33 | export * 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /programs/.gitignore: -------------------------------------------------------------------------------- 1 | # local binary (Makefile) 2 | zstd 3 | zstd32 4 | zstd4 5 | zstd-compress 6 | zstd-decompress 7 | zstd-frugal 8 | zstd-small 9 | zstd-nolegacy 10 | zstd-dictBuilder 11 | zstd-dll 12 | zstd_arm64 13 | zstd_x64 14 | 15 | # Object files 16 | *.o 17 | *.ko 18 | default.profraw 19 | have_zlib 20 | 21 | # Executables 22 | *.exe 23 | *.out 24 | *.app 25 | 26 | # Default result files 27 | dictionary 28 | grillResults.txt 29 | _* 30 | tmp* 31 | *.zst 32 | result 33 | out 34 | 35 | # fuzzer 36 | afl 37 | 38 | # Misc files 39 | *.bat 40 | !windres/generate_res.bat 41 | dirTest* 42 | -------------------------------------------------------------------------------- /programs/BUCK: -------------------------------------------------------------------------------- 1 | cxx_binary( 2 | name='zstd', 3 | headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']), 4 | srcs=glob(['*.c'], excludes=['datagen.c']), 5 | deps=[ 6 | ':datagen', 7 | ':util', 8 | '//lib:zstd', 9 | '//lib:zdict', 10 | '//lib:mem', 11 | '//lib:xxhash', 12 | ], 13 | preprocessor_flags=[ 14 | '-DZSTD_GZCOMPRESS', 15 | '-DZSTD_GZDECOMPRESS', 16 | '-DZSTD_LZMACOMPRESS', 17 | '-DZSTD_LZMADECOMPRES', 18 | '-DZSTD_LZ4COMPRESS', 19 | '-DZSTD_LZ4DECOMPRES', 20 | ], 21 | linker_flags=[ 22 | '-lz', 23 | '-llzma', 24 | '-llz4', 25 | ], 26 | ) 27 | 28 | cxx_library( 29 | name='datagen', 30 | visibility=['PUBLIC'], 31 | header_namespace='', 32 | exported_headers=['datagen.h'], 33 | srcs=['datagen.c'], 34 | deps=['//lib:mem'], 35 | ) 36 | 37 | 38 | cxx_library( 39 | name='util', 40 | visibility=['PUBLIC'], 41 | header_namespace='', 42 | exported_headers=['util.h', 'platform.h'], 43 | deps=['//lib:mem'], 44 | ) 45 | -------------------------------------------------------------------------------- /programs/datagen.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | 12 | #ifndef DATAGEN_H 13 | #define DATAGEN_H 14 | 15 | #include /* size_t */ 16 | 17 | #if defined (__cplusplus) 18 | extern "C" { 19 | #endif 20 | 21 | void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed); 22 | void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed); 23 | /*!RDG_genBuffer 24 | Generate 'size' bytes of compressible data into 'buffer'. 25 | Compressibility can be controlled using 'matchProba', which is floating point value between 0 and 1. 26 | 'LitProba' is optional, it affect variability of individual bytes. If litProba==0.0, default value will be used. 27 | Generated data pattern can be modified using different 'seed'. 28 | For a triplet (matchProba, litProba, seed), the function always generate the same content. 29 | 30 | RDG_genStdout 31 | Same as RDG_genBuffer, but generates data into stdout 32 | */ 33 | 34 | #if defined (__cplusplus) 35 | } /* extern "C" */ 36 | #endif 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /programs/dibio.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* This library is designed for a single-threaded console application. 12 | * It exit() and printf() into stderr when it encounters an error condition. */ 13 | 14 | #ifndef DIBIO_H_003 15 | #define DIBIO_H_003 16 | 17 | 18 | /*-************************************* 19 | * Dependencies 20 | ***************************************/ 21 | #define ZDICT_STATIC_LINKING_ONLY 22 | #include "../lib/zdict.h" /* ZDICT_params_t */ 23 | 24 | 25 | /*-************************************* 26 | * Public functions 27 | ***************************************/ 28 | /*! DiB_trainFromFiles() : 29 | Train a dictionary from a set of files provided by `fileNamesTable`. 30 | Resulting dictionary is written into file `dictFileName`. 31 | `parameters` is optional and can be provided with values set to 0, meaning "default". 32 | @return : 0 == ok. Any other : error. 33 | */ 34 | int DiB_trainFromFiles(const char* dictFileName, size_t maxDictSize, 35 | const char** fileNamesTable, int nbFiles, size_t chunkSize, 36 | ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams, 37 | ZDICT_fastCover_params_t* fastCoverParams, int optimize, unsigned memLimit); 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /programs/lorem.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* lorem ipsum generator */ 12 | 13 | #include /* size_t */ 14 | 15 | /* 16 | * LOREM_genBuffer(): 17 | * Generate @size bytes of compressible data using lorem ipsum generator 18 | * into provided @buffer. 19 | */ 20 | void LOREM_genBuffer(void* buffer, size_t size, unsigned seed); 21 | 22 | /* 23 | * LOREM_genBlock(): 24 | * Similar to LOREM_genBuffer, with additional controls : 25 | * - @first : generate the first sentence 26 | * - @fill : fill the entire @buffer, 27 | * if ==0: generate one paragraph at most. 28 | * @return : nb of bytes generated into @buffer. 29 | */ 30 | size_t LOREM_genBlock(void* buffer, size_t size, 31 | unsigned seed, 32 | int first, int fill); 33 | -------------------------------------------------------------------------------- /programs/windres/verrsrc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | /* minimal set of defines required to generate zstd.res from zstd.rc */ 11 | 12 | #define VS_VERSION_INFO 1 13 | 14 | #define VS_FFI_FILEFLAGSMASK 0x0000003FL 15 | #define VOS_NT_WINDOWS32 0x00040004L 16 | #define VFT_DLL 0x00000002L 17 | #define VFT2_UNKNOWN 0x00000000L 18 | -------------------------------------------------------------------------------- /programs/windres/zstd.rc: -------------------------------------------------------------------------------- 1 | // Microsoft Visual C++ generated resource script. 2 | // 3 | 4 | #include "zstd.h" /* ZSTD_VERSION_STRING */ 5 | #define APSTUDIO_READONLY_SYMBOLS 6 | #include "verrsrc.h" 7 | #undef APSTUDIO_READONLY_SYMBOLS 8 | 9 | 10 | #if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) 11 | LANGUAGE 9, 1 12 | 13 | ///////////////////////////////////////////////////////////////////////////// 14 | // 15 | // Version 16 | // 17 | 18 | VS_VERSION_INFO VERSIONINFO 19 | FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 20 | PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 21 | FILEFLAGSMASK VS_FFI_FILEFLAGSMASK 22 | #ifdef _DEBUG 23 | FILEFLAGS VS_FF_DEBUG 24 | #else 25 | FILEFLAGS 0x0L 26 | #endif 27 | FILEOS VOS_NT_WINDOWS32 28 | FILETYPE VFT_DLL 29 | FILESUBTYPE VFT2_UNKNOWN 30 | BEGIN 31 | BLOCK "StringFileInfo" 32 | BEGIN 33 | BLOCK "040904B0" 34 | BEGIN 35 | VALUE "CompanyName", "Meta Platforms, Inc." 36 | VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm" 37 | VALUE "FileVersion", ZSTD_VERSION_STRING 38 | VALUE "InternalName", "zstd.exe" 39 | VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates." 40 | VALUE "OriginalFilename", "zstd.exe" 41 | VALUE "ProductName", "Zstandard" 42 | VALUE "ProductVersion", ZSTD_VERSION_STRING 43 | END 44 | END 45 | BLOCK "VarFileInfo" 46 | BEGIN 47 | VALUE "Translation", 0x0409, 1200 48 | END 49 | END 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /programs/windres/zstd32.res: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/programs/windres/zstd32.res -------------------------------------------------------------------------------- /programs/windres/zstd64.res: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/programs/windres/zstd64.res -------------------------------------------------------------------------------- /programs/zstdcli_trace.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTDCLI_TRACE_H 12 | #define ZSTDCLI_TRACE_H 13 | 14 | /** 15 | * Enable tracing - log to filename. 16 | */ 17 | void TRACE_enable(char const* filename); 18 | 19 | /** 20 | * Shut down the tracing library. 21 | */ 22 | void TRACE_finish(void); 23 | 24 | #endif /* ZSTDCLI_TRACE_H */ 25 | -------------------------------------------------------------------------------- /programs/zstdgrep.1: -------------------------------------------------------------------------------- 1 | . 2 | .TH "ZSTDGREP" "1" "February 2025" "zstd 1.5.7" "User Commands" 3 | . 4 | .SH "NAME" 5 | \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files 6 | . 7 | .SH "SYNOPSIS" 8 | \fBzstdgrep\fR [\fIgrep\-flags\fR] [\-\-] \fIpattern\fR [\fIfiles\fR \.\.\.] 9 | . 10 | .SH "DESCRIPTION" 11 | \fBzstdgrep\fR runs \fBgrep\fR(1) on files, or \fBstdin\fR if no files argument is given, after decompressing them with \fBzstdcat\fR(1)\. 12 | . 13 | .P 14 | The \fIgrep\-flags\fR and \fIpattern\fR arguments are passed on to \fBgrep\fR(1)\. If an \fB\-e\fR flag is found in the \fIgrep\-flags\fR, \fBzstdgrep\fR will not look for a \fIpattern\fR argument\. 15 | . 16 | .P 17 | Note that modern \fBgrep\fR alternatives such as \fBripgrep\fR (\fBrg\fR(1)) support \fBzstd\fR\-compressed files out of the box, and can prove better alternatives than \fBzstdgrep\fR notably for unsupported complex pattern searches\. Note though that such alternatives may also feature some minor command line differences\. 18 | . 19 | .SH "EXIT STATUS" 20 | In case of missing arguments or missing pattern, 1 will be returned, otherwise 0\. 21 | . 22 | .SH "SEE ALSO" 23 | \fBzstd\fR(1) 24 | . 25 | .SH "AUTHORS" 26 | Thomas Klausner \fIwiz@NetBSD\.org\fR 27 | -------------------------------------------------------------------------------- /programs/zstdgrep.1.md: -------------------------------------------------------------------------------- 1 | zstdgrep(1) -- print lines matching a pattern in zstandard-compressed files 2 | ============================================================================ 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | `zstdgrep` [] [--] [ ...] 8 | 9 | 10 | DESCRIPTION 11 | ----------- 12 | `zstdgrep` runs `grep`(1) on files, or `stdin` if no files argument is given, after decompressing them with `zstdcat`(1). 13 | 14 | The and arguments are passed on to `grep`(1). If an `-e` flag is found in the , `zstdgrep` will not look for a argument. 15 | 16 | Note that modern `grep` alternatives such as `ripgrep` (`rg`(1)) support `zstd`-compressed files out of the box, 17 | and can prove better alternatives than `zstdgrep` notably for unsupported complex pattern searches. 18 | Note though that such alternatives may also feature some minor command line differences. 19 | 20 | EXIT STATUS 21 | ----------- 22 | In case of missing arguments or missing pattern, 1 will be returned, otherwise 0. 23 | 24 | SEE ALSO 25 | -------- 26 | `zstd`(1) 27 | 28 | AUTHORS 29 | ------- 30 | Thomas Klausner 31 | -------------------------------------------------------------------------------- /programs/zstdless: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | zstd=${ZSTD:-zstd} 4 | 5 | # TODO: Address quirks and bugs tied to old versions of less, provide a mechanism to pass flags directly to zstd 6 | 7 | export LESSOPEN="|-${zstd} -cdfq %s" 8 | exec less "$@" 9 | -------------------------------------------------------------------------------- /programs/zstdless.1: -------------------------------------------------------------------------------- 1 | . 2 | .TH "ZSTDLESS" "1" "February 2025" "zstd 1.5.7" "User Commands" 3 | . 4 | .SH "NAME" 5 | \fBzstdless\fR \- view zstandard\-compressed files 6 | . 7 | .SH "SYNOPSIS" 8 | \fBzstdless\fR [\fIflags\fR] [\fIfile\fR \.\.\.] 9 | . 10 | .SH "DESCRIPTION" 11 | \fBzstdless\fR runs \fBless\fR(1) on files or stdin, if no \fIfile\fR argument is given, after decompressing them with \fBzstdcat\fR(1)\. 12 | . 13 | .SH "SEE ALSO" 14 | \fBzstd\fR(1) 15 | -------------------------------------------------------------------------------- /programs/zstdless.1.md: -------------------------------------------------------------------------------- 1 | zstdless(1) -- view zstandard-compressed files 2 | ============================================================================ 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | `zstdless` [] [ ...] 8 | 9 | 10 | DESCRIPTION 11 | ----------- 12 | `zstdless` runs `less`(1) on files or stdin, if no argument is given, after decompressing them with `zstdcat`(1). 13 | 14 | SEE ALSO 15 | -------- 16 | `zstd`(1) 17 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | # local binary (Makefile) 2 | fullbench 3 | fullbench32 4 | fullbench-lib 5 | fuzzer 6 | fuzzer32 7 | fuzzer-dll 8 | largeDictionary 9 | zbufftest 10 | zbufftest32 11 | zbufftest-dll 12 | zstreamtest 13 | zstreamtest32 14 | zstreamtest_asan 15 | zstreamtest_tsan 16 | zstreamtest_ubsan 17 | zstreamtest-dll 18 | datagen 19 | paramgrill 20 | paramgrill32 21 | roundTripCrash 22 | longmatch 23 | symbols 24 | legacy 25 | decodecorpus 26 | pool 27 | poolTests 28 | invalidDictionaries 29 | checkTag 30 | zcat 31 | zstdcat 32 | tm 33 | 34 | # test artifacts 35 | dictionary 36 | grillResults.txt 37 | _* 38 | tmp* 39 | *.zst 40 | *.gz 41 | !gzip/hufts-segv.gz 42 | result 43 | out 44 | *.zstd 45 | hello* 46 | world 47 | 48 | # Tmp test directory 49 | zstdtest 50 | speedTest 51 | versionsTest 52 | namespaceTest 53 | dirTest* 54 | 55 | # fuzzer 56 | afl 57 | 58 | # Local script 59 | startSpeedTest 60 | speedTest.pid 61 | *.bat 62 | 63 | # Generic Object files 64 | *.o 65 | *.ko 66 | 67 | # Generic Executables 68 | *.exe 69 | *.out 70 | *.app 71 | 72 | # Specific exclusions 73 | !golden-decompression/*.zst 74 | -------------------------------------------------------------------------------- /tests/check_size.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # ################################################################ 3 | # Copyright (c) Meta Platforms, Inc. and affiliates. 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # You may select, at your option, one of the above-listed licenses. 10 | # ################################################################ 11 | 12 | import os 13 | import subprocess 14 | import sys 15 | 16 | if len(sys.argv) != 3: 17 | print(f"Usage: {sys.argv[0]} FILE SIZE_LIMIT") 18 | sys.exit(1) 19 | 20 | file = sys.argv[1] 21 | limit = int(sys.argv[2]) 22 | 23 | if not os.path.exists(file): 24 | print(f"{file} does not exist") 25 | sys.exit(1) 26 | 27 | size = os.path.getsize(file) 28 | 29 | if size > limit: 30 | print(f"file {file} is {size} bytes, which is greater than the limit of {limit} bytes") 31 | sys.exit(1) 32 | -------------------------------------------------------------------------------- /tests/cli-tests/.gitignore: -------------------------------------------------------------------------------- 1 | !bin/ 2 | !datagen 3 | !zstdcat 4 | 5 | scratch/ 6 | bin/symlinks 7 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/args.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | println "+ zstd --blah" >&2 4 | zstd --blah 5 | println "+ zstd -xz" >&2 6 | zstd -xz 7 | println "+ zstd --adapt=min=1,maxx=2 file.txt" >&2 8 | zstd --adapt=min=1,maxx=2 file.txt 9 | println "+ zstd --train-cover=k=48,d=8,steps32 file.txt" >&2 10 | zstd --train-cover=k=48,d=8,steps32 file.txt 11 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/args.sh.exit: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/args.sh.stderr.glob: -------------------------------------------------------------------------------- 1 | + zstd --blah 2 | Incorrect parameter: --blah 3 | ... 4 | Usage: zstd * 5 | 6 | Options: 7 | ... 8 | + zstd -xz 9 | Incorrect parameter: -x 10 | ... 11 | Usage: zstd * 12 | 13 | Options: 14 | ... 15 | + zstd --adapt=min=1,maxx=2 file.txt 16 | Incorrect parameter: --adapt=min=1,maxx=2 17 | ... 18 | Usage: zstd * 19 | 20 | Options: 21 | ... 22 | + zstd --train-cover=k=48,d=8,steps32 file.txt 23 | Incorrect parameter: --train-cover=k=48,d=8,steps32 24 | ... 25 | Usage: zstd * 26 | 27 | Options: 28 | ... 29 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/help.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | println "+ zstd -h" 6 | zstd -h 7 | println "+ zstd -H" 8 | zstd -H 9 | println "+ zstd --help" 10 | zstd --help 11 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/help.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | + zstd -h 2 | Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided. 3 | 4 | Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT* 5 | 6 | Options: 7 | -o OUTPUT Write output to a single file, OUTPUT. 8 | -k, --keep Preserve INPUT file(s). *Default* 9 | --rm Remove INPUT file(s) after successful (de)compression to file. 10 | 11 | -# Desired compression level, where `#` is a number between 1 and 19; 12 | lower numbers provide faster compression, higher numbers yield 13 | better compression ratios. *Default: 3* 14 | 15 | -d, --decompress Perform decompression. 16 | -D DICT Use DICT as the dictionary for compression or decompression. 17 | 18 | -f, --force Disable input and output checks. Allows overwriting existing files, 19 | receiving input from the console, printing output to STDOUT, and 20 | operating on links, block devices, etc. Unrecognized formats will be 21 | passed-through through as-is. 22 | 23 | -h Display short usage and exit. 24 | -H, --help Display full help and exit. 25 | -V, --version Display the program version and exit. 26 | 27 | + zstd -H 28 | ... 29 | Advanced options: 30 | ... 31 | + zstd --help 32 | ... 33 | Advanced options: 34 | ... 35 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/memlimit.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 2 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 3 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 4 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 5 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 6 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 7 | Should allow numeric parameter without suffix 8 | Should allow numeric parameter with expected suffix 9 | Should allow numeric parameter with expected suffix 10 | Should allow numeric parameter with expected suffix 11 | Should allow numeric parameter with expected suffix 12 | Should allow numeric parameter with expected suffix 13 | Should allow numeric parameter with expected suffix 14 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/memlimit.sh.stdout.exact: -------------------------------------------------------------------------------- 1 | + zstd --memory=32LB file 2 | + zstd --memory=32LiB file 3 | + zstd --memory=32A file 4 | + zstd --memory=32r82347dn83 file 5 | + zstd --memory=32asbdf file 6 | + zstd --memory=hello file 7 | + zstd --memory=1 file 8 | + zstd --memory=1K file 9 | + zstd --memory=1KB file 10 | + zstd --memory=1KiB file 11 | + zstd --memory=1M file 12 | + zstd --memory=1MB file 13 | + zstd --memory=1MiB file 14 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/output_dir.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | println "+ zstd -r * --output-dir-mirror=\"\"" 4 | zstd -r * --output-dir-mirror="" && die "Should not allow empty output dir!" 5 | println "+ zstd -r * --output-dir-flat=\"\"" 6 | zstd -r * --output-dir-flat="" && die "Should not allow empty output dir!" 7 | exit 0 8 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/output_dir.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | error: output dir cannot be empty string (did you mean to pass '.' instead?) 2 | error: output dir cannot be empty string (did you mean to pass '.' instead?) 3 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/output_dir.sh.stdout.exact: -------------------------------------------------------------------------------- 1 | + zstd -r * --output-dir-mirror="" 2 | + zstd -r * --output-dir-flat="" 3 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | zstd -V 6 | zstd --version 7 | -------------------------------------------------------------------------------- /tests/cli-tests/basic/version.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | *** Zstandard CLI (*-bit) v1.*.*, by Yann Collet *** 2 | *** Zstandard CLI (*-bit) v1.*.*, by Yann Collet *** 3 | -------------------------------------------------------------------------------- /tests/cli-tests/bin/cmp_size: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | usage() 6 | { 7 | printf "USAGE:\n\t$0 [-eq|-ne|-lt|-le|-gt|-ge] FILE1 FILE2\n" 8 | } 9 | 10 | help() 11 | { 12 | printf "Small utility to compare file sizes without printing them with set -x.\n\n" 13 | usage 14 | } 15 | 16 | case "$1" in 17 | -h) help; exit 0 ;; 18 | --help) help; exit 0 ;; 19 | esac 20 | 21 | if ! test -f $2; then 22 | printf "FILE1='%b' is not a file\n\n" "$2" 23 | usage 24 | exit 1 25 | fi 26 | 27 | if ! test -f $3; then 28 | printf "FILE2='%b' is not a file\n\n" "$3" 29 | usage 30 | exit 1 31 | fi 32 | 33 | 34 | size1=$(wc -c < $2) 35 | size2=$(wc -c < $3) 36 | 37 | case "$1" in 38 | -eq) [ "$size1" -eq "$size2" ] ;; 39 | -ne) [ "$size1" -ne "$size2" ] ;; 40 | -lt) [ "$size1" -lt "$size2" ] ;; 41 | -le) [ "$size1" -le "$size2" ] ;; 42 | -gt) [ "$size1" -gt "$size2" ] ;; 43 | -ge) [ "$size1" -ge "$size2" ] ;; 44 | esac 45 | -------------------------------------------------------------------------------- /tests/cli-tests/bin/datagen: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | "$DATAGEN_BIN" $@ 4 | -------------------------------------------------------------------------------- /tests/cli-tests/bin/die: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | println "${*}" 1>&2 4 | exit 1 5 | -------------------------------------------------------------------------------- /tests/cli-tests/bin/println: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | printf '%b\n' "${*}" 3 | -------------------------------------------------------------------------------- /tests/cli-tests/bin/unzstd: -------------------------------------------------------------------------------- 1 | zstd -------------------------------------------------------------------------------- /tests/cli-tests/bin/zstd: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | zstdname=$(basename $0) 4 | 5 | if [ -z "$EXEC_PREFIX" ]; then 6 | "$ZSTD_SYMLINK_DIR/$zstdname" $@ 7 | else 8 | $EXEC_PREFIX "$ZSTD_SYMLINK_DIR/$zstdname" $@ 9 | fi 10 | -------------------------------------------------------------------------------- /tests/cli-tests/bin/zstdcat: -------------------------------------------------------------------------------- 1 | zstd -------------------------------------------------------------------------------- /tests/cli-tests/bin/zstdgrep: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | "$ZSTDGREP_BIN" $@ 3 | -------------------------------------------------------------------------------- /tests/cli-tests/bin/zstdless: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | "$ZSTDLESS_BIN" $@ 3 | -------------------------------------------------------------------------------- /tests/cli-tests/cltools/setup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | echo "1234" > file 6 | zstd file 7 | -------------------------------------------------------------------------------- /tests/cli-tests/cltools/zstdgrep.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | println "+ good path" 6 | zstdgrep "1234" file file.zst 7 | println "+ bad path" 8 | zstdgrep "1234" bad.zst 9 | -------------------------------------------------------------------------------- /tests/cli-tests/cltools/zstdgrep.sh.exit: -------------------------------------------------------------------------------- 1 | 1 -------------------------------------------------------------------------------- /tests/cli-tests/cltools/zstdgrep.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | zstd: can't stat bad.zst : No such file or directory -- ignored 2 | -------------------------------------------------------------------------------- /tests/cli-tests/cltools/zstdgrep.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | + good path 2 | file:1234 3 | file.zst:1234 4 | + bad path 5 | -------------------------------------------------------------------------------- /tests/cli-tests/cltools/zstdless.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | println "+ good path" 6 | zstdless file.zst 7 | println "+ pass parameters" 8 | zstdless -N file.zst # This parameter does not produce line #s when piped, but still serves to test that the flag went to less and not zstd 9 | println "+ bad path" 10 | zstdless bad.zst >&2 11 | -------------------------------------------------------------------------------- /tests/cli-tests/cltools/zstdless.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | zstd: can't stat bad.zst : No such file or directory -- ignored 2 | bad.zst: No such file or directory 3 | -------------------------------------------------------------------------------- /tests/cli-tests/cltools/zstdless.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | + good path 2 | 1234 3 | + pass parameters 4 | 1234 5 | + bad path 6 | -------------------------------------------------------------------------------- /tests/cli-tests/common/format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$COMMON/platform.sh" 4 | 5 | zstd_supports_format() 6 | { 7 | zstd -h | grep > $INTOVOID -- "--format=$1" 8 | } 9 | 10 | format_extension() 11 | { 12 | if [ "$1" = "zstd" ]; then 13 | printf "zst" 14 | elif [ "$1" = "gzip" ]; then 15 | printf "gz" 16 | else 17 | printf "$1" 18 | fi 19 | } 20 | -------------------------------------------------------------------------------- /tests/cli-tests/common/mtime.sh: -------------------------------------------------------------------------------- 1 | . "$COMMON/platform.sh" 2 | 3 | MTIME="stat -c %Y" 4 | case "$UNAME" in 5 | Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;; 6 | esac 7 | 8 | assertSameMTime() { 9 | MT1=$($MTIME "$1") 10 | MT2=$($MTIME "$2") 11 | echo MTIME $MT1 $MT2 12 | [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)" 13 | } 14 | -------------------------------------------------------------------------------- /tests/cli-tests/common/permissions.sh: -------------------------------------------------------------------------------- 1 | . "$COMMON/platform.sh" 2 | 3 | GET_PERMS="stat -c %a" 4 | case "$UNAME" in 5 | Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;; 6 | esac 7 | 8 | assertFilePermissions() { 9 | STAT1=$($GET_PERMS "$1") 10 | STAT2=$2 11 | [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)" 12 | } 13 | 14 | assertSamePermissions() { 15 | STAT1=$($GET_PERMS "$1") 16 | STAT2=$($GET_PERMS "$2") 17 | [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)" 18 | } 19 | -------------------------------------------------------------------------------- /tests/cli-tests/common/platform.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | UNAME=$(uname) 4 | 5 | isWindows=false 6 | INTOVOID="/dev/null" 7 | case "$UNAME" in 8 | GNU) DEVDEVICE="/dev/random" ;; 9 | *) DEVDEVICE="/dev/zero" ;; 10 | esac 11 | case "$OS" in 12 | Windows*) 13 | isWindows=true 14 | INTOVOID="NUL" 15 | DEVDEVICE="NUL" 16 | ;; 17 | esac 18 | 19 | case "$UNAME" in 20 | Darwin) MD5SUM="md5 -r" ;; 21 | NetBSD) MD5SUM="md5 -n" ;; 22 | OpenBSD) MD5SUM="md5" ;; 23 | *) MD5SUM="md5sum" ;; 24 | esac 25 | 26 | md5hash() { 27 | $MD5SUM | dd bs=1 count=32 status=none 28 | echo 29 | } 30 | 31 | DIFF="diff" 32 | case "$UNAME" in 33 | SunOS) DIFF="gdiff" ;; 34 | esac 35 | 36 | if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled' 37 | then 38 | hasMT="" 39 | else 40 | hasMT="true" 41 | fi 42 | 43 | if zstd -vv --version | grep -q 'non-deterministic'; then 44 | NON_DETERMINISTIC="true" 45 | else 46 | NON_DETERMINISTIC="" 47 | fi 48 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/adapt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test --adapt 6 | zstd -f file --adapt -c | zstd -t 7 | 8 | datagen -g100M > file100M 9 | 10 | # Pick parameters to force fast adaptation, even on slow systems 11 | zstd --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression" 12 | 13 | # Adaption still happens with --no-progress 14 | zstd --no-progress --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression" 15 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/basic.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Uncomment the set -v line for debugging 6 | # set -v 7 | 8 | # Test compression flags and check that they work 9 | zstd file ; zstd -t file.zst 10 | zstd -f file ; zstd -t file.zst 11 | zstd -f -z file ; zstd -t file.zst 12 | zstd -f -k file ; zstd -t file.zst 13 | zstd -f -C file ; zstd -t file.zst 14 | zstd -f --check file ; zstd -t file.zst 15 | zstd -f --no-check file ; zstd -t file.zst 16 | zstd -f -- file ; zstd -t file.zst 17 | 18 | # Test output file compression 19 | zstd -o file-out.zst ; zstd -t file-out.zst 20 | zstd -fo file-out.zst; zstd -t file-out.zst 21 | 22 | # Test compression to stdout 23 | zstd -c file | zstd -t 24 | zstd --stdout file | zstd -t 25 | println bob | zstd | zstd -t 26 | 27 | # Test keeping input file when compressing to stdout in gzip mode 28 | if $(command -v $ZSTD_SYMLINK_DIR/gzip); then 29 | $ZSTD_SYMLINK_DIR/gzip -c file | zstd -t ; test -f file 30 | $ZSTD_SYMLINK_DIR/gzip --stdout file | zstd -t ; test -f file 31 | fi 32 | 33 | # Test --rm 34 | cp file file-rm 35 | zstd --rm file-rm; zstd -t file-rm.zst 36 | test ! -f file-rm 37 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/compress-literals.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test --[no-]compress-literals 6 | zstd file --no-compress-literals -1 -c | zstd -t 7 | zstd file --no-compress-literals -19 -c | zstd -t 8 | zstd file --no-compress-literals --fast=1 -c | zstd -t 9 | zstd file --compress-literals -1 -c | zstd -t 10 | zstd file --compress-literals --fast=1 -c | zstd -t 11 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$COMMON/format.sh" 4 | 5 | set -e 6 | 7 | # Test --format 8 | zstd --format=zstd file -f 9 | zstd -t file.zst 10 | for format in "gzip" "lz4" "xz" "lzma"; do 11 | if zstd_supports_format $format; then 12 | zstd --format=$format file 13 | zstd -t file.$(format_extension $format) 14 | zstd -c --format=$format file | zstd -t --format=$format 15 | fi 16 | done 17 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/golden.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-compression/" 6 | cp -r "$GOLDEN_DIR" golden/ 7 | 8 | zstd -rf golden/ --output-dir-mirror golden-compressed/ 9 | zstd -r -t golden-compressed/ 10 | 11 | zstd --target-compressed-block-size=1024 -rf golden/ --output-dir-mirror golden-compressed/ 12 | zstd -r -t golden-compressed/ 13 | 14 | # PR #3517 block splitter corruption test 15 | zstd -rf -19 --zstd=mml=7 golden/ --output-dir-mirror golden-compressed/ 16 | zstd -r -t golden-compressed/ -------------------------------------------------------------------------------- /tests/cli-tests/compression/gzip-compat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Uncomment the set -v line for debugging 6 | # set -v 7 | 8 | # Test gzip specific compression option 9 | if $(command -v $ZSTD_SYMLINK_DIR/gzip); then 10 | $ZSTD_SYMLINK_DIR/gzip --fast file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz 11 | $ZSTD_SYMLINK_DIR/gzip --best file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz 12 | 13 | # Test -n / --no-name: do not embed original filename in archive 14 | $ZSTD_SYMLINK_DIR/gzip -n file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz 15 | $ZSTD_SYMLINK_DIR/gzip --no-name file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz 16 | $ZSTD_SYMLINK_DIR/gzip -c --no-name file | grep -qv file 17 | fi 18 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/long-distance-matcher.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test --long 6 | zstd -f file --long ; zstd -t file.zst 7 | zstd -f file --long=20; zstd -t file.zst 8 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/multi-threaded.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test multi-threaded flags 6 | zstd --single-thread file -f -q ; zstd -t file.zst 7 | zstd -T2 -f file -q ; zstd -t file.zst 8 | zstd --rsyncable -f file -q ; zstd -t file.zst 9 | zstd -T0 -f file -q ; zstd -t file.zst 10 | zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst 11 | zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst 12 | zstd -T0 --jobsize=1M -f file -q ; zstd -t file.zst 13 | 14 | # multi-thread decompression warning test 15 | zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3 16 | zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -d file.zst -o file4 17 | # setting multi-thread via environment variable does not trigger decompression warning 18 | zstd -T0 -f file -q ; zstd -t file.zst; ZSTD_NBTHREADS=0 zstd -df file.zst -o file3 19 | zstd -T0 -f file -q ; zstd -t file.zst; ZSTD_NBTHREADS=2 zstd -df file.zst -o file4 20 | # setting nbThreads==1 does not trigger decompression warning 21 | zstd -T0 -f file -q ; zstd -t file.zst; zstd -T1 -df file.zst -o file3 22 | zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -T1 -df file.zst -o file4 23 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/multi-threaded.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | file.zst : 65537 bytes 2 | file.zst : 65537 bytes 3 | file.zst : 65537 bytes 4 | file.zst : 65537 bytes 5 | file.zst : 65537 bytes 6 | file.zst : 65537 bytes 7 | file.zst : 65537 bytes 8 | file.zst : 65537 bytes 9 | Warning : decompression does not support multi-threading 10 | file.zst : 65537 bytes 11 | file.zst : 65537 bytes 12 | Warning : decompression does not support multi-threading 13 | file.zst : 65537 bytes 14 | file.zst : 65537 bytes 15 | file.zst : 65537 bytes 16 | file.zst : 65537 bytes 17 | file.zst : 65537 bytes 18 | file.zst : 65537 bytes 19 | file.zst : 65537 bytes 20 | file.zst : 65537 bytes 21 | file.zst : 65537 bytes 22 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/multiple-files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # setup 5 | echo "file1" > file1 6 | echo "file2" > file2 7 | 8 | echo "Test zstd ./file1 - file2" 9 | rm -f ./file*.zst 10 | echo "stdin" | zstd ./file1 - ./file2 | zstd -d 11 | cat file1.zst | zstd -d 12 | cat file2.zst | zstd -d 13 | 14 | echo "Test zstd -d ./file1.zst - file2.zst" 15 | rm ./file1 ./file2 16 | echo "stdin" | zstd - | zstd -d ./file1.zst - file2.zst 17 | cat file1 18 | cat file2 19 | 20 | echo "zstd -d ./file1.zst - file2.zst -c" 21 | echo "stdin" | zstd | zstd -d ./file1.zst - file2.zst -c 22 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/multiple-files.sh.stdout.exact: -------------------------------------------------------------------------------- 1 | Test zstd ./file1 - file2 2 | stdin 3 | file1 4 | file2 5 | Test zstd -d ./file1.zst - file2.zst 6 | stdin 7 | file1 8 | file2 9 | zstd -d ./file1.zst - file2.zst -c 10 | file1 11 | stdin 12 | file2 13 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/row-match-finder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test --[no-]row-match-finder 6 | zstd file -7f --row-match-finder 7 | zstd file -7f --no-row-match-finder 8 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/setup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen > file 6 | datagen > file0 7 | datagen > file1 8 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/stream-size.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test stream size & hint 6 | datagen -g7654 | zstd --stream-size=7654 | zstd -t 7 | datagen -g7654 | zstd --size-hint=7000 | zstd -t 8 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/verbose-wlog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | . "$COMMON/platform.sh" 6 | 7 | zstd < file -vv -19 -o file.19.zst 8 | zstd -vv -l file.19.zst 9 | 10 | zstd < file -vv -19 --long -o file.19.long.zst 11 | zstd -vv -l file.19.long.zst 12 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/verbose-wlog.sh.stderr.glob: -------------------------------------------------------------------------------- 1 | ... 2 | *wlog=23* 3 | ... 4 | *wlog=27* 5 | ... 6 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/verbose-wlog.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | ... 2 | *Window Size: 8388608 B* 3 | ... 4 | *Window Size: 134217728 B* 5 | ... 6 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/window-resize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | datagen -g1G > file 3 | zstd --long=30 -1 --single-thread --no-content-size -f file 4 | zstd -l -v file.zst 5 | 6 | # We want to ignore stderr (its outputting "*** zstd command line interface 7 | # 64-bits v1.5.3, by Yann Collet ***") 8 | 9 | rm file file.zst 10 | -------------------------------------------------------------------------------- /tests/cli-tests/compression/window-resize.sh.stderr.ignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/cli-tests/compression/window-resize.sh.stderr.ignore -------------------------------------------------------------------------------- /tests/cli-tests/compression/window-resize.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | ... 2 | Window Size: 1.000 GiB (1073741824 B) 3 | ... 4 | -------------------------------------------------------------------------------- /tests/cli-tests/decompression/detectErrors.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression-errors/" 6 | 7 | for file in "$GOLDEN_DIR"/*; do 8 | zstd -t $file && die "should have detected an error" 9 | done 10 | exit 0 11 | 12 | -------------------------------------------------------------------------------- /tests/cli-tests/decompression/golden.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression/" 6 | 7 | zstd -r -t "$GOLDEN_DIR" 8 | -------------------------------------------------------------------------------- /tests/cli-tests/decompression/pass-through.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | . "$COMMON/platform.sh" 6 | 7 | echo "" > 1 8 | echo "2" > 2 9 | echo "23" > 3 10 | echo "234" > 4 11 | echo "some data" > file 12 | 13 | println "+ passthrough enabled" 14 | 15 | zstd file 16 | 17 | # Test short files 18 | zstd -dc --pass-through 1 2 3 4 19 | 20 | # Test *cat symlinks 21 | zstdcat file 22 | "$ZSTD_SYMLINK_DIR/zcat" file 23 | "$ZSTD_SYMLINK_DIR/gzcat" file 24 | 25 | # Test multiple files with mix of compressed & not 26 | zstdcat file file.zst 27 | zstdcat file.zst file 28 | 29 | # Test --pass-through 30 | zstd -dc --pass-through file 31 | zstd -d --pass-through file -o pass-through-file 32 | 33 | # Test legacy implicit passthrough with -fc 34 | zstd -dcf file 35 | zstd -dcf file file.zst 36 | zstd -df < file 37 | zstd -dcf < file file.zst - 38 | zstd -dcf < file.zst file - 39 | 40 | $DIFF file pass-through-file 41 | 42 | println "+ passthrough disabled" 43 | 44 | # Test *cat 45 | zstdcat --no-pass-through file && die "should fail" 46 | "$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail" 47 | "$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail" 48 | # Test zstd without implicit passthrough 49 | zstd -d file -o no-pass-through-file && die "should fail" 50 | zstd -d < file && die "should fail" 51 | 52 | # Test legacy implicit passthrough with -fc 53 | zstd --no-pass-through -dcf file && die "should fail" 54 | zstd --no-pass-through -dcf file file.zst && die "should fail" 55 | zstd --no-pass-through -df < file && die "should fail" 56 | zstd --no-pass-through -dcf < file file.zst - && die "should fail" 57 | zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||: 58 | -------------------------------------------------------------------------------- /tests/cli-tests/decompression/pass-through.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | file :230.00% ( 10 B => 23 B, file.zst) 2 | zstd: file: unsupported format 3 | zstd: file: unsupported format 4 | zstd: file: unsupported format 5 | zstd: file: unsupported format 6 | zstd: /*stdin*\: unsupported format 7 | zstd: file: unsupported format 8 | zstd: file: unsupported format 9 | zstd: /*stdin*\: unsupported format 10 | zstd: /*stdin*\: unsupported format 11 | zstd: file: unsupported format 12 | -------------------------------------------------------------------------------- /tests/cli-tests/decompression/pass-through.sh.stdout.exact: -------------------------------------------------------------------------------- 1 | + passthrough enabled 2 | 3 | 2 4 | 23 5 | 234 6 | some data 7 | some data 8 | some data 9 | some data 10 | some data 11 | some data 12 | some data 13 | some data 14 | some data 15 | some data 16 | some data 17 | some data 18 | some data 19 | some data 20 | some data 21 | some data 22 | + passthrough disabled 23 | some data 24 | some data 25 | some data 26 | -------------------------------------------------------------------------------- /tests/cli-tests/determinism/basic.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$COMMON/platform.sh" 4 | 5 | set -e 6 | 7 | # To update checksums on version change run this from the tests/ directory 8 | # make update-cli-tests 9 | 10 | if [ -n "$NON_DETERMINISTIC" ]; then 11 | # Skip tests if we have a non-deterministic build 12 | cat "$CLI_TESTS/determinism/basic.sh.stdout.exact" 13 | exit 0 14 | fi 15 | 16 | for level in $(seq 1 19); do 17 | for file in $(ls files/); do 18 | file="files/$file" 19 | echo "level $level, file $file" 20 | zstd --single-thread -q -$level $file -c | md5hash 21 | done 22 | done 23 | 24 | for file in $(ls files/); do 25 | file="files/$file" 26 | echo "level 1, long=18, file $file" 27 | zstd --long=18 --single-thread -q -1 $file -c | md5hash 28 | echo "level 19, long=18, file $file" 29 | zstd --long=18 --single-thread -q -19 $file -c | md5hash 30 | done 31 | 32 | for file in $(ls files/); do 33 | file="files/$file" 34 | echo "level -1, file $file" 35 | zstd -q --single-thread --fast=1 $file -c | md5hash 36 | done 37 | -------------------------------------------------------------------------------- /tests/cli-tests/determinism/basic.sh.stderr.exact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/cli-tests/determinism/basic.sh.stderr.exact -------------------------------------------------------------------------------- /tests/cli-tests/determinism/multithread.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$COMMON/platform.sh" 4 | 5 | set -e 6 | 7 | # To update checksums on version change run this from the tests/ directory 8 | # make update-cli-tests 9 | 10 | if [ -n "$NON_DETERMINISTIC" ] || [ -z "$hasMT" ]; then 11 | # Skip tests if we have a non-deterministic build 12 | cat "$CLI_TESTS/determinism/multithread.sh.stdout.exact" 13 | exit 0 14 | fi 15 | 16 | for level in 1 3 7 19; do 17 | for file in $(ls files/); do 18 | file="files/$file" 19 | echo "level $level, file $file" 20 | zstd -T2 -q -$level $file -c | md5hash 21 | done 22 | done 23 | 24 | for file in $(ls files/); do 25 | file="files/$file" 26 | echo "level 1, long=18, file $file" 27 | zstd --long=18 -T2 -q -1 $file -c | md5hash 28 | echo "level 19, long=18, file $file" 29 | zstd --long=18 -T2 -q -19 $file -c | md5hash 30 | done 31 | 32 | for file in $(ls files/); do 33 | file="files/$file" 34 | echo "Vary number of threads on $file" 35 | zstd -qf -1 $file -o $file.zst.good 36 | 37 | zstd -qf -T1 -1 $file 38 | $DIFF $file.zst $file.zst.good 39 | 40 | zstd -qf -T2 -1 $file 41 | $DIFF $file.zst $file.zst.good 42 | 43 | zstd -qf -T4 -1 $file 44 | $DIFF $file.zst $file.zst.good 45 | done 46 | -------------------------------------------------------------------------------- /tests/cli-tests/determinism/multithread.sh.stderr.exact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/cli-tests/determinism/multithread.sh.stderr.exact -------------------------------------------------------------------------------- /tests/cli-tests/determinism/reuse.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$COMMON/platform.sh" 4 | 5 | set -e 6 | 7 | # To update checksums on version change run this from the tests/ directory 8 | # make update-cli-tests 9 | 10 | if [ -n "$NON_DETERMINISTIC" ]; then 11 | # Skip tests if we have a non-deterministic build 12 | cat "$CLI_TESTS/determinism/reuse.sh.stdout.exact" 13 | exit 0 14 | fi 15 | 16 | datagen -g0 > file0 17 | datagen -g1 > file1 18 | datagen -g1000 > file1000 19 | datagen -g100000 > file100000 20 | 21 | validate() { 22 | $DIFF file0.zst file0.zst.good 23 | $DIFF file1.zst file1.zst.good 24 | $DIFF file1000.zst file1000.zst.good 25 | $DIFF file100000.zst file100000.zst.good 26 | } 27 | 28 | # Check that context reuse doesn't impact determinism 29 | for level in $(seq 1 19); do 30 | echo $level 31 | zstd -qf --single-thread -$level file0 -o file0.zst.good 32 | zstd -qf --single-thread -$level file1 -o file1.zst.good 33 | zstd -qf --single-thread -$level file1000 -o file1000.zst.good 34 | zstd -qf --single-thread -$level file100000 -o file100000.zst.good 35 | 36 | zstd -qf --single-thread -$level file0 file1 file1000 file100000 37 | validate 38 | zstd -qf --single-thread -$level file1 file0 file1000 file100000 39 | validate 40 | zstd -qf --single-thread -$level file1000 file1 file0 file100000 41 | validate 42 | zstd -qf --single-thread -$level file100000 file1000 file1 file0 43 | validate 44 | done 45 | -------------------------------------------------------------------------------- /tests/cli-tests/determinism/reuse.sh.stderr.exact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/cli-tests/determinism/reuse.sh.stderr.exact -------------------------------------------------------------------------------- /tests/cli-tests/determinism/reuse.sh.stdout.exact: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | 3 4 | 4 5 | 5 6 | 6 7 | 7 8 | 8 9 | 9 10 | 10 11 | 11 12 | 12 13 | 13 14 | 14 15 | 15 16 | 16 17 | 17 18 | 18 19 | 19 20 | -------------------------------------------------------------------------------- /tests/cli-tests/determinism/setup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cp -r ../files . 6 | -------------------------------------------------------------------------------- /tests/cli-tests/determinism/setup_once: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | . "$COMMON/platform.sh" 6 | 7 | 8 | mkdir files/ 9 | 10 | datagen -g0 > files/g0 11 | datagen -g1 > files/g1 12 | datagen -g10 > files/g10 13 | datagen -g100 > files/g100 14 | datagen -g1000 > files/g1000 15 | datagen -g10000 > files/g10000 16 | datagen -g20000 > files/g20000 17 | datagen -g30000 > files/g30000 18 | datagen -g50000 > files/g50000 19 | datagen -g100000 > files/g100000 20 | datagen -g200000 > files/g200000 21 | datagen -g500000 > files/g500000 22 | datagen -g1000000 > files/g1000000 23 | 24 | datagen -g10000 -P0 > files/g10000-P0 25 | datagen -g10000 -P10 > files/g10000-P10 26 | datagen -g10000 -P25 > files/g10000-P25 27 | datagen -g10000 -P50 > files/g10000-P50 28 | datagen -g10000 -P75 > files/g10000-P75 29 | datagen -g10000 -P90 > files/g10000-P90 30 | datagen -g10000 -P100 > files/g10000-P100 31 | -------------------------------------------------------------------------------- /tests/cli-tests/dict-builder/empty-input.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | for i in $(seq 50); do 4 | datagen -s$i > file$i 5 | done 6 | touch empty 7 | 8 | set -v 9 | zstd -q --train empty file* 10 | -------------------------------------------------------------------------------- /tests/cli-tests/dict-builder/empty-input.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | zstd -q --train empty file* 2 | -------------------------------------------------------------------------------- /tests/cli-tests/dict-builder/no-inputs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -v 3 | zstd --train 4 | -------------------------------------------------------------------------------- /tests/cli-tests/dict-builder/no-inputs.sh.exit: -------------------------------------------------------------------------------- 1 | 14 2 | -------------------------------------------------------------------------------- /tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | zstd --train 2 | ! Warning : nb of samples too low for proper processing ! 3 | ! Please provide _one file per sample_. 4 | ! Alternatively, split file(s) into fixed-size samples, with --split=# 5 | Error 14 : nb of samples too low 6 | -------------------------------------------------------------------------------- /tests/cli-tests/dictionaries/dictionary-mismatch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$COMMON/platform.sh" 4 | 5 | set -e 6 | 7 | if [ false ]; then 8 | for seed in $(seq 100); do 9 | datagen -g1000 -s$seed > file$seed 10 | done 11 | 12 | zstd --train -r . -o dict0 -qq 13 | 14 | for seed in $(seq 101 200); do 15 | datagen -g1000 -s$seed > file$seed 16 | done 17 | 18 | zstd --train -r . -o dict1 -qq 19 | 20 | [ "$($MD5SUM < dict0)" != "$($MD5SUM < dict1)" ] || die "dictionaries must not match" 21 | 22 | datagen -g1000 -s0 > file0 23 | fi 24 | 25 | set -v 26 | zstd files/0 -D dicts/0 -q 27 | zstd -t files/0.zst -D dicts/0 28 | zstd -t files/0.zst -D dicts/1 && die "Must fail" ||: 29 | zstd -t files/0.zst && die "Must fail" ||: 30 | -------------------------------------------------------------------------------- /tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | zstd files/0 -D dicts/0 -q 2 | zstd -t files/0.zst -D dicts/0 3 | files/0.zst : 1000 bytes 4 | zstd -t files/0.zst -D dicts/1 && die "Must fail" ||: 5 | files/0.zst : Decoding error (36) : Dictionary mismatch 6 | zstd -t files/0.zst && die "Must fail" ||: 7 | files/0.zst : Decoding error (36) : Dictionary mismatch 8 | -------------------------------------------------------------------------------- /tests/cli-tests/dictionaries/golden.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | GOLDEN_COMP_DIR="$ZSTD_REPO_DIR/tests/golden-compression/" 6 | GOLDEN_DICT_DIR="$ZSTD_REPO_DIR/tests/golden-dictionaries/" 7 | 8 | zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" "$GOLDEN_COMP_DIR/http" -o http.zst 9 | zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" -t http.zst 10 | -------------------------------------------------------------------------------- /tests/cli-tests/dictionaries/setup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cp -r ../files . 6 | cp -r ../dicts . 7 | -------------------------------------------------------------------------------- /tests/cli-tests/dictionaries/setup_once: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | . "$COMMON/platform.sh" 6 | 7 | 8 | mkdir files/ dicts/ 9 | 10 | for seed in $(seq 50); do 11 | datagen -g1000 -s$seed > files/$seed 12 | done 13 | 14 | zstd --train -r files -o dicts/0 -qq 15 | 16 | for seed in $(seq 51 100); do 17 | datagen -g1000 -s$seed > files/$seed 18 | done 19 | 20 | zstd --train -r files -o dicts/1 -qq 21 | 22 | cmp dicts/0 dicts/1 && die "dictionaries must not match!" 23 | 24 | datagen -g1000 > files/0 25 | -------------------------------------------------------------------------------- /tests/cli-tests/file-handling/directory-mirror.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # setup 5 | mkdir -p src/.hidden src/dir 6 | mkdir mid dst 7 | 8 | echo "file1" > src/file1 9 | echo "file2" > src/.file2 10 | echo "file3" > src/.hidden/.file3 11 | echo "file4" > src/dir/.file4 12 | 13 | # relative paths 14 | zstd -q -r --output-dir-mirror mid/ src/ 15 | zstd -q -d -r --output-dir-mirror dst/ mid/src/ 16 | 17 | diff --brief --recursive --new-file src/ dst/mid/src/ 18 | 19 | # reset 20 | rm -rf mid dst 21 | mkdir mid dst 22 | 23 | # from inside the directory 24 | (cd src; zstd -q -r --output-dir-mirror ../mid/ ./) 25 | (cd mid; zstd -q -d -r --output-dir-mirror ../dst/ ./) 26 | 27 | diff --brief --recursive --new-file src/ dst/ 28 | 29 | # reset 30 | rm -rf mid dst 31 | mkdir mid dst 32 | 33 | # absolute paths 34 | export BASE_PATH="$(pwd)" 35 | 36 | zstd -q -r --output-dir-mirror mid/ "${BASE_PATH}/src/" 37 | zstd -q -d -r --output-dir-mirror dst/ "${BASE_PATH}/mid/${BASE_PATH}/src/" 38 | 39 | diff --brief --recursive --new-file src/ "dst/${BASE_PATH}/mid/${BASE_PATH}/src/" 40 | 41 | # reset 42 | rm -rf mid dst 43 | mkdir mid dst 44 | 45 | # dots 46 | zstd -q -r --output-dir-mirror mid/ ./src/./ 47 | zstd -q -d -r --output-dir-mirror dst/ ./mid/./src/./ 48 | 49 | diff --brief --recursive --new-file src/ dst/mid/src/ 50 | -------------------------------------------------------------------------------- /tests/cli-tests/file-handling/directory-mirror.sh.stderr.exact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/cli-tests/file-handling/directory-mirror.sh.stderr.exact -------------------------------------------------------------------------------- /tests/cli-tests/file-handling/directory-mirror.sh.stdout.exact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/cli-tests/file-handling/directory-mirror.sh.stdout.exact -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # motivated by issue #3523 4 | 5 | datagen > file 6 | mkdir out 7 | chmod 000 out 8 | 9 | zstd file -q --trace-file-stat -o out/file.zst 10 | zstd -tq out/file.zst 11 | 12 | chmod 777 out 13 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isLink(file) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(2) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_getFileSize(file) 6 | Trace:FileStat: > UTIL_stat(-1, file) 7 | Trace:FileStat: < 1 8 | Trace:FileStat: < 65537 9 | Trace:FileStat: > UTIL_stat(-1, file) 10 | Trace:FileStat: < 1 11 | Trace:FileStat: > UTIL_isDirectoryStat() 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_stat(-1, file) 14 | Trace:FileStat: < 1 15 | Trace:FileStat: > UTIL_isSameFile(file, out/file.zst) 16 | Trace:FileStat: > UTIL_stat(-1, file) 17 | Trace:FileStat: < 1 18 | Trace:FileStat: > UTIL_stat(-1, out/file.zst) 19 | Trace:FileStat: < 0 20 | Trace:FileStat: < 0 21 | Trace:FileStat: > UTIL_isRegularFile(out/file.zst) 22 | Trace:FileStat: > UTIL_stat(-1, out/file.zst) 23 | Trace:FileStat: < 0 24 | Trace:FileStat: < 0 25 | Trace:FileStat: > UTIL_isFdRegularFile(-1) 26 | Trace:FileStat: < 0 27 | Trace:FileStat: > UTIL_isFileDescriptorPipe(out/file.zst) 28 | Trace:FileStat: < 0 29 | zstd: out/file.zst: Permission denied 30 | zstd: can't stat out/file.zst : Permission denied -- ignored 31 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/compress-file-to-file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen > file 6 | chmod 642 file 7 | 8 | zstd file -q --trace-file-stat -o file.zst 9 | zstd -tq file.zst 10 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.glob: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isLink(file) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(2) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_getFileSize(file) 6 | Trace:FileStat: > UTIL_stat(-1, file) 7 | Trace:FileStat: < 1 8 | Trace:FileStat: < 65537 9 | Trace:FileStat: > UTIL_stat(-1, file) 10 | Trace:FileStat: < 1 11 | Trace:FileStat: > UTIL_isDirectoryStat() 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_stat(-1, file) 14 | Trace:FileStat: < 1 15 | Trace:FileStat: > UTIL_isSameFile(file, file.zst) 16 | Trace:FileStat: > UTIL_stat(-1, file) 17 | Trace:FileStat: < 1 18 | Trace:FileStat: > UTIL_stat(-1, file.zst) 19 | Trace:FileStat: < 0 20 | Trace:FileStat: < 0 21 | Trace:FileStat: > UTIL_isRegularFile(file.zst) 22 | Trace:FileStat: > UTIL_stat(-1, file.zst) 23 | Trace:FileStat: < 0 24 | Trace:FileStat: < 0 25 | Trace:FileStat: > UTIL_isFdRegularFile(*) 26 | Trace:FileStat: > UTIL_stat(*, ) 27 | Trace:FileStat: < 1 28 | Trace:FileStat: < 1 29 | Trace:FileStat: > UTIL_isRegularFile(file.zst) 30 | Trace:FileStat: > UTIL_stat(-1, file.zst) 31 | Trace:FileStat: < 1 32 | Trace:FileStat: < 1 33 | Trace:FileStat: > UTIL_getFileSize(file) 34 | Trace:FileStat: > UTIL_stat(-1, file) 35 | Trace:FileStat: < 1 36 | Trace:FileStat: < 65537 37 | Trace:FileStat: > UTIL_setFileStat(4, file.zst) 38 | Trace:FileStat: > UTIL_stat(4, file.zst) 39 | Trace:FileStat: < 1 40 | Trace:FileStat: > UTIL_chmod(file.zst, 0642) 41 | Trace:FileStat: > fchmod 42 | Trace:FileStat: < 0 43 | Trace:FileStat: < 0 44 | Trace:FileStat: < 0 45 | Trace:FileStat: > UTIL_utime(file.zst) 46 | Trace:FileStat: < 0 47 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/compress-file-to-stdout.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen > file 6 | 7 | zstd file -cq --trace-file-stat > file.zst 8 | zstd -tq file.zst 9 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isLink(file) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(1) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isConsole(2) 6 | Trace:FileStat: < 0 7 | Trace:FileStat: > UTIL_getFileSize(file) 8 | Trace:FileStat: > UTIL_stat(-1, file) 9 | Trace:FileStat: < 1 10 | Trace:FileStat: < 65537 11 | Trace:FileStat: > UTIL_stat(-1, file) 12 | Trace:FileStat: < 1 13 | Trace:FileStat: > UTIL_isDirectoryStat() 14 | Trace:FileStat: < 0 15 | Trace:FileStat: > UTIL_stat(-1, file) 16 | Trace:FileStat: < 1 17 | Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) 18 | Trace:FileStat: > UTIL_stat(-1, /*stdout*\) 19 | Trace:FileStat: < 0 20 | Trace:FileStat: < 0 21 | Trace:FileStat: > UTIL_getFileSize(file) 22 | Trace:FileStat: > UTIL_stat(-1, file) 23 | Trace:FileStat: < 1 24 | Trace:FileStat: < 65537 25 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/compress-stdin-to-file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen > file 6 | 7 | zstd < file -q --trace-file-stat -o file.zst 8 | zstd -tq file.zst 9 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.glob: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isConsole(0) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(2) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_getFileSize(/*stdin*\) 6 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 7 | Trace:FileStat: < 0 8 | Trace:FileStat: < -1 9 | Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst) 10 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 11 | Trace:FileStat: < 0 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_isRegularFile(file.zst) 14 | Trace:FileStat: > UTIL_stat(-1, file.zst) 15 | Trace:FileStat: < 0 16 | Trace:FileStat: < 0 17 | Trace:FileStat: > UTIL_isFdRegularFile(*) 18 | Trace:FileStat: > UTIL_stat(*, ) 19 | Trace:FileStat: < 1 20 | Trace:FileStat: < 1 21 | Trace:FileStat: > UTIL_isRegularFile(file.zst) 22 | Trace:FileStat: > UTIL_stat(-1, file.zst) 23 | Trace:FileStat: < 1 24 | Trace:FileStat: < 1 25 | Trace:FileStat: > UTIL_getFileSize(/*stdin*\) 26 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 27 | Trace:FileStat: < 0 28 | Trace:FileStat: < -1 29 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/compress-stdin-to-stdout.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen > file 6 | 7 | zstd < file -cq --trace-file-stat > file.zst 8 | zstd -tq file.zst 9 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isConsole(0) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(1) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isConsole(2) 6 | Trace:FileStat: < 0 7 | Trace:FileStat: > UTIL_getFileSize(/*stdin*\) 8 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 9 | Trace:FileStat: < 0 10 | Trace:FileStat: < -1 11 | Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) 12 | Trace:FileStat: > UTIL_stat(-1, /*stdout*\) 13 | Trace:FileStat: < 0 14 | Trace:FileStat: < 0 15 | Trace:FileStat: > UTIL_getFileSize(/*stdin*\) 16 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 17 | Trace:FileStat: < 0 18 | Trace:FileStat: < -1 19 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/decompress-file-to-file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen | zstd -q > file.zst 6 | chmod 642 file.zst 7 | 8 | zstd -dq --trace-file-stat file.zst 9 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.glob: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isLink(file.zst) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(1) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isConsole(2) 6 | Trace:FileStat: < 0 7 | Trace:FileStat: > UTIL_isDirectory(file.zst) 8 | Trace:FileStat: > UTIL_stat(-1, file.zst) 9 | Trace:FileStat: < 1 10 | Trace:FileStat: > UTIL_isDirectoryStat() 11 | Trace:FileStat: < 0 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_stat(-1, file.zst) 14 | Trace:FileStat: < 1 15 | Trace:FileStat: > UTIL_isSameFile(file.zst, file) 16 | Trace:FileStat: > UTIL_stat(-1, file.zst) 17 | Trace:FileStat: < 1 18 | Trace:FileStat: > UTIL_stat(-1, file) 19 | Trace:FileStat: < 0 20 | Trace:FileStat: < 0 21 | Trace:FileStat: > UTIL_isRegularFile(file) 22 | Trace:FileStat: > UTIL_stat(-1, file) 23 | Trace:FileStat: < 0 24 | Trace:FileStat: < 0 25 | Trace:FileStat: > UTIL_isFdRegularFile(*) 26 | Trace:FileStat: > UTIL_stat(*, ) 27 | Trace:FileStat: < 1 28 | Trace:FileStat: < 1 29 | Trace:FileStat: > UTIL_isRegularFile(file) 30 | Trace:FileStat: > UTIL_stat(-1, file) 31 | Trace:FileStat: < 1 32 | Trace:FileStat: < 1 33 | Trace:FileStat: > UTIL_setFileStat(4, file) 34 | Trace:FileStat: > UTIL_stat(4, file) 35 | Trace:FileStat: < 1 36 | Trace:FileStat: > UTIL_chmod(file, 0642) 37 | Trace:FileStat: > fchmod 38 | Trace:FileStat: < 0 39 | Trace:FileStat: < 0 40 | Trace:FileStat: < 0 41 | Trace:FileStat: > UTIL_utime(file) 42 | Trace:FileStat: < 0 43 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/decompress-file-to-stdout.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen | zstd -q > file.zst 6 | 7 | zstd -dcq --trace-file-stat file.zst > file 8 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isLink(file.zst) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(1) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isConsole(2) 6 | Trace:FileStat: < 0 7 | Trace:FileStat: > UTIL_isDirectory(file.zst) 8 | Trace:FileStat: > UTIL_stat(-1, file.zst) 9 | Trace:FileStat: < 1 10 | Trace:FileStat: > UTIL_isDirectoryStat() 11 | Trace:FileStat: < 0 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_stat(-1, file.zst) 14 | Trace:FileStat: < 1 15 | Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) 16 | Trace:FileStat: > UTIL_stat(-1, /*stdout*\) 17 | Trace:FileStat: < 0 18 | Trace:FileStat: < 0 19 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/decompress-stdin-to-file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen | zstd -q > file.zst 6 | 7 | zstd -dcq --trace-file-stat < file.zst -o file 8 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.glob: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isConsole(0) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(2) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isDirectory(/*stdin*\) 6 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 7 | Trace:FileStat: < 0 8 | Trace:FileStat: < 0 9 | Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file) 10 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 11 | Trace:FileStat: < 0 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_isRegularFile(file) 14 | Trace:FileStat: > UTIL_stat(-1, file) 15 | Trace:FileStat: < 0 16 | Trace:FileStat: < 0 17 | Trace:FileStat: > UTIL_isFdRegularFile(*) 18 | Trace:FileStat: > UTIL_stat(*, ) 19 | Trace:FileStat: < 1 20 | Trace:FileStat: < 1 21 | Trace:FileStat: > UTIL_isRegularFile(file) 22 | Trace:FileStat: > UTIL_stat(-1, file) 23 | Trace:FileStat: < 1 24 | Trace:FileStat: < 1 25 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen | zstd -q > file.zst 6 | 7 | zstd -dcq --trace-file-stat < file.zst > file 8 | -------------------------------------------------------------------------------- /tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isConsole(0) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(1) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isConsole(2) 6 | Trace:FileStat: < 0 7 | Trace:FileStat: > UTIL_isDirectory(/*stdin*\) 8 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 9 | Trace:FileStat: < 0 10 | Trace:FileStat: < 0 11 | Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) 12 | Trace:FileStat: > UTIL_stat(-1, /*stdout*\) 13 | Trace:FileStat: < 0 14 | Trace:FileStat: < 0 15 | -------------------------------------------------------------------------------- /tests/cli-tests/progress/no-progress.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #!/bin/sh 4 | 5 | . "$COMMON/platform.sh" 6 | 7 | set -e 8 | 9 | echo hello > hello 10 | echo world > world 11 | 12 | zstd -q hello world 13 | 14 | println >&2 "Tests cases where progress information should not be printed" 15 | 16 | for args in \ 17 | "" \ 18 | "--fake-stderr-is-console -q" \ 19 | "--fake-stderr-is-console -qq --progress" \ 20 | "--no-progress --fake-stderr-is-console" \ 21 | "--no-progress --fake-stderr-is-console -v" 22 | do 23 | println >&2 "args = $args" 24 | println >&2 "compress file to file" 25 | zstd $args -f hello 26 | println >&2 "compress pipe to pipe" 27 | zstd $args < hello > $INTOVOID 28 | println >&2 "compress pipe to file" 29 | zstd $args < hello -fo hello.zst 30 | println >&2 "compress file to pipe" 31 | zstd $args hello -c > $INTOVOID 32 | println >&2 "compress 2 files" 33 | zstd $args -f hello world 34 | 35 | println >&2 "decompress file to file" 36 | zstd $args -d -f hello.zst 37 | println >&2 "decompress pipe to pipe" 38 | zstd $args -d < hello.zst > $INTOVOID 39 | println >&2 "decompress pipe to file" 40 | zstd $args -d < hello.zst -fo hello 41 | println >&2 "decompress file to pipe" 42 | zstd $args -d hello.zst -c > $INTOVOID 43 | println >&2 "decompress 2 files" 44 | zstd $args -d -f hello.zst world.zst 45 | println >&2 "" 46 | done 47 | -------------------------------------------------------------------------------- /tests/cli-tests/progress/progress.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$COMMON/platform.sh" 4 | 5 | set -e 6 | 7 | println >&2 "Tests cases where progress information should be printed" 8 | 9 | echo hello > hello 10 | echo world > world 11 | 12 | zstd -q hello world 13 | 14 | for args in \ 15 | "--progress" \ 16 | "--fake-stderr-is-console" \ 17 | "--progress --fake-stderr-is-console -q"; do 18 | println >&2 "args = $args" 19 | println >&2 "compress file to file" 20 | zstd $args -f hello 21 | println >&2 "compress pipe to pipe" 22 | zstd $args < hello > $INTOVOID 23 | println >&2 "compress pipe to file" 24 | zstd $args < hello -fo hello.zst 25 | println >&2 "compress file to pipe" 26 | zstd $args hello -c > $INTOVOID 27 | println >&2 "compress 2 files" 28 | zstd $args -f hello world 29 | 30 | println >&2 "decompress file to file" 31 | zstd $args -d -f hello.zst 32 | println >&2 "decompress pipe to pipe" 33 | zstd $args -d < hello.zst > $INTOVOID 34 | println >&2 "decompress pipe to file" 35 | zstd $args -d < hello.zst -fo hello 36 | println >&2 "decompress file to pipe" 37 | zstd $args -d hello.zst -c > $INTOVOID 38 | println >&2 "decompress 2 files" 39 | zstd $args -d -f hello.zst world.zst 40 | println >&2 "" 41 | done 42 | -------------------------------------------------------------------------------- /tests/cli-tests/zstd-symlinks/setup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | println "hello" > hello 5 | println "world" > world 6 | zstd hello world 7 | -------------------------------------------------------------------------------- /tests/cli-tests/zstd-symlinks/zstdcat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # Test zstdcat symlink in bin/ 5 | zstdcat hello.zst 6 | zstdcat hello.zst world 7 | zstdcat hello world.zst 8 | zstdcat hello.zst world.zst 9 | 10 | # Test local zstdcat symlink 11 | ln -s $(which zstd) ./zstdcat 12 | ./zstdcat hello.zst 13 | -------------------------------------------------------------------------------- /tests/cli-tests/zstd-symlinks/zstdcat.sh.stdout.exact: -------------------------------------------------------------------------------- 1 | hello 2 | hello 3 | world 4 | hello 5 | world 6 | hello 7 | world 8 | hello 9 | -------------------------------------------------------------------------------- /tests/dict-files/zero-weight-dict: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/dict-files/zero-weight-dict -------------------------------------------------------------------------------- /tests/external_matchfinder.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Yann Collet, Meta Platforms, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef EXTERNAL_MATCHFINDER 12 | #define EXTERNAL_MATCHFINDER 13 | 14 | #define ZSTD_STATIC_LINKING_ONLY 15 | #include "zstd.h" 16 | 17 | /* See external_matchfinder.c for details on each test case */ 18 | typedef enum { 19 | EMF_ZERO_SEQS = 0, 20 | EMF_ONE_BIG_SEQ = 1, 21 | EMF_LOTS_OF_SEQS = 2, 22 | EMF_BIG_ERROR = 3, 23 | EMF_SMALL_ERROR = 4, 24 | EMF_INVALID_OFFSET = 5, 25 | EMF_INVALID_MATCHLEN = 6, 26 | EMF_INVALID_LITLEN = 7, 27 | EMF_INVALID_LAST_LITS = 8 28 | } EMF_testCase; 29 | 30 | size_t zstreamSequenceProducer( 31 | void* sequenceProducerState, 32 | ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, 33 | const void* src, size_t srcSize, 34 | const void* dict, size_t dictSize, 35 | int compressionLevel, 36 | size_t windowSize 37 | ); 38 | 39 | #endif /* EXTERNAL_MATCHFINDER */ 40 | -------------------------------------------------------------------------------- /tests/fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | # test artefacts 2 | corpora 3 | block_decompress 4 | block_round_trip 5 | dictionary_decompress 6 | dictionary_loader 7 | dictionary_round_trip 8 | dictionary_stream_round_trip 9 | raw_dictionary_round_trip 10 | simple_compress 11 | simple_decompress 12 | simple_round_trip 13 | stream_decompress 14 | stream_round_trip 15 | zstd_frame_info 16 | decompress_dstSize_tooSmall 17 | fse_read_ncount 18 | sequence_compression_api 19 | seekable_roundtrip 20 | huf_decompress 21 | huf_round_trip 22 | fuzz-*.log 23 | rt_lib_* 24 | d_lib_* 25 | crash-* 26 | decompress_cross_format 27 | generate_sequences 28 | 29 | # misc 30 | trace 31 | tmp* 32 | -------------------------------------------------------------------------------- /tests/fuzz/fuzz_helpers.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #include "fuzz_helpers.h" 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | void* FUZZ_malloc(size_t size) 17 | { 18 | if (size > 0) { 19 | void* const mem = malloc(size); 20 | FUZZ_ASSERT(mem); 21 | return mem; 22 | } 23 | return NULL; 24 | } 25 | 26 | void* FUZZ_malloc_rand(size_t size, FUZZ_dataProducer_t *producer) 27 | { 28 | if (size > 0) { 29 | void* const mem = malloc(size); 30 | FUZZ_ASSERT(mem); 31 | return mem; 32 | } else { 33 | uintptr_t ptr = 0; 34 | /* Return junk pointer 50% of the time */ 35 | if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) 36 | ptr += FUZZ_dataProducer_int32Range(producer, -1000000, 1000000); 37 | return (void*)ptr; 38 | } 39 | } 40 | 41 | int FUZZ_memcmp(void const* lhs, void const* rhs, size_t size) 42 | { 43 | if (size == 0) { 44 | return 0; 45 | } 46 | return memcmp(lhs, rhs, size); 47 | } 48 | -------------------------------------------------------------------------------- /tests/fuzz/seq_prod_fuzz_example/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) Yann Collet, Meta Platforms, Inc. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under both the BSD-style license (found in the 5 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 6 | # in the COPYING file in the root directory of this source tree). 7 | # You may select, at your option, one of the above-listed licenses. 8 | 9 | CC = clang 10 | CFLAGS = -g -fno-omit-frame-pointer -fsanitize=undefined,address,fuzzer -I../ -I../../../lib/ 11 | 12 | .PHONY: default 13 | default: example_seq_prod.o 14 | 15 | example_seq_prod.o: example_seq_prod.c 16 | $(CC) -c $(CFLAGS) $^ -o $@ 17 | -------------------------------------------------------------------------------- /tests/fuzz/seq_prod_fuzz_example/README.md: -------------------------------------------------------------------------------- 1 | # Fuzzing a Custom Sequence Producer Plugin 2 | This directory contains example code for using a custom sequence producer in the zstd fuzzers. 3 | 4 | You can build and run the code in this directory using these commands: 5 | ``` 6 | $ make corpora 7 | $ make -C seq_prod_fuzz_example/ 8 | $ python3 ./fuzz.py build all --enable-fuzzer --enable-asan --enable-ubsan --cc clang --cxx clang++ --custom-seq-prod=seq_prod_fuzz_example/example_seq_prod.o 9 | $ python3 ./fuzz.py libfuzzer simple_round_trip 10 | ``` 11 | 12 | See `../fuzz_third_party_seq_prod.h` and `../README.md` for more information on zstd fuzzing. 13 | -------------------------------------------------------------------------------- /tests/fuzz/seq_prod_fuzz_example/example_seq_prod.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Yann Collet, Meta Platforms, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #include "fuzz_third_party_seq_prod.h" 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | _Thread_local size_t threadLocalState; 18 | 19 | size_t FUZZ_seqProdSetup(void) { 20 | threadLocalState = 0; 21 | return 0; 22 | } 23 | 24 | size_t FUZZ_seqProdTearDown(void) { 25 | return 0; 26 | } 27 | 28 | void* FUZZ_createSeqProdState(void) { 29 | return calloc(1, sizeof(size_t)); 30 | } 31 | 32 | size_t FUZZ_freeSeqProdState(void* state) { 33 | free(state); 34 | return 0; 35 | } 36 | 37 | size_t FUZZ_thirdPartySeqProd( 38 | void* sequenceProducerState, 39 | ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, 40 | const void* src, size_t srcSize, 41 | const void* dict, size_t dictSize, 42 | int compressionLevel, 43 | size_t windowSize 44 | ) { 45 | /* Try to catch unsafe use of the shared state */ 46 | size_t* const sharedStatePtr = (size_t*)sequenceProducerState; 47 | assert(*sharedStatePtr == threadLocalState); 48 | (*sharedStatePtr)++; threadLocalState++; 49 | 50 | /* Check that fallback is enabled when FUZZ_THIRD_PARTY_SEQ_PROD is defined */ 51 | return ZSTD_SEQUENCE_PRODUCER_ERROR; 52 | } 53 | -------------------------------------------------------------------------------- /tests/fuzz/zstd_frame_info.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /** 12 | * This fuzz target fuzzes all of the helper functions that consume compressed 13 | * input. 14 | */ 15 | 16 | #include 17 | #include 18 | #include 19 | #include "fuzz_helpers.h" 20 | #include "zstd_helpers.h" 21 | 22 | int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) 23 | { 24 | ZSTD_FrameHeader zfh; 25 | if (size == 0) { 26 | src = NULL; 27 | } 28 | /* You can fuzz any helper functions here that are fast, and take zstd 29 | * compressed data as input. E.g. don't expect the input to be a dictionary, 30 | * so don't fuzz ZSTD_getDictID_fromDict(). 31 | */ 32 | ZSTD_getFrameContentSize(src, size); 33 | ZSTD_getDecompressedSize(src, size); 34 | ZSTD_findFrameCompressedSize(src, size); 35 | ZSTD_getDictID_fromFrame(src, size); 36 | ZSTD_findDecompressedSize(src, size); 37 | ZSTD_decompressBound(src, size); 38 | ZSTD_frameHeaderSize(src, size); 39 | ZSTD_isFrame(src, size); 40 | ZSTD_getFrameHeader(&zfh, src, size); 41 | ZSTD_getFrameHeader_advanced(&zfh, src, size, ZSTD_f_zstd1); 42 | return 0; 43 | } 44 | -------------------------------------------------------------------------------- /tests/golden-compression/PR-3517-block-splitter-corruption-test: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/golden-compression/PR-3517-block-splitter-corruption-test -------------------------------------------------------------------------------- /tests/golden-compression/http: -------------------------------------------------------------------------------- 1 | ads60.vertamedia.comhttp://ads60.vertamedia.com/ops/43C53990160C658B/46991http://ads60.vertamedia.com/ve/43C53990160C658B/53http://ads60.vertamedia.com/ve/43C53990160C658B/54http://ads60.vertamedia.com/ve/43C53990160C658B/55http://ads60.vertamedia.com/ve/43C53990160C658B/56http://ads60.vertamedia.com/ve/43C53990160C658B/57http://ads60.vertamedia.com/ve/43C53990160C658B/66http://ads60.vertamedia.com/ve/43C53990160C658B/71 -------------------------------------------------------------------------------- /tests/golden-compression/huffman-compressed-larger: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/golden-compression/huffman-compressed-larger -------------------------------------------------------------------------------- /tests/golden-compression/large-literal-and-match-lengths: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/golden-compression/large-literal-and-match-lengths -------------------------------------------------------------------------------- /tests/golden-decompression-errors/.gitignore: -------------------------------------------------------------------------------- 1 | !*.zst 2 | -------------------------------------------------------------------------------- /tests/golden-decompression-errors/off0.bin.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/golden-decompression-errors/off0.bin.zst -------------------------------------------------------------------------------- /tests/golden-decompression-errors/truncated_huff_state.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/golden-decompression-errors/truncated_huff_state.zst -------------------------------------------------------------------------------- /tests/golden-decompression-errors/zeroSeq_extraneous.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/golden-decompression-errors/zeroSeq_extraneous.zst -------------------------------------------------------------------------------- /tests/golden-decompression/block-128k.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/golden-decompression/block-128k.zst -------------------------------------------------------------------------------- /tests/golden-decompression/empty-block.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/golden-decompression/empty-block.zst -------------------------------------------------------------------------------- /tests/golden-decompression/rle-first-block.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/golden-decompression/rle-first-block.zst -------------------------------------------------------------------------------- /tests/golden-decompression/zeroSeq_2B.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/golden-decompression/zeroSeq_2B.zst -------------------------------------------------------------------------------- /tests/golden-dictionaries/http-dict-missing-symbols: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/golden-dictionaries/http-dict-missing-symbols -------------------------------------------------------------------------------- /tests/gzip/gzip-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Test the obsolescent GZIP environment variable. 3 | 4 | # Copyright 2015-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | #echo PATH=$PATH 23 | #gzip --version 24 | 25 | echo a >exp || framework_failure_ 26 | gzip in || framework_failure_ 27 | 28 | fail=0 29 | GZIP=-qv gzip -d out 2>err || fail=1 30 | compare exp out || fail=1 31 | 32 | for badopt in -- -c --stdout -d --decompress -f --force -h --help -k --keep \ 33 | -l --list -L --license -r --recursive -Sxxx --suffix=xxx '--suffix xxx' \ 34 | -t --test -V --version 35 | do 36 | GZIP=$badopt gzip -d out 2>err && fail=1 37 | done 38 | 39 | for goodopt in -n --no-name -N --name -q --quiet -v --verbose \ 40 | -1 --fast -2 -3 -4 -5 -6 -7 -8 -9 --best 41 | do 42 | GZIP=$goodopt gzip -d out 2>err || fail=1 43 | compare exp out || fail=1 44 | done 45 | 46 | Exit $fail 47 | -------------------------------------------------------------------------------- /tests/gzip/helin-segv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Before gzip-1.4, gzip -d would segfault on some inputs. 3 | 4 | # Copyright (C) 2010-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | # This test case was provided by Aki Helin. 23 | printf '\037\235\220\0\0\0\304' > helin.gz || framework_failure_ 24 | printf '\0\0' > exp || framework_failure_ 25 | 26 | fail=0 27 | 28 | gzip -dc helin.gz > out || fail=1 29 | compare exp out || fail=1 30 | 31 | Exit $fail 32 | -------------------------------------------------------------------------------- /tests/gzip/hufts-segv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/facebook/zstd/448cd340879adc0ffe36ed1e26823ee2dcb3217b/tests/gzip/hufts-segv.gz -------------------------------------------------------------------------------- /tests/gzip/hufts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Exercise a bug whereby an invalid input could make gzip -d misbehave. 3 | 4 | # Copyright (C) 2009-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | printf '\n...: invalid compressed data--format violated\n' > exp \ 23 | || framework_failure_ 24 | 25 | fail=0 26 | gzip -dc "$abs_srcdir/hufts-segv.gz" > out 2> err 27 | test $? = 1 || fail=1 28 | 29 | compare /dev/null out || fail=1 30 | 31 | sed 's/.*hufts-segv.gz: /...: /' err > k; mv k err || fail=1 32 | compare exp err || fail=1 33 | 34 | Exit $fail 35 | -------------------------------------------------------------------------------- /tests/gzip/init.cfg: -------------------------------------------------------------------------------- 1 | # This file is sourced by init.sh, *before* its initialization. 2 | 3 | # This goes hand in hand with the "exec 9>&2;" in Makefile.am's 4 | # TESTS_ENVIRONMENT definition. 5 | stderr_fileno_=9 6 | -------------------------------------------------------------------------------- /tests/gzip/list.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Exercise the --list option. 3 | 4 | # Copyright 2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | echo zoology zucchini > in || framework_failure_ 23 | cp in orig || framework_failure_ 24 | 25 | gzip -l in && fail=1 26 | gzip -9 in || fail=1 27 | gzip -l in.gz >out1 || fail=1 28 | gzip -l in.gz | cat >out2 || fail=1 29 | compare out1 out2 || fail=1 30 | 31 | Exit $fail 32 | -------------------------------------------------------------------------------- /tests/gzip/memcpy-abuse.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Before gzip-1.4, this the use of memcpy in inflate_codes could 3 | # mistakenly operate on overlapping regions. Exercise that code. 4 | 5 | # Copyright (C) 2010-2016 Free Software Foundation, Inc. 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # limit so don't run it by default. 20 | 21 | . "${srcdir=.}/init.sh"; path_prepend_ . 22 | 23 | # The input must be larger than 32KiB and slightly 24 | # less uniform than e.g., all zeros. 25 | printf wxy%032767d 0 | tee in | gzip > in.gz || framework_failure_ 26 | 27 | fail=0 28 | 29 | # Before the fix, this would call memcpy with overlapping regions. 30 | gzip -dc in.gz > out || fail=1 31 | 32 | compare in out || fail=1 33 | 34 | Exit $fail 35 | -------------------------------------------------------------------------------- /tests/gzip/null-suffix-clobber.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Before gzip-1.5, gzip -d -S '' k.gz would delete F.gz and not create "F" 3 | 4 | # Copyright (C) 2010-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | printf anything | gzip > F.gz || framework_failure_ 23 | echo y > yes || framework_failure_ 24 | echo "gzip: invalid suffix ''" > expected-err || framework_failure_ 25 | 26 | fail=0 27 | 28 | gzip ---presume-input-tty -d -S '' F.gz < yes > out 2>err && fail=1 29 | 30 | compare /dev/null out || fail=1 31 | compare expected-err err || fail=1 32 | 33 | test -f F.gz || fail=1 34 | 35 | Exit $fail 36 | -------------------------------------------------------------------------------- /tests/gzip/stdin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Ensure that gzip interprets "-" as stdin. 3 | 4 | # Copyright (C) 2009-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | printf a | gzip > in || framework_failure_ 23 | printf aaa > exp || framework_failure_ 24 | 25 | fail=0 26 | gzip -dc in - in < in > out 2>err || fail=1 27 | 28 | compare exp out || fail=1 29 | compare /dev/null err || fail=1 30 | 31 | Exit $fail 32 | -------------------------------------------------------------------------------- /tests/gzip/trailing-nul.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # gzip accepts trailing NUL bytes; don't fail if there is exactly one. 3 | # Before gzip-1.4, this would fail. 4 | 5 | # Copyright (C) 2009-2016 Free Software Foundation, Inc. 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # limit so don't run it by default. 20 | 21 | . "${srcdir=.}/init.sh"; path_prepend_ . 22 | 23 | (echo 0 | gzip; printf '\0') > 0.gz || framework_failure_ 24 | (echo 00 | gzip; printf '\0\0') > 00.gz || framework_failure_ 25 | (echo 1 | gzip; printf '\1') > 1.gz || framework_failure_ 26 | 27 | fail=0 28 | 29 | for i in 0 00 1; do 30 | gzip -d $i.gz; ret=$? 31 | test $ret -eq $i || fail=1 32 | test $ret = 1 && continue 33 | echo $i > exp || fail=1 34 | compare exp $i || fail=1 35 | done 36 | 37 | Exit $fail 38 | -------------------------------------------------------------------------------- /tests/gzip/unpack-invalid.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # gzip should report invalid 'unpack' input when uncompressing. 3 | # With gzip-1.5, it would output invalid data instead. 4 | 5 | # Copyright (C) 2012-2016 Free Software Foundation, Inc. 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # limit so don't run it by default. 20 | 21 | . "${srcdir=.}/init.sh"; path_prepend_ . 22 | 23 | for input in \ 24 | '\037\036\000\000\037\213\010\000\000\000\000\000\002\003\036\000\000\000\002\003\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\037\000\302\240\037\213\010\000\000\000\000\000\002\003\355\301' \ 25 | '\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\076\366\017\370\036\016\030\000\000\000\000\000\000\000\000\000\034\010\105\140\104\025\020\047\000\000\037\036\016\030\000\000\000'; do 26 | 27 | printf "$input" >in || framework_failure_ 28 | 29 | if gzip -d out 2>err; then 30 | fail=1 31 | else 32 | fail=0 33 | fi 34 | done 35 | 36 | Exit $fail 37 | -------------------------------------------------------------------------------- /tests/gzip/z-suffix.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Check that -Sz works. 3 | 4 | # Copyright 2014-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | printf anything > F && cp F G || framework_failure_ 23 | gzip -Sz F || fail=1 24 | test ! -f F || fail=1 25 | test -f Fz || fail=1 26 | gzip -dSz F || fail=1 27 | test ! -f Fz || fail=1 28 | compare F G || fail\1 29 | 30 | Exit $fail 31 | -------------------------------------------------------------------------------- /tests/gzip/zdiff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Exercise zdiff with two compressed inputs. 3 | # Before gzip-1.4, this would fail. 4 | 5 | # Copyright (C) 2009-2016 Free Software Foundation, Inc. 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # limit so don't run it by default. 20 | 21 | . "${srcdir=.}/init.sh"; path_prepend_ . 22 | 23 | echo a > a || framework_failure_ 24 | echo b > b || framework_failure_ 25 | gzip a b || framework_failure_ 26 | 27 | cat < exp 28 | 1c1 29 | < a 30 | --- 31 | > b 32 | EOF 33 | 34 | fail=0 35 | zdiff a.gz b.gz > out 2>&1 36 | test $? = 1 || fail=1 37 | 38 | compare exp out || fail=1 39 | 40 | rm -f out 41 | # expect success, for equal files 42 | zdiff a.gz a.gz > out 2> err || fail=1 43 | # expect no output 44 | test -s out && fail=1 45 | # expect no stderr 46 | test -s err && fail=1 47 | 48 | Exit $fail 49 | -------------------------------------------------------------------------------- /tests/gzip/zgrep-context.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Ensure that zgrep -15 works. Before gzip-1.5, it would fail. 3 | 4 | # Copyright (C) 2012-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | # A limited replacement for seq: handle 1 or 2 args; increment must be 1 23 | seq() 24 | { 25 | case $# in 26 | 1) start=1 final=$1;; 27 | 2) start=$1 final=$2;; 28 | *) echo you lose 1>&2; exit 1;; 29 | esac 30 | awk 'BEGIN{for(i='$start';i<='$final';i++) print i}' < /dev/null 31 | } 32 | 33 | seq 40 > in || framework_failure_ 34 | gzip < in > in.gz || framework_failure_ 35 | seq 2 32 > exp || framework_failure_ 36 | 37 | : ${GREP=grep} 38 | $GREP -15 17 - < in > out && compare exp out || { 39 | echo >&2 "$0: $GREP does not support context options; skipping this test" 40 | exit 77 41 | } 42 | 43 | fail=0 44 | zgrep -15 17 - < in.gz > out || fail=1 45 | compare exp out || fail=1 46 | 47 | Exit $fail 48 | -------------------------------------------------------------------------------- /tests/gzip/zgrep-f.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Ensure that zgrep -f - works like grep -f - 3 | # Before gzip-1.4, it would fail. 4 | 5 | # Copyright (C) 2009-2016 Free Software Foundation, Inc. 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # limit so don't run it by default. 20 | 21 | . "${srcdir=.}/init.sh"; path_prepend_ . 22 | 23 | printf 'needle\nn2\n' > n || framework_failure_ 24 | cp n haystack || framework_failure_ 25 | gzip haystack || framework_failure_ 26 | 27 | fail=0 28 | zgrep -f - haystack.gz < n > out 2>&1 || fail=1 29 | 30 | compare out n || fail=1 31 | 32 | if ${BASH_VERSION+:} false; then 33 | set +o posix 34 | # This failed with gzip 1.6. 35 | cat n n >nn || framework_failure_ 36 | eval 'zgrep -h -f <(cat n) haystack.gz haystack.gz' >out || fail=1 37 | compare out nn || fail=1 38 | fi 39 | 40 | # This failed with gzip 1.4. 41 | echo a-b | zgrep -e - > /dev/null || fail=1 42 | 43 | Exit $fail 44 | -------------------------------------------------------------------------------- /tests/gzip/znew-k.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Check that znew -K works without compress(1). 3 | 4 | # Copyright (C) 2010-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | cat <<'EOF' >compress || framework_failure_ 23 | #!/bin/sh 24 | echo >&2 'compress has been invoked' 25 | exit 1 26 | EOF 27 | chmod +x compress || framework_failure_ 28 | 29 | # Note that the basename must have a length of 6 or greater. 30 | # Otherwise, "test -f $name" below would fail. 31 | name=123456.Z 32 | 33 | printf '%1012977s' ' ' | gzip -c > $name || framework_failure_ 34 | 35 | fail=0 36 | 37 | znew -K $name || fail=1 38 | test -f $name || fail=1 39 | 40 | Exit $fail 41 | -------------------------------------------------------------------------------- /tests/loremOut.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* LOREM_genOut(): 12 | * Generate @size bytes of compressible data using lorem ipsum generator into 13 | * stdout. 14 | */ 15 | void LOREM_genOut(unsigned long long size, unsigned seed); 16 | -------------------------------------------------------------------------------- /tests/rateLimiter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # ################################################################ 4 | # Copyright (c) Meta Platforms, Inc. and affiliates. 5 | # All rights reserved. 6 | # 7 | # This source code is licensed under both the BSD-style license (found in the 8 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 9 | # in the COPYING file in the root directory of this source tree). 10 | # You may select, at your option, one of the above-listed licenses. 11 | # ########################################################################## 12 | 13 | # Rate limiter, replacement for pv 14 | # this rate limiter does not "catch up" after a blocking period 15 | # Limitations: 16 | # - only accepts limit speed in MB/s 17 | 18 | import sys 19 | import time 20 | 21 | MB = 1024 * 1024 22 | rate = float(sys.argv[1]) * MB 23 | start = time.time() 24 | total_read = 0 25 | 26 | # sys.stderr.close() # remove error message, for Ctrl+C 27 | 28 | try: 29 | buf = " " 30 | while len(buf): 31 | now = time.time() 32 | to_read = max(int(rate * (now - start)), 1) 33 | max_buf_size = 1 * MB 34 | to_read = min(to_read, max_buf_size) 35 | start = now 36 | 37 | buf = sys.stdin.buffer.read(to_read) 38 | sys.stdout.buffer.write(buf) 39 | 40 | except (KeyboardInterrupt, BrokenPipeError) as e: 41 | pass 42 | -------------------------------------------------------------------------------- /tests/regression/.gitignore: -------------------------------------------------------------------------------- 1 | # regression test artifacts 2 | data-cache 3 | cache 4 | test 5 | -------------------------------------------------------------------------------- /tests/regression/README.md: -------------------------------------------------------------------------------- 1 | # Regression tests 2 | 3 | The regression tests run zstd in many scenarios and ensures that the size of the compressed results doesn't change. This helps us ensure that we don't accidentally regress zstd's compression ratio. 4 | 5 | These tests get run every night by CircleCI. If the job fails you can read the diff printed by the job to ensure the change isn't a regression. If all is well you can download the `results.csv` artifact and commit the new results. Or you can rebuild it yourself following the instructions below. 6 | 7 | ## Rebuilding results.csv 8 | 9 | From the root of the zstd repo run: 10 | 11 | ``` 12 | # Build the zstd binary 13 | make clean 14 | make -j zstd 15 | 16 | # Build the regression test binary 17 | cd tests/regression 18 | make clean 19 | make -j test 20 | 21 | # Run the regression test 22 | ./test --cache data-cache --zstd ../../zstd --output results.csv 23 | 24 | # Check results.csv to ensure the new results are okay 25 | git diff 26 | 27 | # Then submit the PR 28 | ``` 29 | -------------------------------------------------------------------------------- /tests/regression/levels.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef LEVEL 12 | # error LEVEL(x) must be defined 13 | #endif 14 | #ifndef FAST_LEVEL 15 | # error FAST_LEVEL(x) must be defined 16 | #endif 17 | #ifndef ROW_LEVEL 18 | # error ROW_LEVEL(x, y) must be defined 19 | #endif 20 | 21 | /** 22 | * The levels are chosen to trigger every strategy in every source size, 23 | * as well as some fast levels and the default level. 24 | * If you change the compression levels, you should probably update these. 25 | */ 26 | 27 | FAST_LEVEL(5) 28 | 29 | FAST_LEVEL(3) 30 | 31 | FAST_LEVEL(1) 32 | LEVEL(0) 33 | LEVEL(1) 34 | 35 | LEVEL(3) 36 | LEVEL(4) 37 | /* ROW_LEVEL triggers the row hash (force enabled and disabled) with different 38 | * dictionary strategies, and 16/32/64 row entries based on the level/searchLog. 39 | * 1 == enabled, 2 == disabled. 40 | */ 41 | ROW_LEVEL(5, 1) 42 | ROW_LEVEL(5, 2) /* 16-entry rows */ 43 | LEVEL(5) 44 | LEVEL(6) 45 | ROW_LEVEL(7, 1) 46 | ROW_LEVEL(7, 2) /* 16-entry rows */ 47 | LEVEL(7) 48 | 49 | LEVEL(9) 50 | 51 | ROW_LEVEL(11, 1) 52 | ROW_LEVEL(11, 2) /* 32-entry rows */ 53 | ROW_LEVEL(12, 1) 54 | ROW_LEVEL(12, 2) /* 64-entry rows */ 55 | LEVEL(13) 56 | 57 | LEVEL(16) 58 | 59 | LEVEL(19) 60 | -------------------------------------------------------------------------------- /tests/regression/result.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #include "result.h" 12 | 13 | const char* result_get_error_string(result_t result) { 14 | switch (result_get_error(result)) { 15 | case result_error_ok: 16 | return "okay"; 17 | case result_error_skip: 18 | return "skip"; 19 | case result_error_system_error: 20 | return "system error"; 21 | case result_error_compression_error: 22 | return "compression error"; 23 | case result_error_decompression_error: 24 | return "decompression error"; 25 | case result_error_round_trip_error: 26 | return "round trip error"; 27 | default: 28 | return "unknown error"; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /zlibWrapper/.gitignore: -------------------------------------------------------------------------------- 1 | # object artifacts 2 | *.o 3 | 4 | # Default result files 5 | _* 6 | example 7 | example_zstd.* 8 | example_gz.* 9 | fitblk 10 | fitblk_zstd.* 11 | zwrapbench 12 | foo.gz 13 | 14 | minigzip 15 | minigzip_zstd 16 | example 17 | example_zstd 18 | fitblk 19 | fitblk_zstd 20 | zwrapbench 21 | 22 | # Misc files 23 | *.bat 24 | *.zip 25 | *.txt 26 | 27 | # Directories 28 | minizip/ 29 | -------------------------------------------------------------------------------- /zlibWrapper/BUCK: -------------------------------------------------------------------------------- 1 | cxx_library( 2 | name='zlib_wrapper', 3 | visibility=['PUBLIC'], 4 | exported_linker_flags=['-lz'], 5 | header_namespace='', 6 | exported_headers=['zstd_zlibwrapper.h'], 7 | headers=[ 8 | 'gzcompatibility.h', 9 | 'gzguts.h', 10 | ], 11 | srcs=glob(['*.c']), 12 | deps=[ 13 | '//lib:zstd', 14 | '//lib:zstd_common', 15 | ], 16 | ) 17 | 18 | cxx_binary( 19 | name='minigzip', 20 | srcs=['examples/minigzip.c'], 21 | deps=[':zlib_wrapper'], 22 | ) 23 | -------------------------------------------------------------------------------- /zlibWrapper/gzclose.c: -------------------------------------------------------------------------------- 1 | /* gzclose.c contains minimal changes required to be compiled with zlibWrapper: 2 | * - gz_statep was converted to union to work with -Wstrict-aliasing=1 */ 3 | 4 | /* gzclose.c -- zlib gzclose() function 5 | * Copyright (C) 2004, 2010 Mark Adler 6 | * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html 7 | */ 8 | 9 | #include "gzguts.h" 10 | 11 | /* gzclose() is in a separate file so that it is linked in only if it is used. 12 | That way the other gzclose functions can be used instead to avoid linking in 13 | unneeded compression or decompression routines. */ 14 | int ZEXPORT gzclose(gzFile file) { 15 | #ifndef NO_GZCOMPRESS 16 | gz_statep state; 17 | 18 | if (file == NULL) 19 | return Z_STREAM_ERROR; 20 | state.file = file; 21 | 22 | return state.state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file); 23 | #else 24 | return gzclose_r(file); 25 | #endif 26 | } 27 | --------------------------------------------------------------------------------