├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── builder └── Dockerfile ├── dict.go ├── dict_example_test.go ├── dict_test.go ├── doc.go ├── go.mod ├── gozstd.go ├── gozstd_example_test.go ├── gozstd_test.go ├── gozstd_timing_test.go ├── libzstd_darwin_amd64.a ├── libzstd_darwin_amd64.go ├── libzstd_darwin_arm64.a ├── libzstd_darwin_arm64.go ├── libzstd_freebsd_amd64.a ├── libzstd_freebsd_amd64.go ├── libzstd_illumos_amd64.a ├── libzstd_illumos_amd64.go ├── libzstd_linux_amd64.a ├── libzstd_linux_amd64.go ├── libzstd_linux_arm.a ├── libzstd_linux_arm.go ├── libzstd_linux_arm64.a ├── libzstd_linux_arm64.go ├── libzstd_linux_musl_amd64.a ├── libzstd_linux_musl_amd64.go ├── libzstd_linux_musl_arm64.a ├── libzstd_linux_musl_arm64.go ├── libzstd_windows_amd64.a ├── libzstd_windows_amd64.go ├── reader.go ├── reader_example_test.go ├── reader_test.go ├── reader_timing_test.go ├── stream.go ├── stream_test.go ├── stream_timing_test.go ├── writer.go ├── writer_example_test.go ├── writer_test.go ├── writer_timing_test.go ├── zdict.h ├── zstd.h ├── zstd ├── .buckconfig ├── .buckversion ├── .circleci │ ├── config.yml │ └── images │ │ └── primary │ │ └── Dockerfile ├── .cirrus.yml ├── .gitattributes ├── .github │ ├── ISSUE_TEMPLATE │ │ ├── bug_report.md │ │ └── feature_request.md │ ├── dependabot.yml │ └── workflows │ │ ├── commit.yml │ │ ├── dev-long-tests.yml │ │ ├── dev-short-tests.yml │ │ ├── nightly.yml │ │ ├── publish-release-artifacts.yml │ │ ├── scorecards.yml │ │ └── windows-artifacts.yml ├── .gitignore ├── CHANGELOG ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── COPYING ├── LICENSE ├── Makefile ├── Package.swift ├── README.md ├── SECURITY.md ├── TESTING.md ├── build │ ├── .gitignore │ ├── LICENSE │ ├── README.md │ ├── VS2008 │ │ ├── fullbench │ │ │ └── fullbench.vcproj │ │ ├── fuzzer │ │ │ └── fuzzer.vcproj │ │ ├── zstd.sln │ │ ├── zstd │ │ │ └── zstd.vcproj │ │ └── zstdlib │ │ │ └── zstdlib.vcproj │ ├── VS2010 │ │ ├── CompileAsCpp.props │ │ ├── datagen │ │ │ └── datagen.vcxproj │ │ ├── fullbench-dll │ │ │ └── fullbench-dll.vcxproj │ │ ├── fullbench │ │ │ └── fullbench.vcxproj │ │ ├── fuzzer │ │ │ └── fuzzer.vcxproj │ │ ├── libzstd-dll │ │ │ ├── libzstd-dll.rc │ │ │ └── libzstd-dll.vcxproj │ │ ├── libzstd │ │ │ └── libzstd.vcxproj │ │ ├── zstd.sln │ │ └── zstd │ │ │ ├── zstd.rc │ │ │ └── zstd.vcxproj │ ├── VS_scripts │ │ ├── README.md │ │ ├── build.VS2010.cmd │ │ ├── build.VS2012.cmd │ │ ├── build.VS2013.cmd │ │ ├── build.VS2015.cmd │ │ ├── build.VS2017.cmd │ │ ├── build.VS2017Community.cmd │ │ ├── build.VS2017Enterprise.cmd │ │ ├── build.VS2017Professional.cmd │ │ └── build.generic.cmd │ ├── cmake │ │ ├── .gitignore │ │ ├── CMakeLists.txt │ │ ├── CMakeModules │ │ │ ├── AddZstdCompilationFlags.cmake │ │ │ ├── FindLibLZ4.cmake │ │ │ ├── GetZstdLibraryVersion.cmake │ │ │ └── JoinPaths.cmake │ │ ├── README.md │ │ ├── contrib │ │ │ ├── CMakeLists.txt │ │ │ ├── gen_html │ │ │ │ └── CMakeLists.txt │ │ │ └── pzstd │ │ │ │ └── CMakeLists.txt │ │ ├── lib │ │ │ ├── .gitignore │ │ │ ├── CMakeLists.txt │ │ │ └── cmake_uninstall.cmake.in │ │ ├── programs │ │ │ ├── .gitignore │ │ │ └── CMakeLists.txt │ │ ├── tests │ │ │ ├── .gitignore │ │ │ └── CMakeLists.txt │ │ └── zstdConfig.cmake.in │ ├── meson │ │ ├── GetZstdLibraryVersion.py │ │ ├── InstallSymlink.py │ │ ├── README.md │ │ ├── contrib │ │ │ ├── gen_html │ │ │ │ └── meson.build │ │ │ ├── meson.build │ │ │ └── pzstd │ │ │ │ └── meson.build │ │ ├── lib │ │ │ └── meson.build │ │ ├── meson.build │ │ ├── meson_options.txt │ │ ├── programs │ │ │ └── meson.build │ │ └── tests │ │ │ ├── meson.build │ │ │ └── valgrindTest.py │ └── single_file_libs │ │ ├── .gitignore │ │ ├── README.md │ │ ├── build_decoder_test.sh │ │ ├── build_library_test.sh │ │ ├── combine.py │ │ ├── combine.sh │ │ ├── create_single_file_decoder.sh │ │ ├── create_single_file_library.sh │ │ ├── examples │ │ ├── README.md │ │ ├── emscripten.c │ │ ├── roundtrip.c │ │ ├── shell.html │ │ ├── simple.c │ │ ├── testcard-dxt1.inl │ │ ├── testcard-zstd.inl │ │ └── testcard.png │ │ ├── zstd-in.c │ │ └── zstddeclib-in.c ├── contrib │ ├── VS2005 │ │ ├── README.md │ │ ├── fullbench │ │ │ └── fullbench.vcproj │ │ ├── fuzzer │ │ │ └── fuzzer.vcproj │ │ ├── zstd.sln │ │ ├── zstd │ │ │ └── zstd.vcproj │ │ └── zstdlib │ │ │ └── zstdlib.vcproj │ ├── cleanTabs │ ├── diagnose_corruption │ │ ├── .gitignore │ │ ├── Makefile │ │ └── check_flipped_bits.c │ ├── docker │ │ ├── Dockerfile │ │ └── README.md │ ├── externalSequenceProducer │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── main.c │ │ ├── sequence_producer.c │ │ └── sequence_producer.h │ ├── freestanding_lib │ │ └── freestanding.py │ ├── gen_html │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── gen-zstd-manual.sh │ │ └── gen_html.cpp │ ├── largeNbDicts │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ └── largeNbDicts.c │ ├── linux-kernel │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── btrfs-benchmark.sh │ │ ├── btrfs-extract-benchmark.sh │ │ ├── decompress_sources.h │ │ ├── linux.mk │ │ ├── linux_zstd.h │ │ ├── mem.h │ │ ├── squashfs-benchmark.sh │ │ ├── test │ │ │ ├── Makefile │ │ │ ├── include │ │ │ │ ├── asm │ │ │ │ │ └── unaligned.h │ │ │ │ └── linux │ │ │ │ │ ├── compiler.h │ │ │ │ │ ├── errno.h │ │ │ │ │ ├── kernel.h │ │ │ │ │ ├── limits.h │ │ │ │ │ ├── math64.h │ │ │ │ │ ├── module.h │ │ │ │ │ ├── printk.h │ │ │ │ │ ├── stddef.h │ │ │ │ │ ├── swab.h │ │ │ │ │ ├── types.h │ │ │ │ │ └── xxhash.h │ │ │ ├── macro-test.sh │ │ │ ├── static_test.c │ │ │ └── test.c │ │ ├── zstd_common_module.c │ │ ├── zstd_compress_module.c │ │ ├── zstd_decompress_module.c │ │ └── zstd_deps.h │ ├── match_finders │ │ ├── README.md │ │ ├── zstd_edist.c │ │ └── zstd_edist.h │ ├── premake │ │ ├── premake4.lua │ │ └── zstd.lua │ ├── pzstd │ │ ├── .gitignore │ │ ├── BUCK │ │ ├── ErrorHolder.h │ │ ├── Logging.h │ │ ├── Makefile │ │ ├── Options.cpp │ │ ├── Options.h │ │ ├── Pzstd.cpp │ │ ├── Pzstd.h │ │ ├── README.md │ │ ├── SkippableFrame.cpp │ │ ├── SkippableFrame.h │ │ ├── images │ │ │ ├── Cspeed.png │ │ │ └── Dspeed.png │ │ ├── main.cpp │ │ ├── test │ │ │ ├── BUCK │ │ │ ├── OptionsTest.cpp │ │ │ ├── PzstdTest.cpp │ │ │ ├── RoundTrip.h │ │ │ └── RoundTripTest.cpp │ │ └── utils │ │ │ ├── BUCK │ │ │ ├── Buffer.h │ │ │ ├── FileSystem.h │ │ │ ├── Likely.h │ │ │ ├── Portability.h │ │ │ ├── Range.h │ │ │ ├── ResourcePool.h │ │ │ ├── ScopeGuard.h │ │ │ ├── ThreadPool.h │ │ │ ├── WorkQueue.h │ │ │ └── test │ │ │ ├── BUCK │ │ │ ├── BufferTest.cpp │ │ │ ├── RangeTest.cpp │ │ │ ├── ResourcePoolTest.cpp │ │ │ ├── ScopeGuardTest.cpp │ │ │ ├── ThreadPoolTest.cpp │ │ │ └── WorkQueueTest.cpp │ ├── recovery │ │ ├── Makefile │ │ └── recover_directory.c │ ├── seekable_format │ │ ├── README.md │ │ ├── examples │ │ │ ├── .gitignore │ │ │ ├── Makefile │ │ │ ├── parallel_compression.c │ │ │ ├── parallel_processing.c │ │ │ ├── seekable_compression.c │ │ │ ├── seekable_decompression.c │ │ │ └── seekable_decompression_mem.c │ │ ├── tests │ │ │ ├── .gitignore │ │ │ ├── Makefile │ │ │ └── seekable_tests.c │ │ ├── zstd_seekable.h │ │ ├── zstd_seekable_compression_format.md │ │ ├── zstdseek_compress.c │ │ └── zstdseek_decompress.c │ ├── seqBench │ │ ├── Makefile │ │ └── seqBench.c │ └── snap │ │ └── snapcraft.yaml ├── doc │ ├── README.md │ ├── decompressor_errata.md │ ├── decompressor_permissive.md │ ├── educational_decoder │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── harness.c │ │ ├── zstd_decompress.c │ │ └── zstd_decompress.h │ ├── images │ │ ├── CSpeed2.png │ │ ├── DCspeed5.png │ │ ├── DSpeed3.png │ │ ├── cdict_v136.png │ │ ├── dict-cr.png │ │ ├── dict-cs.png │ │ ├── dict-ds.png │ │ ├── zstd_cdict_v1_3_5.png │ │ └── zstd_logo86.png │ ├── zstd_compression_format.md │ └── zstd_manual.html ├── examples │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── common.h │ ├── dictionary_compression.c │ ├── dictionary_decompression.c │ ├── multiple_simple_compression.c │ ├── multiple_streaming_compression.c │ ├── simple_compression.c │ ├── simple_decompression.c │ ├── streaming_compression.c │ ├── streaming_compression_thread_pool.c │ ├── streaming_decompression.c │ └── streaming_memory_usage.c ├── lib │ ├── .gitignore │ ├── BUCK │ ├── Makefile │ ├── README.md │ ├── common │ │ ├── allocations.h │ │ ├── bits.h │ │ ├── bitstream.h │ │ ├── compiler.h │ │ ├── cpu.h │ │ ├── debug.c │ │ ├── debug.h │ │ ├── entropy_common.c │ │ ├── error_private.c │ │ ├── error_private.h │ │ ├── fse.h │ │ ├── fse_decompress.c │ │ ├── huf.h │ │ ├── mem.h │ │ ├── pool.c │ │ ├── pool.h │ │ ├── portability_macros.h │ │ ├── threading.c │ │ ├── threading.h │ │ ├── xxhash.c │ │ ├── xxhash.h │ │ ├── zstd_common.c │ │ ├── zstd_deps.h │ │ ├── zstd_internal.h │ │ └── zstd_trace.h │ ├── compress │ │ ├── clevels.h │ │ ├── fse_compress.c │ │ ├── hist.c │ │ ├── hist.h │ │ ├── huf_compress.c │ │ ├── zstd_compress.c │ │ ├── zstd_compress_internal.h │ │ ├── zstd_compress_literals.c │ │ ├── zstd_compress_literals.h │ │ ├── zstd_compress_sequences.c │ │ ├── zstd_compress_sequences.h │ │ ├── zstd_compress_superblock.c │ │ ├── zstd_compress_superblock.h │ │ ├── zstd_cwksp.h │ │ ├── zstd_double_fast.c │ │ ├── zstd_double_fast.h │ │ ├── zstd_fast.c │ │ ├── zstd_fast.h │ │ ├── zstd_lazy.c │ │ ├── zstd_lazy.h │ │ ├── zstd_ldm.c │ │ ├── zstd_ldm.h │ │ ├── zstd_ldm_geartab.h │ │ ├── zstd_opt.c │ │ ├── zstd_opt.h │ │ ├── zstdmt_compress.c │ │ └── zstdmt_compress.h │ ├── decompress │ │ ├── huf_decompress.c │ │ ├── huf_decompress_amd64.S │ │ ├── zstd_ddict.c │ │ ├── zstd_ddict.h │ │ ├── zstd_decompress.c │ │ ├── zstd_decompress_block.c │ │ ├── zstd_decompress_block.h │ │ └── zstd_decompress_internal.h │ ├── deprecated │ │ ├── zbuff.h │ │ ├── zbuff_common.c │ │ ├── zbuff_compress.c │ │ └── zbuff_decompress.c │ ├── dictBuilder │ │ ├── cover.c │ │ ├── cover.h │ │ ├── divsufsort.c │ │ ├── divsufsort.h │ │ ├── fastcover.c │ │ └── zdict.c │ ├── dll │ │ └── example │ │ │ ├── Makefile │ │ │ ├── README.md │ │ │ ├── build_package.bat │ │ │ ├── fullbench-dll.sln │ │ │ └── fullbench-dll.vcxproj │ ├── legacy │ │ ├── zstd_legacy.h │ │ ├── zstd_v01.c │ │ ├── zstd_v01.h │ │ ├── zstd_v02.c │ │ ├── zstd_v02.h │ │ ├── zstd_v03.c │ │ ├── zstd_v03.h │ │ ├── zstd_v04.c │ │ ├── zstd_v04.h │ │ ├── zstd_v05.c │ │ ├── zstd_v05.h │ │ ├── zstd_v06.c │ │ ├── zstd_v06.h │ │ ├── zstd_v07.c │ │ └── zstd_v07.h │ ├── libzstd.mk │ ├── libzstd.pc.in │ ├── module.modulemap │ ├── zdict.h │ ├── zstd.h │ └── zstd_errors.h ├── programs │ ├── .gitignore │ ├── BUCK │ ├── Makefile │ ├── README.md │ ├── benchfn.c │ ├── benchfn.h │ ├── benchzstd.c │ ├── benchzstd.h │ ├── datagen.c │ ├── datagen.h │ ├── dibio.c │ ├── dibio.h │ ├── fileio.c │ ├── fileio.h │ ├── fileio_asyncio.c │ ├── fileio_asyncio.h │ ├── fileio_common.h │ ├── fileio_types.h │ ├── lorem.c │ ├── lorem.h │ ├── platform.h │ ├── timefn.c │ ├── timefn.h │ ├── util.c │ ├── util.h │ ├── windres │ │ ├── verrsrc.h │ │ ├── zstd.rc │ │ ├── zstd32.res │ │ └── zstd64.res │ ├── zstd.1 │ ├── zstd.1.md │ ├── zstdcli.c │ ├── zstdcli_trace.c │ ├── zstdcli_trace.h │ ├── zstdgrep │ ├── zstdgrep.1 │ ├── zstdgrep.1.md │ ├── zstdless │ ├── zstdless.1 │ └── zstdless.1.md ├── tests │ ├── .gitignore │ ├── DEPRECATED-test-zstd-speed.py │ ├── Makefile │ ├── README.md │ ├── automated_benchmarking.py │ ├── bigdict.c │ ├── checkTag.c │ ├── check_size.py │ ├── cli-tests │ │ ├── .gitignore │ │ ├── README.md │ │ ├── basic │ │ │ ├── args.sh │ │ │ ├── args.sh.exit │ │ │ ├── args.sh.stderr.glob │ │ │ ├── help.sh │ │ │ ├── help.sh.stdout.glob │ │ │ ├── memlimit.sh │ │ │ ├── memlimit.sh.stderr.exact │ │ │ ├── memlimit.sh.stdout.exact │ │ │ ├── output_dir.sh │ │ │ ├── output_dir.sh.stderr.exact │ │ │ ├── output_dir.sh.stdout.exact │ │ │ ├── version.sh │ │ │ └── version.sh.stdout.glob │ │ ├── bin │ │ │ ├── cmp_size │ │ │ ├── datagen │ │ │ ├── die │ │ │ ├── println │ │ │ ├── unzstd │ │ │ ├── zstd │ │ │ ├── zstdcat │ │ │ ├── zstdgrep │ │ │ └── zstdless │ │ ├── cltools │ │ │ ├── setup │ │ │ ├── zstdgrep.sh │ │ │ ├── zstdgrep.sh.exit │ │ │ ├── zstdgrep.sh.stderr.exact │ │ │ ├── zstdgrep.sh.stdout.glob │ │ │ ├── zstdless.sh │ │ │ ├── zstdless.sh.stderr.exact │ │ │ └── zstdless.sh.stdout.glob │ │ ├── common │ │ │ ├── format.sh │ │ │ ├── mtime.sh │ │ │ ├── permissions.sh │ │ │ └── platform.sh │ │ ├── compression │ │ │ ├── adapt.sh │ │ │ ├── basic.sh │ │ │ ├── compress-literals.sh │ │ │ ├── format.sh │ │ │ ├── golden.sh │ │ │ ├── gzip-compat.sh │ │ │ ├── levels.sh │ │ │ ├── levels.sh.stderr.exact │ │ │ ├── long-distance-matcher.sh │ │ │ ├── multi-threaded.sh │ │ │ ├── multi-threaded.sh.stderr.exact │ │ │ ├── multiple-files.sh │ │ │ ├── multiple-files.sh.stdout.exact │ │ │ ├── row-match-finder.sh │ │ │ ├── setup │ │ │ ├── stream-size.sh │ │ │ ├── verbose-wlog.sh │ │ │ ├── verbose-wlog.sh.stderr.glob │ │ │ ├── verbose-wlog.sh.stdout.glob │ │ │ ├── window-resize.sh │ │ │ ├── window-resize.sh.stderr.ignore │ │ │ └── window-resize.sh.stdout.glob │ │ ├── decompression │ │ │ ├── detectErrors.sh │ │ │ ├── golden.sh │ │ │ ├── pass-through.sh │ │ │ ├── pass-through.sh.stderr.exact │ │ │ └── pass-through.sh.stdout.exact │ │ ├── dict-builder │ │ │ ├── empty-input.sh │ │ │ ├── empty-input.sh.stderr.exact │ │ │ ├── no-inputs.sh │ │ │ ├── no-inputs.sh.exit │ │ │ └── no-inputs.sh.stderr.exact │ │ ├── dictionaries │ │ │ ├── dictionary-mismatch.sh │ │ │ ├── dictionary-mismatch.sh.stderr.exact │ │ │ ├── golden.sh │ │ │ ├── setup │ │ │ └── setup_once │ │ ├── file-handling │ │ │ ├── directory-mirror.sh │ │ │ ├── directory-mirror.sh.stderr.exact │ │ │ └── directory-mirror.sh.stdout.exact │ │ ├── file-stat │ │ │ ├── compress-file-to-dir-without-write-perm.sh │ │ │ ├── compress-file-to-dir-without-write-perm.sh.stderr.exact │ │ │ ├── compress-file-to-file.sh │ │ │ ├── compress-file-to-file.sh.stderr.exact │ │ │ ├── compress-file-to-stdout.sh │ │ │ ├── compress-file-to-stdout.sh.stderr.exact │ │ │ ├── compress-stdin-to-file.sh │ │ │ ├── compress-stdin-to-file.sh.stderr.exact │ │ │ ├── compress-stdin-to-stdout.sh │ │ │ ├── compress-stdin-to-stdout.sh.stderr.exact │ │ │ ├── decompress-file-to-file.sh │ │ │ ├── decompress-file-to-file.sh.stderr.exact │ │ │ ├── decompress-file-to-stdout.sh │ │ │ ├── decompress-file-to-stdout.sh.stderr.exact │ │ │ ├── decompress-stdin-to-file.sh │ │ │ ├── decompress-stdin-to-file.sh.stderr.exact │ │ │ ├── decompress-stdin-to-stdout.sh │ │ │ └── decompress-stdin-to-stdout.sh.stderr.exact │ │ ├── progress │ │ │ ├── no-progress.sh │ │ │ ├── no-progress.sh.stderr.glob │ │ │ ├── progress.sh │ │ │ └── progress.sh.stderr.glob │ │ ├── run.py │ │ └── zstd-symlinks │ │ │ ├── setup │ │ │ ├── zstdcat.sh │ │ │ └── zstdcat.sh.stdout.exact │ ├── datagencli.c │ ├── decodecorpus.c │ ├── dict-files │ │ └── zero-weight-dict │ ├── external_matchfinder.c │ ├── external_matchfinder.h │ ├── fullbench.c │ ├── fuzz │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── block_decompress.c │ │ ├── block_round_trip.c │ │ ├── decompress_cross_format.c │ │ ├── decompress_dstSize_tooSmall.c │ │ ├── dictionary_decompress.c │ │ ├── dictionary_loader.c │ │ ├── dictionary_round_trip.c │ │ ├── dictionary_stream_round_trip.c │ │ ├── fse_read_ncount.c │ │ ├── fuzz.h │ │ ├── fuzz.py │ │ ├── fuzz_data_producer.c │ │ ├── fuzz_data_producer.h │ │ ├── fuzz_helpers.c │ │ ├── fuzz_helpers.h │ │ ├── fuzz_third_party_seq_prod.h │ │ ├── generate_sequences.c │ │ ├── huf_decompress.c │ │ ├── huf_round_trip.c │ │ ├── raw_dictionary_round_trip.c │ │ ├── regression_driver.c │ │ ├── seekable_roundtrip.c │ │ ├── seq_prod_fuzz_example │ │ │ ├── Makefile │ │ │ ├── README.md │ │ │ └── example_seq_prod.c │ │ ├── sequence_compression_api.c │ │ ├── simple_compress.c │ │ ├── simple_decompress.c │ │ ├── simple_round_trip.c │ │ ├── stream_decompress.c │ │ ├── stream_round_trip.c │ │ ├── zstd_frame_info.c │ │ ├── zstd_helpers.c │ │ └── zstd_helpers.h │ ├── fuzzer.c │ ├── golden-compression │ │ ├── PR-3517-block-splitter-corruption-test │ │ ├── http │ │ ├── huffman-compressed-larger │ │ └── large-literal-and-match-lengths │ ├── golden-decompression-errors │ │ ├── .gitignore │ │ ├── off0.bin.zst │ │ └── zeroSeq_extraneous.zst │ ├── golden-decompression │ │ ├── block-128k.zst │ │ ├── empty-block.zst │ │ ├── rle-first-block.zst │ │ └── zeroSeq_2B.zst │ ├── golden-dictionaries │ │ └── http-dict-missing-symbols │ ├── gzip │ │ ├── Makefile │ │ ├── gzip-env.sh │ │ ├── helin-segv.sh │ │ ├── help-version.sh │ │ ├── hufts-segv.gz │ │ ├── hufts.sh │ │ ├── init.cfg │ │ ├── init.sh │ │ ├── keep.sh │ │ ├── list.sh │ │ ├── memcpy-abuse.sh │ │ ├── mixed.sh │ │ ├── null-suffix-clobber.sh │ │ ├── stdin.sh │ │ ├── test-driver.sh │ │ ├── trailing-nul.sh │ │ ├── unpack-invalid.sh │ │ ├── z-suffix.sh │ │ ├── zdiff.sh │ │ ├── zgrep-context.sh │ │ ├── zgrep-f.sh │ │ ├── zgrep-signal.sh │ │ └── znew-k.sh │ ├── invalidDictionaries.c │ ├── legacy.c │ ├── libzstd_builds.sh │ ├── longmatch.c │ ├── loremOut.c │ ├── loremOut.h │ ├── paramgrill.c │ ├── playTests.sh │ ├── poolTests.c │ ├── rateLimiter.py │ ├── regression │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── config.c │ │ ├── config.h │ │ ├── data.c │ │ ├── data.h │ │ ├── levels.h │ │ ├── method.c │ │ ├── method.h │ │ ├── result.c │ │ ├── result.h │ │ ├── results.csv │ │ └── test.c │ ├── roundTripCrash.c │ ├── seqgen.c │ ├── seqgen.h │ ├── test-license.py │ ├── test-variants.sh │ ├── test-zstd-versions.py │ └── zstreamtest.c └── zlibWrapper │ ├── .gitignore │ ├── BUCK │ ├── Makefile │ ├── README.md │ ├── examples │ ├── example.c │ ├── example_original.c │ ├── fitblk.c │ ├── fitblk_original.c │ ├── minigzip.c │ └── zwrapbench.c │ ├── gzclose.c │ ├── gzcompatibility.h │ ├── gzguts.h │ ├── gzlib.c │ ├── gzread.c │ ├── gzwrite.c │ ├── zstd_zlibwrapper.c │ └── zstd_zlibwrapper.h └── zstd_errors.h /.gitignore: -------------------------------------------------------------------------------- 1 | tags 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | os: 4 | - linux 5 | - osx 6 | - freebsd 7 | 8 | go: 9 | - 1.14 10 | 11 | script: 12 | # build test for supported platforms 13 | - GOARCH=amd64 go build 14 | 15 | # run tests on a standard platform 16 | - go test -v -coverprofile=coverage.txt -covermode=atomic 17 | - go test -v -race 18 | 19 | after_success: 20 | # Upload coverage results to codecov.io 21 | - bash <(curl -s https://codecov.io/bash) 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Aliaksandr Valialkin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /builder/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG builder_image 2 | FROM $builder_image 3 | RUN apk --update --no-cache add ca-certificates 4 | RUN apk --no-cache add make git 5 | 6 | ENTRYPOINT ["/bin/sh"] -------------------------------------------------------------------------------- /dict_example_test.go: -------------------------------------------------------------------------------- 1 | package gozstd 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | ) 7 | 8 | func ExampleBuildDict() { 9 | // Collect samples for the dictionary. 10 | var samples [][]byte 11 | for i := 0; i < 1000; i++ { 12 | sample := fmt.Sprintf("this is a dict sample number %d", i) 13 | samples = append(samples, []byte(sample)) 14 | } 15 | 16 | // Build a dictionary with the desired size of 8Kb. 17 | dict := BuildDict(samples, 8*1024) 18 | 19 | // Now the dict may be used for compression/decompression. 20 | 21 | // Create CDict from the dict. 22 | cd, err := NewCDict(dict) 23 | if err != nil { 24 | log.Fatalf("cannot create CDict: %s", err) 25 | } 26 | defer cd.Release() 27 | 28 | // Compress multiple blocks with the same CDict. 29 | var compressedBlocks [][]byte 30 | for i := 0; i < 3; i++ { 31 | plainData := fmt.Sprintf("this is line %d for dict compression", i) 32 | compressedData := CompressDict(nil, []byte(plainData), cd) 33 | compressedBlocks = append(compressedBlocks, compressedData) 34 | } 35 | 36 | // The compressedData must be decompressed with the same dict. 37 | 38 | // Create DDict from the dict. 39 | dd, err := NewDDict(dict) 40 | if err != nil { 41 | log.Fatalf("cannot create DDict: %s", err) 42 | } 43 | defer dd.Release() 44 | 45 | // Decompress multiple blocks with the same DDict. 46 | for _, compressedData := range compressedBlocks { 47 | decompressedData, err := DecompressDict(nil, compressedData, dd) 48 | if err != nil { 49 | log.Fatalf("cannot decompress data: %s", err) 50 | } 51 | fmt.Printf("%s\n", decompressedData) 52 | } 53 | 54 | // Output: 55 | // this is line 0 for dict compression 56 | // this is line 1 for dict compression 57 | // this is line 2 for dict compression 58 | } 59 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package gozstd is Go wrapper for zstd. 2 | // 3 | // Gozstd is used in https://github.com/VictoriaMetrics/VictoriaMetrics . 4 | package gozstd 5 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/valyala/gozstd 2 | 3 | go 1.12 4 | -------------------------------------------------------------------------------- /libzstd_darwin_amd64.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/libzstd_darwin_amd64.a -------------------------------------------------------------------------------- /libzstd_darwin_amd64.go: -------------------------------------------------------------------------------- 1 | package gozstd 2 | 3 | /* 4 | #cgo LDFLAGS: ${SRCDIR}/libzstd_darwin_amd64.a 5 | */ 6 | import "C" 7 | -------------------------------------------------------------------------------- /libzstd_darwin_arm64.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/libzstd_darwin_arm64.a -------------------------------------------------------------------------------- /libzstd_darwin_arm64.go: -------------------------------------------------------------------------------- 1 | package gozstd 2 | 3 | /* 4 | #cgo LDFLAGS: ${SRCDIR}/libzstd_darwin_arm64.a 5 | */ 6 | import "C" 7 | -------------------------------------------------------------------------------- /libzstd_freebsd_amd64.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/libzstd_freebsd_amd64.a -------------------------------------------------------------------------------- /libzstd_freebsd_amd64.go: -------------------------------------------------------------------------------- 1 | package gozstd 2 | 3 | /* 4 | #cgo LDFLAGS: ${SRCDIR}/libzstd_freebsd_amd64.a 5 | */ 6 | import "C" 7 | -------------------------------------------------------------------------------- /libzstd_illumos_amd64.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/libzstd_illumos_amd64.a -------------------------------------------------------------------------------- /libzstd_illumos_amd64.go: -------------------------------------------------------------------------------- 1 | package gozstd 2 | 3 | /* 4 | #cgo LDFLAGS: ${SRCDIR}/libzstd_illumos_amd64.a 5 | */ 6 | import "C" 7 | -------------------------------------------------------------------------------- /libzstd_linux_amd64.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/libzstd_linux_amd64.a -------------------------------------------------------------------------------- /libzstd_linux_amd64.go: -------------------------------------------------------------------------------- 1 | // +build !musl 2 | 3 | package gozstd 4 | 5 | /* 6 | #cgo LDFLAGS: ${SRCDIR}/libzstd_linux_amd64.a 7 | */ 8 | import "C" 9 | -------------------------------------------------------------------------------- /libzstd_linux_arm.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/libzstd_linux_arm.a -------------------------------------------------------------------------------- /libzstd_linux_arm.go: -------------------------------------------------------------------------------- 1 | package gozstd 2 | 3 | /* 4 | #cgo LDFLAGS: ${SRCDIR}/libzstd_linux_arm.a 5 | */ 6 | import "C" 7 | -------------------------------------------------------------------------------- /libzstd_linux_arm64.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/libzstd_linux_arm64.a -------------------------------------------------------------------------------- /libzstd_linux_arm64.go: -------------------------------------------------------------------------------- 1 | //go:build !musl 2 | // +build !musl 3 | 4 | package gozstd 5 | 6 | /* 7 | #cgo LDFLAGS: ${SRCDIR}/libzstd_linux_arm64.a 8 | */ 9 | import "C" 10 | -------------------------------------------------------------------------------- /libzstd_linux_musl_amd64.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/libzstd_linux_musl_amd64.a -------------------------------------------------------------------------------- /libzstd_linux_musl_amd64.go: -------------------------------------------------------------------------------- 1 | // +build musl 2 | 3 | package gozstd 4 | 5 | /* 6 | #cgo LDFLAGS: ${SRCDIR}/libzstd_linux_musl_amd64.a 7 | */ 8 | import "C" 9 | -------------------------------------------------------------------------------- /libzstd_linux_musl_arm64.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/libzstd_linux_musl_arm64.a -------------------------------------------------------------------------------- /libzstd_linux_musl_arm64.go: -------------------------------------------------------------------------------- 1 | //go:build musl 2 | // +build musl 3 | 4 | package gozstd 5 | 6 | /* 7 | #cgo LDFLAGS: ${SRCDIR}/libzstd_linux_musl_arm64.a 8 | */ 9 | import "C" 10 | -------------------------------------------------------------------------------- /libzstd_windows_amd64.a: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/libzstd_windows_amd64.a -------------------------------------------------------------------------------- /libzstd_windows_amd64.go: -------------------------------------------------------------------------------- 1 | package gozstd 2 | 3 | /* 4 | #cgo LDFLAGS: ${SRCDIR}/libzstd_windows_amd64.a 5 | */ 6 | import "C" 7 | -------------------------------------------------------------------------------- /reader_example_test.go: -------------------------------------------------------------------------------- 1 | package gozstd 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | ) 10 | 11 | func ExampleReader() { 12 | // Compress the data. 13 | compressedData := Compress(nil, []byte("line 0\nline 1\nline 2")) 14 | 15 | // Read it via Reader. 16 | r := bytes.NewReader(compressedData) 17 | zr := NewReader(r) 18 | defer zr.Release() 19 | 20 | var a []int 21 | for i := 0; i < 3; i++ { 22 | var n int 23 | if _, err := fmt.Fscanf(zr, "line %d\n", &n); err != nil { 24 | log.Fatalf("cannot read line: %s", err) 25 | } 26 | a = append(a, n) 27 | } 28 | 29 | // Make sure there are no data left in zr. 30 | buf := make([]byte, 1) 31 | if _, err := zr.Read(buf); err != io.EOF { 32 | log.Fatalf("unexpected error; got %v; want %v", err, io.EOF) 33 | } 34 | 35 | fmt.Println(a) 36 | 37 | // Output: 38 | // [0 1 2] 39 | } 40 | 41 | func ExampleReader_Reset() { 42 | zr := NewReader(nil) 43 | defer zr.Release() 44 | 45 | // Read from different sources using the same Reader. 46 | for i := 0; i < 3; i++ { 47 | compressedData := Compress(nil, []byte(fmt.Sprintf("line %d", i))) 48 | r := bytes.NewReader(compressedData) 49 | zr.Reset(r, nil) 50 | 51 | data, err := ioutil.ReadAll(zr) 52 | if err != nil { 53 | log.Fatalf("unexpected error when reading compressed data: %s", err) 54 | } 55 | fmt.Printf("%s\n", data) 56 | } 57 | 58 | // Output: 59 | // line 0 60 | // line 1 61 | // line 2 62 | } 63 | -------------------------------------------------------------------------------- /zstd/.buckconfig: -------------------------------------------------------------------------------- 1 | [cxx] 2 | cppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4 3 | cflags = -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wpointer-arith 4 | cxxppflags = -DXXH_NAMESPACE=ZSTD_ -DZSTD_LEGACY_SUPPORT=4 5 | cxxflags = -std=c++11 -Wno-deprecated-declarations 6 | gtest_dep = //contrib/pzstd:gtest 7 | 8 | [httpserver] 9 | port = 0 10 | -------------------------------------------------------------------------------- /zstd/.buckversion: -------------------------------------------------------------------------------- 1 | c8dec2e8da52d483f6dd7c6cd2ad694e8e6fed2b 2 | -------------------------------------------------------------------------------- /zstd/.circleci/images/primary/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM circleci/buildpack-deps@sha256:f6f10c11b7b8ccfd4f4a5b830c3256803604ce61292b60cb22e26b12f62b0e8c 2 | 3 | RUN sudo dpkg --add-architecture i386 4 | RUN sudo apt-get -y -qq update 5 | RUN sudo apt-get -y install \ 6 | gcc-multilib-powerpc-linux-gnu gcc-arm-linux-gnueabi \ 7 | libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross \ 8 | libc6-dev-ppc64-powerpc-cross zstd gzip coreutils \ 9 | libcurl4-openssl-dev 10 | -------------------------------------------------------------------------------- /zstd/.cirrus.yml: -------------------------------------------------------------------------------- 1 | task: 2 | name: FreeBSD (shortest) 3 | freebsd_instance: 4 | matrix: 5 | image_family: freebsd-14-0 6 | image_family: freebsd-13-2 7 | install_script: pkg install -y gmake coreutils 8 | script: | 9 | MOREFLAGS="-Werror" gmake -j all 10 | gmake shortest 11 | -------------------------------------------------------------------------------- /zstd/.gitattributes: -------------------------------------------------------------------------------- 1 | # Set the default behavior 2 | * text eol=lf 3 | 4 | # Explicitly declare source files 5 | *.c text eol=lf 6 | *.h text eol=lf 7 | 8 | # Denote files that should not be modified. 9 | *.odt binary 10 | *.png binary 11 | 12 | # Visual Studio 13 | *.sln text eol=crlf 14 | *.vcxproj* text eol=crlf 15 | *.vcproj* text eol=crlf 16 | *.suo binary 17 | *.rc text eol=crlf 18 | 19 | # Windows 20 | *.bat text eol=crlf 21 | *.cmd text eol=crlf 22 | -------------------------------------------------------------------------------- /zstd/.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Downloads data '...' 16 | 2. Run '...' with flags '...' 17 | 3. Scroll up on the log to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots and charts** 24 | If applicable, add screenshots and charts to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. Mac] 28 | - Version [e.g. 22] 29 | - Compiler [e.g. gcc] 30 | - Flags [e.g. O2] 31 | - Other relevant hardware specs [e.g. Dual-core] 32 | - Build system [e.g. Makefile] 33 | 34 | **Additional context** 35 | Add any other context about the problem here. 36 | -------------------------------------------------------------------------------- /zstd/.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /zstd/.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /zstd/.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | *.ko 4 | *.dSYM 5 | 6 | # Libraries 7 | *.lib 8 | *.a 9 | 10 | # Shared objects (inc. Windows DLLs) 11 | *.dll 12 | *.so 13 | *.so.* 14 | *.dylib 15 | 16 | # Executables 17 | /zstd 18 | zstdmt 19 | *.exe 20 | *.out 21 | *.app 22 | 23 | # Test artefacts 24 | tmp* 25 | *.zst 26 | *.zstd 27 | dictionary. 28 | dictionary 29 | NUL 30 | cmakebuild/ 31 | install/ 32 | 33 | # Build artefacts 34 | contrib/linux-kernel/linux/ 35 | projects/ 36 | bin/ 37 | .buckd/ 38 | buck-out/ 39 | build-* 40 | *.gcda 41 | 42 | # IDE 43 | .clang_complete 44 | compile_flags.txt 45 | .clang-format 46 | 47 | # Other files 48 | .directory 49 | _codelite/ 50 | _zstdbench/ 51 | *.idea 52 | *.swp 53 | .DS_Store 54 | googletest/ 55 | *.d 56 | *.vscode 57 | *.code-workspace 58 | compile_commands.json 59 | .clangd 60 | perf.data 61 | perf.data.old 62 | -------------------------------------------------------------------------------- /zstd/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | Facebook has adopted a Code of Conduct that we expect project participants to adhere to. 4 | Please read the [full text](https://code.fb.com/codeofconduct/) 5 | so that you can understand what actions will and will not be tolerated. 6 | -------------------------------------------------------------------------------- /zstd/LICENSE: -------------------------------------------------------------------------------- 1 | BSD License 2 | 3 | For Zstandard software 4 | 5 | Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. 6 | 7 | Redistribution and use in source and binary forms, with or without modification, 8 | are permitted provided that the following conditions are met: 9 | 10 | * Redistributions of source code must retain the above copyright notice, this 11 | list of conditions and the following disclaimer. 12 | 13 | * Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | 17 | * Neither the name Facebook, nor Meta, nor the names of its contributors may 18 | be used to endorse or promote products derived from this software without 19 | specific prior written permission. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 25 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 26 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 28 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /zstd/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.0 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "zstd", 8 | platforms: [ 9 | .macOS(.v10_10), .iOS(.v9), .tvOS(.v9) 10 | ], 11 | products: [ 12 | // Products define the executables and libraries a package produces, and make them visible to other packages. 13 | .library( 14 | name: "libzstd", 15 | targets: [ "libzstd" ]) 16 | ], 17 | dependencies: [ 18 | // Dependencies declare other packages that this package depends on. 19 | // .package(url: /* package url */, from: "1.0.0"), 20 | ], 21 | targets: [ 22 | // Targets are the basic building blocks of a package. A target can define a module or a test suite. 23 | // Targets can depend on other targets in this package, and on products in packages this package depends on. 24 | .target( 25 | name: "libzstd", 26 | path: "lib", 27 | sources: [ "common", "compress", "decompress", "dictBuilder" ], 28 | publicHeadersPath: ".", 29 | cSettings: [ 30 | .headerSearchPath(".") 31 | ]) 32 | ], 33 | swiftLanguageVersions: [.v5], 34 | cLanguageStandard: .gnu11, 35 | cxxLanguageStandard: .gnucxx14 36 | ) 37 | -------------------------------------------------------------------------------- /zstd/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting and Fixing Security Issues 2 | 3 | Please do not open GitHub issues or pull requests - this makes the problem immediately visible to everyone, including malicious actors. Security issues in this open source project can be safely reported via the Meta Bug Bounty program: 4 | 5 | https://www.facebook.com/whitehat 6 | 7 | Meta's security team will triage your report and determine whether or not is it eligible for a bounty under our program. 8 | 9 | # Receiving Vulnerability Notifications 10 | 11 | In the case that a significant security vulnerability is reported to us or discovered by us---without being publicly known---we will, at our discretion, notify high-profile, high-exposure users of Zstandard ahead of our public disclosure of the issue and associated fix. 12 | 13 | If you believe your project would benefit from inclusion in this list, please reach out to one of the maintainers. 14 | 15 | 16 | -------------------------------------------------------------------------------- /zstd/build/.gitignore: -------------------------------------------------------------------------------- 1 | # Visual C++ 2 | .vs/ 3 | *Copy 4 | *.db 5 | *.opensdf 6 | *.sdf 7 | *.suo 8 | *.user 9 | *.opendb 10 | 11 | VS2008/bin/ 12 | VS2010/bin/ 13 | VS2010/zwrapbench/ 14 | VS2012/ 15 | VS2013/ 16 | VS2015/ 17 | Studio* 18 | 19 | # CMake 20 | cmake/build/ 21 | CMakeCache.txt 22 | CMakeFiles 23 | CMakeScripts 24 | Testing 25 | Makefile 26 | cmake_install.cmake 27 | install_manifest.txt 28 | compile_commands.json 29 | CTestTestfile.cmake 30 | build 31 | lib 32 | !cmake/lib 33 | !meson/lib 34 | -------------------------------------------------------------------------------- /zstd/build/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/build/LICENSE -------------------------------------------------------------------------------- /zstd/build/VS2010/CompileAsCpp.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CompileAsCpp 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /zstd/build/VS2010/libzstd-dll/libzstd-dll.rc: -------------------------------------------------------------------------------- 1 | // Microsoft Visual C++ generated resource script. 2 | // 3 | 4 | #include "zstd.h" /* ZSTD_VERSION_STRING */ 5 | #define APSTUDIO_READONLY_SYMBOLS 6 | #include "verrsrc.h" 7 | #undef APSTUDIO_READONLY_SYMBOLS 8 | 9 | 10 | #if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) 11 | LANGUAGE 9, 1 12 | 13 | ///////////////////////////////////////////////////////////////////////////// 14 | // 15 | // Version 16 | // 17 | 18 | VS_VERSION_INFO VERSIONINFO 19 | FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 20 | PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 21 | FILEFLAGSMASK VS_FFI_FILEFLAGSMASK 22 | #ifdef _DEBUG 23 | FILEFLAGS VS_FF_DEBUG 24 | #else 25 | FILEFLAGS 0x0L 26 | #endif 27 | FILEOS VOS_NT_WINDOWS32 28 | FILETYPE VFT_DLL 29 | FILESUBTYPE VFT2_UNKNOWN 30 | BEGIN 31 | BLOCK "StringFileInfo" 32 | BEGIN 33 | BLOCK "040904B0" 34 | BEGIN 35 | VALUE "CompanyName", "Meta Platforms, Inc." 36 | VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm" 37 | VALUE "FileVersion", ZSTD_VERSION_STRING 38 | VALUE "InternalName", "libzstd.dll" 39 | VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates." 40 | VALUE "OriginalFilename", "libzstd.dll" 41 | VALUE "ProductName", "Zstandard" 42 | VALUE "ProductVersion", ZSTD_VERSION_STRING 43 | END 44 | END 45 | BLOCK "VarFileInfo" 46 | BEGIN 47 | VALUE "Translation", 0x0409, 1200 48 | END 49 | END 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /zstd/build/VS2010/zstd/zstd.rc: -------------------------------------------------------------------------------- 1 | // Microsoft Visual C++ generated resource script. 2 | // 3 | 4 | #include "zstd.h" /* ZSTD_VERSION_STRING */ 5 | #define APSTUDIO_READONLY_SYMBOLS 6 | #include "verrsrc.h" 7 | #undef APSTUDIO_READONLY_SYMBOLS 8 | 9 | 10 | #if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) 11 | LANGUAGE 9, 1 12 | 13 | ///////////////////////////////////////////////////////////////////////////// 14 | // 15 | // Version 16 | // 17 | 18 | VS_VERSION_INFO VERSIONINFO 19 | FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 20 | PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 21 | FILEFLAGSMASK VS_FFI_FILEFLAGSMASK 22 | #ifdef _DEBUG 23 | FILEFLAGS VS_FF_DEBUG 24 | #else 25 | FILEFLAGS 0x0L 26 | #endif 27 | FILEOS VOS_NT_WINDOWS32 28 | FILETYPE VFT_DLL 29 | FILESUBTYPE VFT2_UNKNOWN 30 | BEGIN 31 | BLOCK "StringFileInfo" 32 | BEGIN 33 | BLOCK "040904B0" 34 | BEGIN 35 | VALUE "CompanyName", "Meta Platforms, Inc." 36 | VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm" 37 | VALUE "FileVersion", ZSTD_VERSION_STRING 38 | VALUE "InternalName", "zstd.exe" 39 | VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates." 40 | VALUE "OriginalFilename", "zstd.exe" 41 | VALUE "ProductName", "Zstandard" 42 | VALUE "ProductVersion", ZSTD_VERSION_STRING 43 | END 44 | END 45 | BLOCK "VarFileInfo" 46 | BEGIN 47 | VALUE "Translation", 0x0409, 1200 48 | END 49 | END 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /zstd/build/VS_scripts/build.VS2010.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2010 Win32 Release v100 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2010 x64 Release v100 -------------------------------------------------------------------------------- /zstd/build/VS_scripts/build.VS2012.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2012 Win32 Release v110 5 | rem build 64-bit 6 | call "%~p0%build.generic.cmd" VS2012 x64 Release v110 -------------------------------------------------------------------------------- /zstd/build/VS_scripts/build.VS2013.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2013 Win32 Release v120 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2013 x64 Release v120 -------------------------------------------------------------------------------- /zstd/build/VS_scripts/build.VS2015.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2015 Win32 Release v140 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2015 x64 Release v140 -------------------------------------------------------------------------------- /zstd/build/VS_scripts/build.VS2017.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2017 Win32 Release v141 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2017 x64 Release v141 -------------------------------------------------------------------------------- /zstd/build/VS_scripts/build.VS2017Community.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2017Community Win32 Release v141 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2017Community x64 Release v141 -------------------------------------------------------------------------------- /zstd/build/VS_scripts/build.VS2017Enterprise.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2017Enterprise Win32 Release v141 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2017Enterprise x64 Release v141 -------------------------------------------------------------------------------- /zstd/build/VS_scripts/build.VS2017Professional.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem build 32-bit 4 | call "%~p0%build.generic.cmd" VS2017Professional Win32 Release v141 5 | 6 | rem build 64-bit 7 | call "%~p0%build.generic.cmd" VS2017Professional x64 Release v141 -------------------------------------------------------------------------------- /zstd/build/cmake/.gitignore: -------------------------------------------------------------------------------- 1 | # cmake working directory 2 | cmakeBuild 3 | 4 | # cmake artefacts 5 | CMakeCache.txt 6 | CMakeFiles 7 | Makefile 8 | cmake_install.cmake 9 | cmake_uninstall.cmake 10 | *.1 11 | -------------------------------------------------------------------------------- /zstd/build/cmake/CMakeModules/GetZstdLibraryVersion.cmake: -------------------------------------------------------------------------------- 1 | function(GetZstdLibraryVersion _header _major _minor _patch) 2 | # Read file content 3 | file(READ ${_header} CONTENT) 4 | 5 | string(REGEX MATCH ".*define ZSTD_VERSION_MAJOR *([0-9]+).*define ZSTD_VERSION_MINOR *([0-9]+).*define ZSTD_VERSION_RELEASE *([0-9]+)" VERSION_REGEX "${CONTENT}") 6 | set(${_major} ${CMAKE_MATCH_1} PARENT_SCOPE) 7 | set(${_minor} ${CMAKE_MATCH_2} PARENT_SCOPE) 8 | set(${_patch} ${CMAKE_MATCH_3} PARENT_SCOPE) 9 | endfunction() 10 | 11 | -------------------------------------------------------------------------------- /zstd/build/cmake/CMakeModules/JoinPaths.cmake: -------------------------------------------------------------------------------- 1 | # This module provides function for joining paths 2 | # known from most languages 3 | # 4 | # SPDX-License-Identifier: (MIT OR CC0-1.0) 5 | # Copyright 2020 Jan Tojnar 6 | # https://github.com/jtojnar/cmake-snips 7 | # 8 | # Modelled after Python’s os.path.join 9 | # https://docs.python.org/3.7/library/os.path.html#os.path.join 10 | # Windows not supported 11 | function(join_paths joined_path first_path_segment) 12 | set(temp_path "${first_path_segment}") 13 | foreach(current_segment IN LISTS ARGN) 14 | if(NOT ("${current_segment}" STREQUAL "")) 15 | if(IS_ABSOLUTE "${current_segment}") 16 | set(temp_path "${current_segment}") 17 | else() 18 | set(temp_path "${temp_path}/${current_segment}") 19 | endif() 20 | endif() 21 | endforeach() 22 | set(${joined_path} "${temp_path}" PARENT_SCOPE) 23 | endfunction() 24 | -------------------------------------------------------------------------------- /zstd/build/cmake/contrib/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | project(contrib) 11 | 12 | add_subdirectory(pzstd) 13 | add_subdirectory(gen_html) 14 | -------------------------------------------------------------------------------- /zstd/build/cmake/contrib/gen_html/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | project(gen_html) 11 | include(GetZstdLibraryVersion) 12 | 13 | set(CMAKE_INCLUDE_CURRENT_DIR TRUE) 14 | 15 | # Define programs directory, where sources and header files are located 16 | set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib) 17 | set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs) 18 | set(GENHTML_DIR ${ZSTD_SOURCE_DIR}/contrib/gen_html) 19 | set(GENHTML_BINARY ${PROJECT_BINARY_DIR}/gen_html${CMAKE_EXECUTABLE_SUFFIX}) 20 | include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${GENHTML_DIR}) 21 | 22 | add_executable(gen_html ${GENHTML_DIR}/gen_html.cpp) 23 | 24 | GetZstdLibraryVersion(${LIBRARY_DIR}/zstd.h VMAJOR VMINOR VRELEASE) 25 | set(LIBVERSION "${VMAJOR}.${VMINOR}.${VRELEASE}") 26 | add_custom_target(zstd_manual.html ALL 27 | ${GENHTML_BINARY} "${LIBVERSION}" "${LIBRARY_DIR}/zstd.h" "${PROJECT_BINARY_DIR}/zstd_manual.html" 28 | DEPENDS gen_html COMMENT "Update zstd manual") 29 | 30 | install(FILES "${PROJECT_BINARY_DIR}/zstd_manual.html" DESTINATION "${CMAKE_INSTALL_DOCDIR}") 31 | -------------------------------------------------------------------------------- /zstd/build/cmake/contrib/pzstd/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | project(pzstd) 11 | 12 | set(CMAKE_INCLUDE_CURRENT_DIR TRUE) 13 | 14 | # Define programs directory, where sources and header files are located 15 | set(LIBRARY_DIR ${ZSTD_SOURCE_DIR}/lib) 16 | set(PROGRAMS_DIR ${ZSTD_SOURCE_DIR}/programs) 17 | set(PZSTD_DIR ${ZSTD_SOURCE_DIR}/contrib/pzstd) 18 | include_directories(${PROGRAMS_DIR} ${LIBRARY_DIR} ${LIBRARY_DIR}/common ${PZSTD_DIR}) 19 | 20 | add_executable(pzstd ${PROGRAMS_DIR}/util.c ${PZSTD_DIR}/main.cpp ${PZSTD_DIR}/Options.cpp ${PZSTD_DIR}/Pzstd.cpp ${PZSTD_DIR}/SkippableFrame.cpp) 21 | target_compile_features(pzstd PRIVATE cxx_std_11) 22 | set_property(TARGET pzstd APPEND PROPERTY COMPILE_DEFINITIONS "NDEBUG") 23 | set_property(TARGET pzstd APPEND PROPERTY COMPILE_OPTIONS "-Wno-shadow") 24 | 25 | if (ZSTD_BUILD_SHARED) 26 | set(ZSTD_LIB libzstd_shared) 27 | else() 28 | set(ZSTD_LIB libzstd_static) 29 | endif() 30 | 31 | set(THREADS_PREFER_PTHREAD_FLAG ON) 32 | find_package(Threads REQUIRED) 33 | if (CMAKE_USE_PTHREADS_INIT) 34 | target_link_libraries(pzstd ${ZSTD_LIB} ${CMAKE_THREAD_LIBS_INIT}) 35 | else() 36 | message(SEND_ERROR "ZSTD currently does not support thread libraries other than pthreads") 37 | endif() 38 | 39 | install(TARGETS pzstd RUNTIME DESTINATION "bin") 40 | -------------------------------------------------------------------------------- /zstd/build/cmake/lib/.gitignore: -------------------------------------------------------------------------------- 1 | # cmake build artefact 2 | libzstd.pc 3 | -------------------------------------------------------------------------------- /zstd/build/cmake/lib/cmake_uninstall.cmake.in: -------------------------------------------------------------------------------- 1 | 2 | if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt") 3 | message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt") 4 | endif() 5 | 6 | file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files) 7 | string(REGEX REPLACE "\n" ";" files "${files}") 8 | foreach(file ${files}) 9 | message(STATUS "Uninstalling $ENV{DESTDIR}${file}") 10 | if(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") 11 | exec_program( 12 | "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" 13 | OUTPUT_VARIABLE rm_out 14 | RETURN_VALUE rm_retval 15 | ) 16 | if(NOT "${rm_retval}" STREQUAL 0) 17 | message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}") 18 | endif() 19 | else() 20 | message(STATUS "File $ENV{DESTDIR}${file} does not exist.") 21 | endif() 22 | endforeach() 23 | -------------------------------------------------------------------------------- /zstd/build/cmake/programs/.gitignore: -------------------------------------------------------------------------------- 1 | # produced by make 2 | zstd 3 | zstd-frugal 4 | unzstd 5 | zstdcat 6 | -------------------------------------------------------------------------------- /zstd/build/cmake/tests/.gitignore: -------------------------------------------------------------------------------- 1 | # produced by make 2 | datagen 3 | fullbench 4 | fuzzer 5 | paramgrill 6 | 7 | -------------------------------------------------------------------------------- /zstd/build/cmake/zstdConfig.cmake.in: -------------------------------------------------------------------------------- 1 | @PACKAGE_INIT@ 2 | 3 | include(CMakeFindDependencyMacro) 4 | if(@ZSTD_MULTITHREAD_SUPPORT@ AND "@UNIX@") 5 | find_dependency(Threads) 6 | endif() 7 | 8 | include("${CMAKE_CURRENT_LIST_DIR}/zstdTargets.cmake") 9 | 10 | check_required_components("zstd") 11 | -------------------------------------------------------------------------------- /zstd/build/meson/GetZstdLibraryVersion.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # ############################################################################# 3 | # Copyright (c) 2018-present lzutao 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | import re 11 | 12 | 13 | def find_version_tuple(filepath): 14 | version_file_data = None 15 | with open(filepath) as fd: 16 | version_file_data = fd.read() 17 | 18 | patterns = r"""#\s*define\s+ZSTD_VERSION_MAJOR\s+([0-9]+) 19 | #\s*define\s+ZSTD_VERSION_MINOR\s+([0-9]+) 20 | #\s*define\s+ZSTD_VERSION_RELEASE\s+([0-9]+) 21 | """ 22 | regex = re.compile(patterns, re.MULTILINE) 23 | version_match = regex.search(version_file_data) 24 | if version_match: 25 | return version_match.groups() 26 | raise Exception("Unable to find version string") 27 | 28 | 29 | def main(): 30 | import argparse 31 | parser = argparse.ArgumentParser(description='Print zstd version from lib/zstd.h') 32 | parser.add_argument('file', help='path to lib/zstd.h') 33 | args = parser.parse_args() 34 | version_tuple = find_version_tuple(args.file) 35 | print('.'.join(version_tuple)) 36 | 37 | 38 | if __name__ == '__main__': 39 | main() 40 | -------------------------------------------------------------------------------- /zstd/build/meson/README.md: -------------------------------------------------------------------------------- 1 | Meson build system for zstandard 2 | ================================ 3 | 4 | Meson is a build system designed to optimize programmer productivity. 5 | It aims to do this by providing simple, out-of-the-box support for 6 | modern software development tools and practices, such as unit tests, 7 | coverage reports, Valgrind, CCache and the like. 8 | 9 | This Meson build system is provided with no guarantee and maintained 10 | by Dima Krasner \. 11 | 12 | It outputs one `libzstd`, either shared or static, depending on 13 | `default_library` option. 14 | 15 | ## How to build 16 | 17 | `cd` to this meson directory (`build/meson`) 18 | 19 | ```sh 20 | meson setup -Dbin_programs=true -Dbin_contrib=true builddir 21 | cd builddir 22 | ninja # to build 23 | ninja install # to install 24 | ``` 25 | 26 | You might want to install it in staging directory: 27 | 28 | ```sh 29 | DESTDIR=./staging ninja install 30 | ``` 31 | 32 | To configure build options, use: 33 | 34 | ```sh 35 | meson configure 36 | ``` 37 | 38 | See [man meson(1)](https://manpages.debian.org/testing/meson/meson.1.en.html). 39 | -------------------------------------------------------------------------------- /zstd/build/meson/contrib/gen_html/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ############################################################################# 9 | 10 | zstd_rootdir = '../../../..' 11 | 12 | gen_html_includes = include_directories(join_paths(zstd_rootdir, 'programs'), 13 | join_paths(zstd_rootdir, 'lib'), 14 | join_paths(zstd_rootdir, 'lib/common'), 15 | join_paths(zstd_rootdir, 'contrib/gen_html')) 16 | 17 | gen_html = executable('gen_html', 18 | join_paths(zstd_rootdir, 'contrib/gen_html/gen_html.cpp'), 19 | include_directories: gen_html_includes, 20 | native: true, 21 | install: false) 22 | 23 | # Update zstd manual 24 | zstd_manual_html = custom_target('zstd_manual.html', 25 | output : 'zstd_manual.html', 26 | command : [gen_html, 27 | zstd_version, 28 | join_paths(meson.current_source_dir(), zstd_rootdir, 'lib/zstd.h'), 29 | '@OUTPUT@'], 30 | install : false) 31 | -------------------------------------------------------------------------------- /zstd/build/meson/contrib/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present Dima Krasner 3 | # lzutao 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # ############################################################################# 10 | 11 | subdir('pzstd') 12 | subdir('gen_html') 13 | -------------------------------------------------------------------------------- /zstd/build/meson/contrib/pzstd/meson.build: -------------------------------------------------------------------------------- 1 | # ############################################################################# 2 | # Copyright (c) 2018-present lzutao 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ############################################################################# 9 | 10 | zstd_rootdir = '../../../..' 11 | 12 | pzstd_includes = include_directories(join_paths(zstd_rootdir, 'programs'), 13 | join_paths(zstd_rootdir, 'contrib/pzstd')) 14 | pzstd_sources = [join_paths(zstd_rootdir, 'programs/util.c'), 15 | join_paths(zstd_rootdir, 'contrib/pzstd/main.cpp'), 16 | join_paths(zstd_rootdir, 'contrib/pzstd/Options.cpp'), 17 | join_paths(zstd_rootdir, 'contrib/pzstd/Pzstd.cpp'), 18 | join_paths(zstd_rootdir, 'contrib/pzstd/SkippableFrame.cpp')] 19 | pzstd = executable('pzstd', 20 | pzstd_sources, 21 | cpp_args: pzstd_warning_flags, 22 | include_directories: pzstd_includes, 23 | dependencies: [ libzstd_dep, thread_dep ], 24 | override_options: ['b_ndebug=true'], 25 | install: true) 26 | -------------------------------------------------------------------------------- /zstd/build/single_file_libs/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # build artifacts 3 | zstddeclib.c 4 | zstdenclib.c 5 | zstd.c 6 | zstd.h 7 | 8 | # test artifacts 9 | temp* 10 | -------------------------------------------------------------------------------- /zstd/build/single_file_libs/create_single_file_decoder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Where to find the sources 4 | ZSTD_SRC_ROOT="../../lib" 5 | 6 | # Amalgamate the sources 7 | echo "Amalgamating files..." 8 | # Using the faster Python script if we have 3.8 or higher 9 | if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then 10 | ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c 11 | else 12 | ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c 13 | fi 14 | # Did combining work? 15 | if [ $? -ne 0 ]; then 16 | echo "Combine script: FAILED" 17 | exit 1 18 | fi 19 | echo "Combine script: PASSED" 20 | -------------------------------------------------------------------------------- /zstd/build/single_file_libs/create_single_file_library.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Where to find the sources 4 | ZSTD_SRC_ROOT="../../lib" 5 | 6 | # Amalgamate the sources 7 | echo "Amalgamating files..." 8 | # Using the faster Python script if we have 3.8 or higher 9 | if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then 10 | ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c 11 | else 12 | ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c 13 | fi 14 | # Did combining work? 15 | if [ $? -ne 0 ]; then 16 | echo "Combine script: FAILED" 17 | exit 1 18 | fi 19 | echo "Combine script: PASSED" 20 | -------------------------------------------------------------------------------- /zstd/build/single_file_libs/examples/README.md: -------------------------------------------------------------------------------- 1 | # Single File ZStandard Examples 2 | 3 | The examples `#include` the generated `zstddeclib.c` directly but work equally as well when including `zstd.h` and compiling the amalgamated source separately. 4 | 5 | `simple.c` is the most basic example of decompressing content and verifying the result. 6 | 7 | `emscripten.c` is a bare-bones [Emscripten](https://github.com/emscripten-core/emscripten) compiled WebGL demo using Zstd to further compress a DXT1 texture (see the [original PNG image](testcard.png)). The 256x256 texture would normally be 32kB, but even when bundled with the Zstd decompressor the resulting WebAssembly weighs in at 41kB (`shell.html` is a support file to run the Wasm). 8 | 9 | `roundtrip.c` is an example to use with the optional [amalgamated library](../create_single_file_library.sh) showing compression the decompression. 10 | 11 | The example files in this directory are released under a [Creative Commons Zero license](https://creativecommons.org/publicdomain/zero/1.0/) (or Public Domain, whichever is applicable in your jurisdiction). 12 | -------------------------------------------------------------------------------- /zstd/build/single_file_libs/examples/shell.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Emscripten Shell 9 | 13 | 14 | 15 | 16 | 29 | {{{ SCRIPT }}} 30 | 31 | 32 | -------------------------------------------------------------------------------- /zstd/build/single_file_libs/examples/testcard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/build/single_file_libs/examples/testcard.png -------------------------------------------------------------------------------- /zstd/contrib/VS2005/README.md: -------------------------------------------------------------------------------- 1 | ## Project Support Notice 2 | 3 | The VS2005 Project directory has been moved to the contrib directory in order to indicate that it will no longer be supported. 4 | -------------------------------------------------------------------------------- /zstd/contrib/cleanTabs: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | sed -i '' $'s/\t/ /g' ../lib/**/*.{h,c} ../programs/*.{h,c} ../tests/*.c ./**/*.{h,cpp} ../examples/*.c ../zlibWrapper/*.{h,c} 3 | -------------------------------------------------------------------------------- /zstd/contrib/diagnose_corruption/.gitignore: -------------------------------------------------------------------------------- 1 | check_flipped_bits 2 | -------------------------------------------------------------------------------- /zstd/contrib/diagnose_corruption/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | .PHONY: all 11 | all: check_flipped_bits 12 | 13 | ZSTDLIBDIR ?= ../../lib 14 | 15 | CFLAGS ?= -O3 16 | CFLAGS += -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZSTDLIBDIR)/compress \ 17 | -I$(ZSTDLIBDIR)/decompress 18 | CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ 19 | -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ 20 | -Wstrict-prototypes -Wundef \ 21 | -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ 22 | -Wredundant-decls -Wmissing-prototypes 23 | CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) 24 | FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) 25 | 26 | .PHONY: $(ZSTDLIBDIR)/libzstd.a 27 | $(ZSTDLIBDIR)/libzstd.a: 28 | $(MAKE) -C $(ZSTDLIBDIR) libzstd.a 29 | 30 | check_flipped_bits: check_flipped_bits.c $(ZSTDLIBDIR)/libzstd.a 31 | $(CC) $(FLAGS) $< -o $@$(EXT) $(ZSTDLIBDIR)/libzstd.a 32 | 33 | .PHONY: clean 34 | clean: 35 | rm -f check_flipped_bits 36 | -------------------------------------------------------------------------------- /zstd/contrib/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Dockerfile 2 | # First image to build the binary 3 | FROM alpine@sha256:69665d02cb32192e52e07644d76bc6f25abeb5410edc1c7a81a10ba3f0efb90a as builder 4 | 5 | RUN apk --no-cache add make gcc libc-dev 6 | COPY . /src 7 | RUN mkdir /pkg && cd /src && make && make DESTDIR=/pkg install 8 | 9 | # Second minimal image to only keep the built binary 10 | FROM alpine@sha256:69665d02cb32192e52e07644d76bc6f25abeb5410edc1c7a81a10ba3f0efb90a 11 | 12 | # Copy the built files 13 | COPY --from=builder /pkg / 14 | 15 | # Copy the license as well 16 | RUN mkdir -p /usr/local/share/licenses/zstd 17 | COPY --from=builder /src/LICENSE /usr/local/share/licences/zstd/ 18 | 19 | # Just run `zstd` if no other command is given 20 | CMD ["/usr/local/bin/zstd"] 21 | -------------------------------------------------------------------------------- /zstd/contrib/docker/README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Requirement 3 | 4 | The `Dockerfile` script requires a version of `docker` >= 17.05 5 | 6 | ## Installing docker 7 | 8 | The official docker install docs use a ppa with a modern version available: 9 | https://docs.docker.com/install/linux/docker-ce/ubuntu/ 10 | 11 | ## How to run 12 | 13 | `docker build -t zstd .` 14 | 15 | ## test 16 | 17 | ``` 18 | echo foo | docker run -i --rm zstd | docker run -i --rm zstd zstdcat 19 | foo 20 | ``` 21 | -------------------------------------------------------------------------------- /zstd/contrib/externalSequenceProducer/.gitignore: -------------------------------------------------------------------------------- 1 | # build artifacts 2 | externalSequenceProducer 3 | -------------------------------------------------------------------------------- /zstd/contrib/externalSequenceProducer/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Yann Collet, Meta Platforms, Inc. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | PROGDIR = ../../programs 11 | LIBDIR = ../../lib 12 | 13 | LIBZSTD = $(LIBDIR)/libzstd.a 14 | 15 | CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/compress -I$(LIBDIR)/common 16 | 17 | CFLAGS ?= -O3 18 | CFLAGS += -std=gnu99 19 | DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ 20 | -Wstrict-aliasing=1 -Wswitch-enum \ 21 | -Wstrict-prototypes -Wundef -Wpointer-arith \ 22 | -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ 23 | -Wredundant-decls 24 | CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) 25 | 26 | default: externalSequenceProducer 27 | 28 | all: externalSequenceProducer 29 | 30 | externalSequenceProducer: sequence_producer.c main.c $(LIBZSTD) 31 | $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ 32 | 33 | .PHONY: $(LIBZSTD) 34 | $(LIBZSTD): 35 | $(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)" 36 | 37 | clean: 38 | $(RM) *.o 39 | $(MAKE) -C $(LIBDIR) clean > /dev/null 40 | $(RM) externalSequenceProducer 41 | -------------------------------------------------------------------------------- /zstd/contrib/externalSequenceProducer/README.md: -------------------------------------------------------------------------------- 1 | externalSequenceProducer 2 | ===================== 3 | 4 | `externalSequenceProducer` is a test tool for the Block-Level Sequence Producer API. 5 | It demonstrates how to use the API to perform a simple round-trip test. 6 | 7 | A sample sequence producer is provided in sequence_producer.c, but the user can swap 8 | this out with a different one if desired. The sample sequence producer implements 9 | LZ parsing with a 1KB hashtable. Dictionary-based parsing is not currently supported. 10 | 11 | Command line : 12 | ``` 13 | externalSequenceProducer filename 14 | ``` 15 | -------------------------------------------------------------------------------- /zstd/contrib/externalSequenceProducer/sequence_producer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Yann Collet, Meta Platforms, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef MATCHFINDER_H 12 | #define MATCHFINDER_H 13 | 14 | #define ZSTD_STATIC_LINKING_ONLY 15 | #include "zstd.h" 16 | 17 | size_t simpleSequenceProducer( 18 | void* sequenceProducerState, 19 | ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, 20 | const void* src, size_t srcSize, 21 | const void* dict, size_t dictSize, 22 | int compressionLevel, 23 | size_t windowSize 24 | ); 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /zstd/contrib/gen_html/.gitignore: -------------------------------------------------------------------------------- 1 | # make artefact 2 | gen_html 3 | zstd_manual.html 4 | -------------------------------------------------------------------------------- /zstd/contrib/gen_html/README.md: -------------------------------------------------------------------------------- 1 | gen_html - a program for automatic generation of zstd manual 2 | ============================================================ 3 | 4 | #### Introduction 5 | 6 | This simple C++ program generates a single-page HTML manual from `zstd.h`. 7 | 8 | The format of recognized comment blocks is following: 9 | - comments of type `/*!` mean: this is a function declaration; switch comments with declarations 10 | - comments of type `/**` and `/*-` mean: this is a comment; use a `

` header for the first line 11 | - comments of type `/*=` and `/**=` mean: use a `

` header and show also all functions until first empty line 12 | - comments of type `/*X` where `X` is different from above-mentioned are ignored 13 | 14 | Moreover: 15 | - `ZSTDLIB_API` is removed to improve readability 16 | - `typedef` are detected and included even if uncommented 17 | - comments of type `/**<` and `/*!<` are detected and only function declaration is highlighted (bold) 18 | 19 | 20 | #### Usage 21 | 22 | The program requires 3 parameters: 23 | ``` 24 | gen_html [zstd_version] [input_file] [output_html] 25 | ``` 26 | 27 | To compile program and generate zstd manual we have used: 28 | ``` 29 | make 30 | ./gen_html.exe 1.1.1 ../../lib/zstd.h zstd_manual.html 31 | ``` 32 | -------------------------------------------------------------------------------- /zstd/contrib/gen_html/gen-zstd-manual.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | LIBVER_MAJOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h` 4 | LIBVER_MINOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h` 5 | LIBVER_PATCH_SCRIPT=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h` 6 | LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT 7 | 8 | echo ZSTD_VERSION=$LIBVER_SCRIPT 9 | ./gen_html $LIBVER_SCRIPT ../../lib/zstd.h ./zstd_manual.html 10 | -------------------------------------------------------------------------------- /zstd/contrib/largeNbDicts/.gitignore: -------------------------------------------------------------------------------- 1 | # build artifacts 2 | largeNbDicts 3 | -------------------------------------------------------------------------------- /zstd/contrib/largeNbDicts/README.md: -------------------------------------------------------------------------------- 1 | largeNbDicts 2 | ===================== 3 | 4 | `largeNbDicts` is a benchmark test tool 5 | dedicated to the specific scenario of 6 | dictionary decompression using a very large number of dictionaries. 7 | When dictionaries are constantly changing, they are always "cold", 8 | suffering from increased latency due to cache misses. 9 | 10 | The tool is created in a bid to investigate performance for this scenario, 11 | and experiment mitigation techniques. 12 | 13 | Command line : 14 | ``` 15 | largeNbDicts [Options] filename(s) 16 | 17 | Options : 18 | -z : benchmark compression (default) 19 | -d : benchmark decompression 20 | -r : recursively load all files in subdirectories (default: off) 21 | -B# : split input into blocks of size # (default: no split) 22 | -# : use compression level # (default: 3) 23 | -D # : use # as a dictionary (default: create one) 24 | -i# : nb benchmark rounds (default: 6) 25 | --nbBlocks=#: use # blocks for bench (default: one per file) 26 | --nbDicts=# : create # dictionaries for bench (default: one per block) 27 | -h : help (this text) 28 | 29 | Advanced Options (see zstd.h for documentation) : 30 | --dedicated-dict-search 31 | --dict-content-type=# 32 | --dict-attach-pref=# 33 | ``` 34 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/.gitignore: -------------------------------------------------------------------------------- 1 | !lib/zstd 2 | !lib/zstd/* 3 | *.o 4 | *.a 5 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/README.md: -------------------------------------------------------------------------------- 1 | # Zstd in the Linux Kernel 2 | 3 | This directory contains the scripts needed to transform upstream zstd into the version imported into the kernel. All the transforms are automated and tested by our continuous integration. 4 | 5 | ## Upgrading Zstd in the Linux Kernel 6 | 7 | 1. `cd` into this directory. 8 | 2. Run `make libzstd` and read the output. Make sure that all the diffs printed and changes made by the script are correct. 9 | 3. Run `make test` and ensure that it passes. 10 | 4. Import zstd into the Linux Kernel `make import LINUX=/path/to/linux/repo` 11 | 5. Inspect the diff for sanity. 12 | 6. Check the Linux Kernel history for zstd. If any patches were made to the kernel version of zstd, but not to upstream zstd, then port them upstream if necessary. 13 | 7. Test the diff. Benchmark if necessary. Make sure to test multiple architectures: At least x86, i386, and arm. 14 | 8. Submit the patch to the LKML. 15 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/decompress_sources.h: -------------------------------------------------------------------------------- 1 | /* SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause */ 2 | /* 3 | * Copyright (c) Meta Platforms, Inc. and affiliates. 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | /* 13 | * This file includes every .c file needed for decompression. 14 | * It is used by lib/decompress_unzstd.c to include the decompression 15 | * source into the translation-unit, so it can be used for kernel 16 | * decompression. 17 | */ 18 | 19 | /* 20 | * Disable the ASM Huffman implementation because we need to 21 | * include all the sources. 22 | */ 23 | #define ZSTD_DISABLE_ASM 1 24 | 25 | #include "common/debug.c" 26 | #include "common/entropy_common.c" 27 | #include "common/error_private.c" 28 | #include "common/fse_decompress.c" 29 | #include "common/zstd_common.c" 30 | #include "decompress/huf_decompress.c" 31 | #include "decompress/zstd_ddict.c" 32 | #include "decompress/zstd_decompress.c" 33 | #include "decompress/zstd_decompress_block.c" 34 | #include "zstd_decompress_module.c" 35 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/linux.mk: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause 2 | # ################################################################ 3 | # Copyright (c) Meta Platforms, Inc. and affiliates. 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # You may select, at your option, one of the above-listed licenses. 10 | # ################################################################ 11 | obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o 12 | obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o 13 | obj-$(CONFIG_ZSTD_COMMON) += zstd_common.o 14 | 15 | zstd_compress-y := \ 16 | zstd_compress_module.o \ 17 | compress/fse_compress.o \ 18 | compress/hist.o \ 19 | compress/huf_compress.o \ 20 | compress/zstd_compress.o \ 21 | compress/zstd_compress_literals.o \ 22 | compress/zstd_compress_sequences.o \ 23 | compress/zstd_compress_superblock.o \ 24 | compress/zstd_double_fast.o \ 25 | compress/zstd_fast.o \ 26 | compress/zstd_lazy.o \ 27 | compress/zstd_ldm.o \ 28 | compress/zstd_opt.o \ 29 | 30 | zstd_decompress-y := \ 31 | zstd_decompress_module.o \ 32 | decompress/huf_decompress.o \ 33 | decompress/zstd_ddict.o \ 34 | decompress/zstd_decompress.o \ 35 | decompress/zstd_decompress_block.o \ 36 | 37 | zstd_common-y := \ 38 | zstd_common_module.o \ 39 | common/debug.o \ 40 | common/entropy_common.o \ 41 | common/error_private.o \ 42 | common/fse_decompress.o \ 43 | common/zstd_common.o \ 44 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/squashfs-benchmark.sh: -------------------------------------------------------------------------------- 1 | # !/bin/sh 2 | set -e 3 | 4 | # Benchmarks run on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM. 5 | # The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor and 6 | # 16 GB of RAM and an SSD. 7 | 8 | # $BENCHMARK_DIR is generated with the following commands, from the Ubuntu image 9 | # ubuntu-16.10-desktop-amd64.iso. 10 | # > mkdir mnt 11 | # > sudo mount -o loop ubuntu-16.10-desktop-amd64.iso mnt 12 | # > cp mnt/casper/filesystem.squashfs . 13 | # > sudo unsquashfs filesystem.squashfs 14 | 15 | # $HOME is on a ext4 filesystem 16 | BENCHMARK_DIR="$HOME/squashfs-root/" 17 | BENCHMARK_FS="$HOME/filesystem.squashfs" 18 | 19 | # Normalize the environment 20 | sudo rm -f $BENCHMARK_FS 2> /dev/null > /dev/null || true 21 | sudo umount /mnt/squashfs 2> /dev/null > /dev/null || true 22 | 23 | # Run the benchmark 24 | echo "Compression" 25 | echo "sudo mksquashfs $BENCHMARK_DIR $BENCHMARK_FS $@" 26 | time sudo mksquashfs $BENCHMARK_DIR $BENCHMARK_FS $@ 2> /dev/null > /dev/null 27 | 28 | echo "Approximate compression ratio" 29 | printf "%d / %d\n" \ 30 | $(sudo du -sx --block-size=1 $BENCHMARK_DIR | cut -f1) \ 31 | $(sudo du -sx --block-size=1 $BENCHMARK_FS | cut -f1); 32 | 33 | # Mount the filesystem 34 | sudo mount -t squashfs $BENCHMARK_FS /mnt/squashfs 35 | 36 | echo "Decompression" 37 | time sudo tar -c /mnt/squashfs 2> /dev/null | wc -c > /dev/null 38 | 39 | sudo umount /mnt/squashfs 40 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/test/include/linux/compiler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_COMPILER_H 11 | #define LINUX_COMPILER_H 12 | 13 | #ifndef inline 14 | #define inline __inline __attribute__((unused)) 15 | #endif 16 | 17 | #ifndef noinline 18 | #define noinline __attribute__((noinline)) 19 | #endif 20 | 21 | #define fallthrough __attribute__((__fallthrough__)) 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/test/include/linux/errno.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_ERRNO_H 11 | #define LINUX_ERRNO_H 12 | 13 | #define EINVAL 22 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/test/include/linux/kernel.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_KERNEL_H 11 | #define LINUX_KERNEL_H 12 | 13 | #define WARN_ON(x) 14 | 15 | #define PTR_ALIGN(p, a) (typeof(p))ALIGN((unsigned long long)(p), (a)) 16 | #define ALIGN(x, a) ALIGN_MASK((x), (a) - 1) 17 | #define ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/test/include/linux/limits.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_LIMITS_H 11 | #define LINUX_LIMITS_H 12 | 13 | #include 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/test/include/linux/math64.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_MATH64_H 11 | #define LINUX_MATH64_H 12 | 13 | #define div_u64(dividend, divisor) ((dividend) / (divisor)) 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/test/include/linux/module.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_MODULE_H 11 | #define LINUX_MODULE_H 12 | 13 | #define EXPORT_SYMBOL(symbol) \ 14 | void* __##symbol = symbol 15 | #define EXPORT_SYMBOL_GPL(symbol) \ 16 | void* __##symbol = symbol 17 | #define MODULE_LICENSE(license) 18 | #define MODULE_DESCRIPTION(description) 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/test/include/linux/printk.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_PRINTK_H 11 | #define LINUX_PRINTK_H 12 | 13 | #define pr_debug(...) 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/test/include/linux/stddef.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_STDDEF_H 11 | #define LINUX_STDDEF_H 12 | 13 | #include 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/test/include/linux/swab.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_SWAB_H 11 | #define LINUX_SWAB_H 12 | 13 | #define swab32(x) __builtin_bswap32((x)) 14 | #define swab64(x) __builtin_bswap64((x)) 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/test/include/linux/types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #ifndef LINUX_TYPES_H 11 | #define LINUX_TYPES_H 12 | 13 | #include 14 | #include 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/test/macro-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -e 4 | 5 | SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) 6 | INCLUDE_DIR="$SCRIPT_DIR/../linux/include" 7 | LIB_DIR="$SCRIPT_DIR/../linux/lib" 8 | 9 | 10 | print() { 11 | printf '%b' "${*}" 12 | } 13 | 14 | println() { 15 | printf '%b\n' "${*}" 16 | } 17 | 18 | die() { 19 | println "$@" 1>&2 20 | exit 1 21 | } 22 | 23 | test_not_present() { 24 | print "Testing that '$1' is not present... " 25 | grep -r $1 "$INCLUDE_DIR" "$LIB_DIR" && die "Fail!" 26 | println "Okay" 27 | } 28 | 29 | println "This test checks that the macro removal process worked as expected" 30 | println "If this test fails, then freestanding.py wasn't able to remove one of these" 31 | println "macros from the source code completely. You'll either need to rewrite the check" 32 | println "or improve freestanding.py." 33 | println "" 34 | 35 | test_not_present "ZSTD_NO_INTRINSICS" 36 | test_not_present "ZSTD_NO_UNUSED_FUNCTIONS" 37 | test_not_present "ZSTD_LEGACY_SUPPORT" 38 | test_not_present "STATIC_BMI2" 39 | test_not_present "ZSTD_DLL_EXPORT" 40 | test_not_present "ZSTD_DLL_IMPORT" 41 | test_not_present "__ICCARM__" 42 | test_not_present "_MSC_VER" 43 | test_not_present "_WIN32" 44 | test_not_present "__linux__" 45 | -------------------------------------------------------------------------------- /zstd/contrib/linux-kernel/zstd_common_module.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: GPL-2.0+ OR BSD-3-Clause 2 | /* 3 | * Copyright (c) Meta Platforms, Inc. and affiliates. 4 | * All rights reserved. 5 | * 6 | * This source code is licensed under both the BSD-style license (found in the 7 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | * in the COPYING file in the root directory of this source tree). 9 | * You may select, at your option, one of the above-listed licenses. 10 | */ 11 | 12 | #include 13 | 14 | #include "common/huf.h" 15 | #include "common/fse.h" 16 | #include "common/zstd_internal.h" 17 | 18 | // Export symbols shared by compress and decompress into a common module 19 | 20 | #undef ZSTD_isError /* defined within zstd_internal.h */ 21 | EXPORT_SYMBOL_GPL(FSE_readNCount); 22 | EXPORT_SYMBOL_GPL(HUF_readStats); 23 | EXPORT_SYMBOL_GPL(HUF_readStats_wksp); 24 | EXPORT_SYMBOL_GPL(ZSTD_isError); 25 | EXPORT_SYMBOL_GPL(ZSTD_getErrorName); 26 | EXPORT_SYMBOL_GPL(ZSTD_getErrorCode); 27 | 28 | MODULE_LICENSE("Dual BSD/GPL"); 29 | MODULE_DESCRIPTION("Zstd Common"); 30 | -------------------------------------------------------------------------------- /zstd/contrib/premake/premake4.lua: -------------------------------------------------------------------------------- 1 | -- Include zstd.lua in your GENie or premake4 file, which exposes a project_zstd function 2 | dofile('zstd.lua') 3 | 4 | solution 'example' 5 | configurations { 'Debug', 'Release' } 6 | project_zstd('../../lib/') 7 | -------------------------------------------------------------------------------- /zstd/contrib/pzstd/.gitignore: -------------------------------------------------------------------------------- 1 | # compilation result 2 | pzstd 3 | -------------------------------------------------------------------------------- /zstd/contrib/pzstd/ErrorHolder.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | #pragma once 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | namespace pzstd { 17 | 18 | // Coordinates graceful shutdown of the pzstd pipeline 19 | class ErrorHolder { 20 | std::atomic error_; 21 | std::string message_; 22 | 23 | public: 24 | ErrorHolder() : error_(false) {} 25 | 26 | bool hasError() noexcept { 27 | return error_.load(); 28 | } 29 | 30 | void setError(std::string message) noexcept { 31 | // Given multiple possibly concurrent calls, exactly one will ever succeed. 32 | bool expected = false; 33 | if (error_.compare_exchange_strong(expected, true)) { 34 | message_ = std::move(message); 35 | } 36 | } 37 | 38 | bool check(bool predicate, std::string message) noexcept { 39 | if (!predicate) { 40 | setError(std::move(message)); 41 | } 42 | return !hasError(); 43 | } 44 | 45 | std::string getError() noexcept { 46 | error_.store(false); 47 | return std::move(message_); 48 | } 49 | 50 | ~ErrorHolder() { 51 | assert(!hasError()); 52 | } 53 | }; 54 | } 55 | -------------------------------------------------------------------------------- /zstd/contrib/pzstd/SkippableFrame.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | #include "SkippableFrame.h" 10 | #include "mem.h" 11 | #include "utils/Range.h" 12 | 13 | #include 14 | 15 | using namespace pzstd; 16 | 17 | SkippableFrame::SkippableFrame(std::uint32_t size) : frameSize_(size) { 18 | MEM_writeLE32(data_.data(), kSkippableFrameMagicNumber); 19 | MEM_writeLE32(data_.data() + 4, kFrameContentsSize); 20 | MEM_writeLE32(data_.data() + 8, frameSize_); 21 | } 22 | 23 | /* static */ std::size_t SkippableFrame::tryRead(ByteRange bytes) { 24 | if (bytes.size() < SkippableFrame::kSize || 25 | MEM_readLE32(bytes.begin()) != kSkippableFrameMagicNumber || 26 | MEM_readLE32(bytes.begin() + 4) != kFrameContentsSize) { 27 | return 0; 28 | } 29 | return MEM_readLE32(bytes.begin() + 8); 30 | } 31 | -------------------------------------------------------------------------------- /zstd/contrib/pzstd/images/Cspeed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/contrib/pzstd/images/Cspeed.png -------------------------------------------------------------------------------- /zstd/contrib/pzstd/images/Dspeed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/contrib/pzstd/images/Dspeed.png -------------------------------------------------------------------------------- /zstd/contrib/pzstd/main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | #include "ErrorHolder.h" 10 | #include "Options.h" 11 | #include "Pzstd.h" 12 | 13 | using namespace pzstd; 14 | 15 | int main(int argc, const char** argv) { 16 | Options options; 17 | switch (options.parse(argc, argv)) { 18 | case Options::Status::Failure: 19 | return 1; 20 | case Options::Status::Message: 21 | return 0; 22 | default: 23 | break; 24 | } 25 | 26 | return pzstdMain(options); 27 | } 28 | -------------------------------------------------------------------------------- /zstd/contrib/pzstd/test/BUCK: -------------------------------------------------------------------------------- 1 | cxx_test( 2 | name='options_test', 3 | srcs=['OptionsTest.cpp'], 4 | deps=['//contrib/pzstd:options'], 5 | ) 6 | 7 | cxx_test( 8 | name='pzstd_test', 9 | srcs=['PzstdTest.cpp'], 10 | deps=[ 11 | ':round_trip', 12 | '//contrib/pzstd:libpzstd', 13 | '//contrib/pzstd/utils:scope_guard', 14 | '//programs:datagen', 15 | ], 16 | ) 17 | 18 | cxx_binary( 19 | name='round_trip_test', 20 | srcs=['RoundTripTest.cpp'], 21 | deps=[ 22 | ':round_trip', 23 | '//contrib/pzstd/utils:scope_guard', 24 | '//programs:datagen', 25 | ] 26 | ) 27 | 28 | cxx_library( 29 | name='round_trip', 30 | header_namespace='test', 31 | exported_headers=['RoundTrip.h'], 32 | deps=[ 33 | '//contrib/pzstd:libpzstd', 34 | '//contrib/pzstd:options', 35 | '//contrib/pzstd/utils:scope_guard', 36 | ] 37 | ) 38 | -------------------------------------------------------------------------------- /zstd/contrib/pzstd/utils/Likely.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | 10 | /** 11 | * Compiler hints to indicate the fast path of an "if" branch: whether 12 | * the if condition is likely to be true or false. 13 | * 14 | * @author Tudor Bosman (tudorb@fb.com) 15 | */ 16 | 17 | #pragma once 18 | 19 | #undef LIKELY 20 | #undef UNLIKELY 21 | 22 | #if defined(__GNUC__) && __GNUC__ >= 4 23 | #define LIKELY(x) (__builtin_expect((x), 1)) 24 | #define UNLIKELY(x) (__builtin_expect((x), 0)) 25 | #else 26 | #define LIKELY(x) (x) 27 | #define UNLIKELY(x) (x) 28 | #endif 29 | -------------------------------------------------------------------------------- /zstd/contrib/pzstd/utils/Portability.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | 10 | #pragma once 11 | 12 | #include 13 | 14 | // Required for windows, which defines min/max, but we want the std:: version. 15 | #undef min 16 | #undef max 17 | -------------------------------------------------------------------------------- /zstd/contrib/pzstd/utils/ScopeGuard.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | #pragma once 10 | 11 | #include 12 | 13 | namespace pzstd { 14 | 15 | /** 16 | * Dismissable scope guard. 17 | * `Function` must be callable and take no parameters. 18 | * Unless `dismiss()` is called, the callable is executed upon destruction of 19 | * `ScopeGuard`. 20 | * 21 | * Example: 22 | * 23 | * auto guard = makeScopeGuard([&] { cleanup(); }); 24 | */ 25 | template 26 | class ScopeGuard { 27 | Function function; 28 | bool dismissed; 29 | 30 | public: 31 | explicit ScopeGuard(Function&& function) 32 | : function(std::move(function)), dismissed(false) {} 33 | 34 | void dismiss() { 35 | dismissed = true; 36 | } 37 | 38 | ~ScopeGuard() noexcept { 39 | if (!dismissed) { 40 | function(); 41 | } 42 | } 43 | }; 44 | 45 | /// Creates a scope guard from `function`. 46 | template 47 | ScopeGuard makeScopeGuard(Function&& function) { 48 | return ScopeGuard(std::forward(function)); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /zstd/contrib/pzstd/utils/test/BUCK: -------------------------------------------------------------------------------- 1 | cxx_test( 2 | name='buffer_test', 3 | srcs=['BufferTest.cpp'], 4 | deps=['//contrib/pzstd/utils:buffer'], 5 | ) 6 | 7 | cxx_test( 8 | name='range_test', 9 | srcs=['RangeTest.cpp'], 10 | deps=['//contrib/pzstd/utils:range'], 11 | ) 12 | 13 | cxx_test( 14 | name='resource_pool_test', 15 | srcs=['ResourcePoolTest.cpp'], 16 | deps=['//contrib/pzstd/utils:resource_pool'], 17 | ) 18 | 19 | cxx_test( 20 | name='scope_guard_test', 21 | srcs=['ScopeGuardTest.cpp'], 22 | deps=['//contrib/pzstd/utils:scope_guard'], 23 | ) 24 | 25 | cxx_test( 26 | name='thread_pool_test', 27 | srcs=['ThreadPoolTest.cpp'], 28 | deps=['//contrib/pzstd/utils:thread_pool'], 29 | ) 30 | 31 | cxx_test( 32 | name='work_queue_test', 33 | srcs=['RangeTest.cpp'], 34 | deps=['//contrib/pzstd/utils:work_queue'], 35 | ) 36 | -------------------------------------------------------------------------------- /zstd/contrib/pzstd/utils/test/ScopeGuardTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | */ 9 | #include "utils/ScopeGuard.h" 10 | 11 | #include 12 | 13 | using namespace pzstd; 14 | 15 | TEST(ScopeGuard, Dismiss) { 16 | { 17 | auto guard = makeScopeGuard([&] { EXPECT_TRUE(false); }); 18 | guard.dismiss(); 19 | } 20 | } 21 | 22 | TEST(ScopeGuard, Executes) { 23 | bool executed = false; 24 | { 25 | auto guard = makeScopeGuard([&] { executed = true; }); 26 | } 27 | EXPECT_TRUE(executed); 28 | } 29 | -------------------------------------------------------------------------------- /zstd/contrib/recovery/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | .PHONY: all 11 | all: recover_directory 12 | 13 | ZSTDLIBDIR ?= ../../lib 14 | PROGRAMDIR ?= ../../programs 15 | 16 | CFLAGS ?= -O3 17 | CFLAGS += -I$(ZSTDLIBDIR) -I$(PROGRAMDIR) 18 | CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ 19 | -Wstrict-aliasing=1 -Wswitch-enum \ 20 | -Wstrict-prototypes -Wundef \ 21 | -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ 22 | -Wredundant-decls -Wmissing-prototypes 23 | CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) 24 | FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) 25 | 26 | .PHONY: $(ZSTDLIBDIR)/libzstd.a 27 | $(ZSTDLIBDIR)/libzstd.a: 28 | $(MAKE) -C $(ZSTDLIBDIR) libzstd.a 29 | 30 | recover_directory: recover_directory.c $(ZSTDLIBDIR)/libzstd.a $(PROGRAMDIR)/util.c 31 | $(CC) $(FLAGS) $^ -o $@$(EXT) 32 | 33 | .PHONY: clean 34 | clean: 35 | rm -f recover_directory 36 | -------------------------------------------------------------------------------- /zstd/contrib/seekable_format/examples/.gitignore: -------------------------------------------------------------------------------- 1 | seekable_compression 2 | seekable_decompression 3 | seekable_decompression_mem 4 | parallel_processing 5 | parallel_compression 6 | -------------------------------------------------------------------------------- /zstd/contrib/seekable_format/tests/.gitignore: -------------------------------------------------------------------------------- 1 | seekable_tests 2 | -------------------------------------------------------------------------------- /zstd/contrib/seekable_format/tests/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # ################################################################ 9 | 10 | # This Makefile presumes libzstd is built, using `make` in / or /lib/ 11 | 12 | ZSTDLIB_PATH = ../../../lib 13 | ZSTDLIB_NAME = libzstd.a 14 | ZSTDLIB = $(ZSTDLIB_PATH)/$(ZSTDLIB_NAME) 15 | 16 | CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I../ -I$(ZSTDLIB_PATH) -I$(ZSTDLIB_PATH)/common 17 | 18 | CFLAGS ?= -O3 19 | CFLAGS += -g -Wall -Wextra -Wcast-qual -Wcast-align -Wconversion \ 20 | -Wformat=2 -Wstrict-aliasing=1 21 | 22 | SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_decompress.c $(ZSTDLIB) 23 | 24 | .PHONY: default clean test 25 | default: test 26 | 27 | test: seekable_tests 28 | ./seekable_tests 29 | 30 | $(ZSTDLIB): 31 | $(MAKE) -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME) 32 | 33 | seekable_tests : $(SEEKABLE_OBJS) 34 | 35 | clean: 36 | @$(RM) core *.o tmp* result* *.zst \ 37 | seekable_tests 38 | @echo Cleaning completed 39 | -------------------------------------------------------------------------------- /zstd/contrib/snap/snapcraft.yaml: -------------------------------------------------------------------------------- 1 | name: zstd 2 | version: git 3 | summary: Zstandard - Fast real-time compression algorithm 4 | description: | 5 | Zstandard, or zstd as short version, is a fast lossless compression 6 | algorithm, targeting real-time compression scenarios at zlib-level and better 7 | compression ratios. It's backed by a very fast entropy stage, provided by 8 | Huff0 and FSE library 9 | 10 | grade: devel # must be 'stable' to release into candidate/stable channels 11 | confinement: devmode # use 'strict' once you have the right plugs and slots 12 | 13 | apps: 14 | zstd: 15 | command: usr/local/bin/zstd 16 | plugs: [home, removable-media] 17 | zstdgrep: 18 | command: usr/local/bin/zstdgrep 19 | plugs: [home, removable-media] 20 | zstdless: 21 | command: usr/local/bin/zstdless 22 | plugs: [home, removable-media] 23 | 24 | parts: 25 | zstd: 26 | source: . 27 | plugin: make 28 | build-packages: [g++] 29 | -------------------------------------------------------------------------------- /zstd/doc/README.md: -------------------------------------------------------------------------------- 1 | Zstandard Documentation 2 | ======================= 3 | 4 | This directory contains material defining the Zstandard format, 5 | as well as detailed instructions to use `zstd` library. 6 | 7 | __`zstd_manual.html`__ : Documentation of `zstd.h` API, in html format. 8 | Unfortunately, Github doesn't display `html` files in parsed format, just as source code. 9 | For a readable display of html API documentation of latest release, 10 | use this link: [https://raw.githack.com/facebook/zstd/release/doc/zstd_manual.html](https://raw.githack.com/facebook/zstd/release/doc/zstd_manual.html) . 11 | 12 | __`zstd_compression_format.md`__ : This document defines the Zstandard compression format. 13 | Compliant decoders must adhere to this document, 14 | and compliant encoders must generate data that follows it. 15 | 16 | Should you look for resources to develop your own port of Zstandard algorithm, 17 | you may find the following resources useful : 18 | 19 | __`educational_decoder`__ : This directory contains an implementation of a Zstandard decoder, 20 | compliant with the Zstandard compression format. 21 | It can be used, for example, to better understand the format, 22 | or as the basis for a separate implementation of Zstandard decoder. 23 | 24 | [__`decode_corpus`__](https://github.com/facebook/zstd/tree/dev/tests#decodecorpus---tool-to-generate-zstandard-frames-for-decoder-testing) : 25 | This tool, stored in `/tests` directory, is able to generate random valid frames, 26 | which is useful if you wish to test your decoder and verify it fully supports the specification. 27 | -------------------------------------------------------------------------------- /zstd/doc/educational_decoder/.gitignore: -------------------------------------------------------------------------------- 1 | # Build artifacts 2 | harness 3 | -------------------------------------------------------------------------------- /zstd/doc/images/CSpeed2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/doc/images/CSpeed2.png -------------------------------------------------------------------------------- /zstd/doc/images/DCspeed5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/doc/images/DCspeed5.png -------------------------------------------------------------------------------- /zstd/doc/images/DSpeed3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/doc/images/DSpeed3.png -------------------------------------------------------------------------------- /zstd/doc/images/cdict_v136.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/doc/images/cdict_v136.png -------------------------------------------------------------------------------- /zstd/doc/images/dict-cr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/doc/images/dict-cr.png -------------------------------------------------------------------------------- /zstd/doc/images/dict-cs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/doc/images/dict-cs.png -------------------------------------------------------------------------------- /zstd/doc/images/dict-ds.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/doc/images/dict-ds.png -------------------------------------------------------------------------------- /zstd/doc/images/zstd_cdict_v1_3_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/doc/images/zstd_cdict_v1_3_5.png -------------------------------------------------------------------------------- /zstd/doc/images/zstd_logo86.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/doc/images/zstd_logo86.png -------------------------------------------------------------------------------- /zstd/examples/.gitignore: -------------------------------------------------------------------------------- 1 | #build 2 | simple_compression 3 | simple_decompression 4 | multiple_simple_compression 5 | dictionary_compression 6 | dictionary_decompression 7 | streaming_compression 8 | streaming_decompression 9 | multiple_streaming_compression 10 | streaming_memory_usage 11 | 12 | #test artefact 13 | tmp* 14 | test* 15 | *.zst 16 | -------------------------------------------------------------------------------- /zstd/lib/.gitignore: -------------------------------------------------------------------------------- 1 | # make install artefact 2 | libzstd.pc 3 | libzstd-nomt 4 | -------------------------------------------------------------------------------- /zstd/lib/common/debug.c: -------------------------------------------------------------------------------- 1 | /* ****************************************************************** 2 | * debug 3 | * Part of FSE library 4 | * Copyright (c) Meta Platforms, Inc. and affiliates. 5 | * 6 | * You can contact the author at : 7 | * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy 8 | * 9 | * This source code is licensed under both the BSD-style license (found in the 10 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 11 | * in the COPYING file in the root directory of this source tree). 12 | * You may select, at your option, one of the above-listed licenses. 13 | ****************************************************************** */ 14 | 15 | 16 | /* 17 | * This module only hosts one global variable 18 | * which can be used to dynamically influence the verbosity of traces, 19 | * such as DEBUGLOG and RAWLOG 20 | */ 21 | 22 | #include "debug.h" 23 | 24 | #if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2) 25 | /* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a 26 | * translation unit is empty. So remove this from Linux kernel builds, but 27 | * otherwise just leave it in. 28 | */ 29 | int g_debuglevel = DEBUGLEVEL; 30 | #endif 31 | -------------------------------------------------------------------------------- /zstd/lib/common/xxhash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xxHash - Extremely Fast Hash algorithm 3 | * Copyright (c) Yann Collet - Meta Platforms, Inc 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* 12 | * xxhash.c instantiates functions defined in xxhash.h 13 | */ 14 | 15 | #define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ 16 | #define XXH_IMPLEMENTATION /* access definitions */ 17 | 18 | #include "xxhash.h" 19 | -------------------------------------------------------------------------------- /zstd/lib/compress/zstd_compress_superblock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_COMPRESS_ADVANCED_H 12 | #define ZSTD_COMPRESS_ADVANCED_H 13 | 14 | /*-************************************* 15 | * Dependencies 16 | ***************************************/ 17 | 18 | #include "../zstd.h" /* ZSTD_CCtx */ 19 | 20 | /*-************************************* 21 | * Target Compressed Block Size 22 | ***************************************/ 23 | 24 | /* ZSTD_compressSuperBlock() : 25 | * Used to compress a super block when targetCBlockSize is being used. 26 | * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */ 27 | size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, 28 | void* dst, size_t dstCapacity, 29 | void const* src, size_t srcSize, 30 | unsigned lastBlock); 31 | 32 | #endif /* ZSTD_COMPRESS_ADVANCED_H */ 33 | -------------------------------------------------------------------------------- /zstd/lib/compress/zstd_fast.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTD_FAST_H 12 | #define ZSTD_FAST_H 13 | 14 | #if defined (__cplusplus) 15 | extern "C" { 16 | #endif 17 | 18 | #include "../common/mem.h" /* U32 */ 19 | #include "zstd_compress_internal.h" 20 | 21 | void ZSTD_fillHashTable(ZSTD_matchState_t* ms, 22 | void const* end, ZSTD_dictTableLoadMethod_e dtlm, 23 | ZSTD_tableFillPurpose_e tfp); 24 | size_t ZSTD_compressBlock_fast( 25 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 26 | void const* src, size_t srcSize); 27 | size_t ZSTD_compressBlock_fast_dictMatchState( 28 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 29 | void const* src, size_t srcSize); 30 | size_t ZSTD_compressBlock_fast_extDict( 31 | ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], 32 | void const* src, size_t srcSize); 33 | 34 | #if defined (__cplusplus) 35 | } 36 | #endif 37 | 38 | #endif /* ZSTD_FAST_H */ 39 | -------------------------------------------------------------------------------- /zstd/lib/decompress/zstd_ddict.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | 12 | #ifndef ZSTD_DDICT_H 13 | #define ZSTD_DDICT_H 14 | 15 | /*-******************************************************* 16 | * Dependencies 17 | *********************************************************/ 18 | #include "../common/zstd_deps.h" /* size_t */ 19 | #include "../zstd.h" /* ZSTD_DDict, and several public functions */ 20 | 21 | 22 | /*-******************************************************* 23 | * Interface 24 | *********************************************************/ 25 | 26 | /* note: several prototypes are already published in `zstd.h` : 27 | * ZSTD_createDDict() 28 | * ZSTD_createDDict_byReference() 29 | * ZSTD_createDDict_advanced() 30 | * ZSTD_freeDDict() 31 | * ZSTD_initStaticDDict() 32 | * ZSTD_sizeof_DDict() 33 | * ZSTD_estimateDDictSize() 34 | * ZSTD_getDictID_fromDict() 35 | */ 36 | 37 | const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); 38 | size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); 39 | 40 | void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); 41 | 42 | 43 | 44 | #endif /* ZSTD_DDICT_H */ 45 | -------------------------------------------------------------------------------- /zstd/lib/deprecated/zbuff_common.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /*-************************************* 12 | * Dependencies 13 | ***************************************/ 14 | #include "../common/error_private.h" 15 | #include "zbuff.h" 16 | 17 | /*-**************************************** 18 | * ZBUFF Error Management (deprecated) 19 | ******************************************/ 20 | 21 | /*! ZBUFF_isError() : 22 | * tells if a return value is an error code */ 23 | unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } 24 | /*! ZBUFF_getErrorName() : 25 | * provides error code string from function result (useful for debugging) */ 26 | const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } 27 | -------------------------------------------------------------------------------- /zstd/lib/dll/example/Makefile: -------------------------------------------------------------------------------- 1 | # ################################################################ 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # All rights reserved. 4 | # 5 | # This source code is licensed under both the BSD-style license (found in the 6 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | # in the COPYING file in the root directory of this source tree). 8 | # You may select, at your option, one of the above-listed licenses. 9 | # ################################################################ 10 | 11 | VOID := /dev/null 12 | ZSTDDIR := ../include 13 | LIBDIR := ../static 14 | DLLDIR := ../dll 15 | 16 | CFLAGS ?= -O3 # can select custom flags. For example : CFLAGS="-O2 -g" make 17 | CFLAGS += -Wall -Wextra -Wundef -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ 18 | -Wdeclaration-after-statement -Wstrict-prototypes \ 19 | -Wpointer-arith -Wstrict-aliasing=1 20 | CFLAGS += $(MOREFLAGS) 21 | CPPFLAGS:= -I$(ZSTDDIR) -DXXH_NAMESPACE=ZSTD_ 22 | FLAGS := $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) 23 | 24 | 25 | # Define *.exe as extension for Windows systems 26 | ifneq (,$(filter Windows%,$(OS))) 27 | EXT =.exe 28 | else 29 | EXT = 30 | endif 31 | 32 | .PHONY: default fullbench-dll fullbench-lib 33 | 34 | 35 | default: all 36 | 37 | all: fullbench-dll fullbench-lib 38 | 39 | 40 | fullbench-lib: fullbench.c datagen.c 41 | $(CC) $(FLAGS) $^ -o $@$(EXT) $(LIBDIR)/libzstd_static.lib 42 | 43 | fullbench-dll: fullbench.c datagen.c 44 | $(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(DLLDIR)/libzstd.dll 45 | 46 | clean: 47 | @$(RM) fullbench-dll$(EXT) fullbench-lib$(EXT) \ 48 | @echo Cleaning completed 49 | -------------------------------------------------------------------------------- /zstd/lib/dll/example/build_package.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | MKDIR bin\dll bin\static bin\example bin\include 3 | COPY tests\fullbench.c bin\example\ 4 | COPY programs\datagen.c bin\example\ 5 | COPY programs\datagen.h bin\example\ 6 | COPY programs\util.h bin\example\ 7 | COPY programs\platform.h bin\example\ 8 | COPY lib\common\mem.h bin\example\ 9 | COPY lib\common\zstd_internal.h bin\example\ 10 | COPY lib\common\error_private.h bin\example\ 11 | COPY lib\common\xxhash.h bin\example\ 12 | COPY lib\libzstd.a bin\static\libzstd_static.lib 13 | COPY lib\dll\libzstd.* bin\dll\ 14 | COPY lib\dll\example\Makefile bin\example\ 15 | COPY lib\dll\example\fullbench-dll.* bin\example\ 16 | COPY lib\dll\example\README.md bin\ 17 | COPY lib\zstd.h bin\include\ 18 | COPY lib\common\zstd_errors.h bin\include\ 19 | COPY lib\dictBuilder\zdict.h bin\include\ 20 | COPY programs\zstd.exe bin\zstd.exe 21 | -------------------------------------------------------------------------------- /zstd/lib/dll/example/fullbench-dll.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | # Visual Studio Express 2012 for Windows Desktop 3 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fullbench-dll", "fullbench-dll.vcxproj", "{13992FD2-077E-4954-B065-A428198201A9}" 4 | EndProject 5 | Global 6 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 7 | Debug|Win32 = Debug|Win32 8 | Debug|x64 = Debug|x64 9 | Release|Win32 = Release|Win32 10 | Release|x64 = Release|x64 11 | EndGlobalSection 12 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 13 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.ActiveCfg = Debug|Win32 14 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|Win32.Build.0 = Debug|Win32 15 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.ActiveCfg = Debug|x64 16 | {13992FD2-077E-4954-B065-A428198201A9}.Debug|x64.Build.0 = Debug|x64 17 | {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.ActiveCfg = Release|Win32 18 | {13992FD2-077E-4954-B065-A428198201A9}.Release|Win32.Build.0 = Release|Win32 19 | {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.ActiveCfg = Release|x64 20 | {13992FD2-077E-4954-B065-A428198201A9}.Release|x64.Build.0 = Release|x64 21 | EndGlobalSection 22 | GlobalSection(SolutionProperties) = preSolution 23 | HideSolutionNode = FALSE 24 | EndGlobalSection 25 | EndGlobal 26 | -------------------------------------------------------------------------------- /zstd/lib/libzstd.pc.in: -------------------------------------------------------------------------------- 1 | # ZSTD - standard compression algorithm 2 | # Copyright (c) Meta Platforms, Inc. and affiliates. 3 | # BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) 4 | 5 | prefix=@PREFIX@ 6 | exec_prefix=@EXEC_PREFIX@ 7 | includedir=@INCLUDEDIR@ 8 | libdir=@LIBDIR@ 9 | 10 | Name: zstd 11 | Description: fast lossless compression algorithm library 12 | URL: https://facebook.github.io/zstd/ 13 | Version: @VERSION@ 14 | Libs: -L${libdir} -lzstd 15 | Libs.private: @LIBS_PRIVATE@ 16 | Cflags: -I${includedir} 17 | -------------------------------------------------------------------------------- /zstd/lib/module.modulemap: -------------------------------------------------------------------------------- 1 | module libzstd [extern_c] { 2 | header "zstd.h" 3 | export * 4 | config_macros [exhaustive] \ 5 | /* zstd.h */ \ 6 | ZSTD_STATIC_LINKING_ONLY, \ 7 | ZSTDLIB_VISIBILITY, \ 8 | ZSTDLIB_VISIBLE, \ 9 | ZSTDLIB_HIDDEN, \ 10 | ZSTD_DLL_EXPORT, \ 11 | ZSTDLIB_STATIC_API, \ 12 | ZSTD_DISABLE_DEPRECATE_WARNINGS, \ 13 | ZSTD_CLEVEL_DEFAULT, \ 14 | /* zdict.h */ \ 15 | ZDICT_STATIC_LINKING_ONLY, \ 16 | ZDICTLIB_VISIBLE, \ 17 | ZDICTLIB_HIDDEN, \ 18 | ZDICTLIB_VISIBILITY, \ 19 | ZDICTLIB_STATIC_API, \ 20 | ZDICT_DISABLE_DEPRECATE_WARNINGS, \ 21 | /* zstd_errors.h */ \ 22 | ZSTDERRORLIB_VISIBLE, \ 23 | ZSTDERRORLIB_HIDDEN, \ 24 | ZSTDERRORLIB_VISIBILITY 25 | 26 | module dictbuilder [extern_c] { 27 | header "zdict.h" 28 | export * 29 | } 30 | 31 | module errors [extern_c] { 32 | header "zstd_errors.h" 33 | export * 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /zstd/programs/.gitignore: -------------------------------------------------------------------------------- 1 | # local binary (Makefile) 2 | zstd 3 | zstd32 4 | zstd4 5 | zstd-compress 6 | zstd-decompress 7 | zstd-frugal 8 | zstd-small 9 | zstd-nolegacy 10 | zstd-dictBuilder 11 | zstd-dll 12 | zstd_arm64 13 | zstd_x64 14 | 15 | # Object files 16 | *.o 17 | *.ko 18 | default.profraw 19 | have_zlib 20 | 21 | # Executables 22 | *.exe 23 | *.out 24 | *.app 25 | 26 | # Default result files 27 | dictionary 28 | grillResults.txt 29 | _* 30 | tmp* 31 | *.zst 32 | result 33 | out 34 | 35 | # fuzzer 36 | afl 37 | 38 | # Misc files 39 | *.bat 40 | !windres/generate_res.bat 41 | dirTest* 42 | -------------------------------------------------------------------------------- /zstd/programs/BUCK: -------------------------------------------------------------------------------- 1 | cxx_binary( 2 | name='zstd', 3 | headers=glob(['*.h'], excludes=['datagen.h', 'platform.h', 'util.h']), 4 | srcs=glob(['*.c'], excludes=['datagen.c']), 5 | deps=[ 6 | ':datagen', 7 | ':util', 8 | '//lib:zstd', 9 | '//lib:zdict', 10 | '//lib:mem', 11 | '//lib:xxhash', 12 | ], 13 | preprocessor_flags=[ 14 | '-DZSTD_GZCOMPRESS', 15 | '-DZSTD_GZDECOMPRESS', 16 | '-DZSTD_LZMACOMPRESS', 17 | '-DZSTD_LZMADECOMPRES', 18 | '-DZSTD_LZ4COMPRESS', 19 | '-DZSTD_LZ4DECOMPRES', 20 | ], 21 | linker_flags=[ 22 | '-lz', 23 | '-llzma', 24 | '-llz4', 25 | ], 26 | ) 27 | 28 | cxx_library( 29 | name='datagen', 30 | visibility=['PUBLIC'], 31 | header_namespace='', 32 | exported_headers=['datagen.h'], 33 | srcs=['datagen.c'], 34 | deps=['//lib:mem'], 35 | ) 36 | 37 | 38 | cxx_library( 39 | name='util', 40 | visibility=['PUBLIC'], 41 | header_namespace='', 42 | exported_headers=['util.h', 'platform.h'], 43 | deps=['//lib:mem'], 44 | ) 45 | -------------------------------------------------------------------------------- /zstd/programs/datagen.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | 12 | #ifndef DATAGEN_H 13 | #define DATAGEN_H 14 | 15 | #include /* size_t */ 16 | 17 | void RDG_genStdout(unsigned long long size, double matchProba, double litProba, unsigned seed); 18 | void RDG_genBuffer(void* buffer, size_t size, double matchProba, double litProba, unsigned seed); 19 | /*!RDG_genBuffer 20 | Generate 'size' bytes of compressible data into 'buffer'. 21 | Compressibility can be controlled using 'matchProba', which is floating point value between 0 and 1. 22 | 'LitProba' is optional, it affect variability of individual bytes. If litProba==0.0, default value will be used. 23 | Generated data pattern can be modified using different 'seed'. 24 | For a triplet (matchProba, litProba, seed), the function always generate the same content. 25 | 26 | RDG_genStdout 27 | Same as RDG_genBuffer, but generates data into stdout 28 | */ 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /zstd/programs/dibio.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* This library is designed for a single-threaded console application. 12 | * It exit() and printf() into stderr when it encounters an error condition. */ 13 | 14 | #ifndef DIBIO_H_003 15 | #define DIBIO_H_003 16 | 17 | 18 | /*-************************************* 19 | * Dependencies 20 | ***************************************/ 21 | #define ZDICT_STATIC_LINKING_ONLY 22 | #include "../lib/zdict.h" /* ZDICT_params_t */ 23 | 24 | 25 | /*-************************************* 26 | * Public functions 27 | ***************************************/ 28 | /*! DiB_trainFromFiles() : 29 | Train a dictionary from a set of files provided by `fileNamesTable`. 30 | Resulting dictionary is written into file `dictFileName`. 31 | `parameters` is optional and can be provided with values set to 0, meaning "default". 32 | @return : 0 == ok. Any other : error. 33 | */ 34 | int DiB_trainFromFiles(const char* dictFileName, size_t maxDictSize, 35 | const char** fileNamesTable, int nbFiles, size_t chunkSize, 36 | ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams, 37 | ZDICT_fastCover_params_t* fastCoverParams, int optimize, unsigned memLimit); 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /zstd/programs/lorem.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* lorem ipsum generator */ 12 | 13 | #include /* size_t */ 14 | 15 | /* 16 | * LOREM_genBuffer(): 17 | * Generate @size bytes of compressible data using lorem ipsum generator 18 | * into provided @buffer. 19 | */ 20 | void LOREM_genBuffer(void* buffer, size_t size, unsigned seed); 21 | 22 | /* 23 | * LOREM_genBlock(): 24 | * Similar to LOREM_genBuffer, with additional controls : 25 | * - @first : generate the first sentence 26 | * - @fill : fill the entire @buffer, 27 | * if ==0: generate one paragraph at most. 28 | * @return : nb of bytes generated into @buffer. 29 | */ 30 | size_t LOREM_genBlock(void* buffer, size_t size, 31 | unsigned seed, 32 | int first, int fill); 33 | -------------------------------------------------------------------------------- /zstd/programs/windres/verrsrc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | /* minimal set of defines required to generate zstd.res from zstd.rc */ 11 | 12 | #define VS_VERSION_INFO 1 13 | 14 | #define VS_FFI_FILEFLAGSMASK 0x0000003FL 15 | #define VOS_NT_WINDOWS32 0x00040004L 16 | #define VFT_DLL 0x00000002L 17 | #define VFT2_UNKNOWN 0x00000000L 18 | -------------------------------------------------------------------------------- /zstd/programs/windres/zstd.rc: -------------------------------------------------------------------------------- 1 | // Microsoft Visual C++ generated resource script. 2 | // 3 | 4 | #include "zstd.h" /* ZSTD_VERSION_STRING */ 5 | #define APSTUDIO_READONLY_SYMBOLS 6 | #include "verrsrc.h" 7 | #undef APSTUDIO_READONLY_SYMBOLS 8 | 9 | 10 | #if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU) 11 | LANGUAGE 9, 1 12 | 13 | ///////////////////////////////////////////////////////////////////////////// 14 | // 15 | // Version 16 | // 17 | 18 | VS_VERSION_INFO VERSIONINFO 19 | FILEVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 20 | PRODUCTVERSION ZSTD_VERSION_MAJOR,ZSTD_VERSION_MINOR,ZSTD_VERSION_RELEASE,0 21 | FILEFLAGSMASK VS_FFI_FILEFLAGSMASK 22 | #ifdef _DEBUG 23 | FILEFLAGS VS_FF_DEBUG 24 | #else 25 | FILEFLAGS 0x0L 26 | #endif 27 | FILEOS VOS_NT_WINDOWS32 28 | FILETYPE VFT_DLL 29 | FILESUBTYPE VFT2_UNKNOWN 30 | BEGIN 31 | BLOCK "StringFileInfo" 32 | BEGIN 33 | BLOCK "040904B0" 34 | BEGIN 35 | VALUE "CompanyName", "Meta Platforms, Inc." 36 | VALUE "FileDescription", "Zstandard - Fast and efficient compression algorithm" 37 | VALUE "FileVersion", ZSTD_VERSION_STRING 38 | VALUE "InternalName", "zstd.exe" 39 | VALUE "LegalCopyright", "Copyright (c) Meta Platforms, Inc. and affiliates." 40 | VALUE "OriginalFilename", "zstd.exe" 41 | VALUE "ProductName", "Zstandard" 42 | VALUE "ProductVersion", ZSTD_VERSION_STRING 43 | END 44 | END 45 | BLOCK "VarFileInfo" 46 | BEGIN 47 | VALUE "Translation", 0x0409, 1200 48 | END 49 | END 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /zstd/programs/windres/zstd32.res: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/programs/windres/zstd32.res -------------------------------------------------------------------------------- /zstd/programs/windres/zstd64.res: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/programs/windres/zstd64.res -------------------------------------------------------------------------------- /zstd/programs/zstdcli_trace.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef ZSTDCLI_TRACE_H 12 | #define ZSTDCLI_TRACE_H 13 | 14 | /** 15 | * Enable tracing - log to filename. 16 | */ 17 | void TRACE_enable(char const* filename); 18 | 19 | /** 20 | * Shut down the tracing library. 21 | */ 22 | void TRACE_finish(void); 23 | 24 | #endif /* ZSTDCLI_TRACE_H */ 25 | -------------------------------------------------------------------------------- /zstd/programs/zstdgrep.1: -------------------------------------------------------------------------------- 1 | . 2 | .TH "ZSTDGREP" "1" "March 2024" "zstd 1.5.6" "User Commands" 3 | . 4 | .SH "NAME" 5 | \fBzstdgrep\fR \- print lines matching a pattern in zstandard\-compressed files 6 | . 7 | .SH "SYNOPSIS" 8 | \fBzstdgrep\fR [\fIgrep\-flags\fR] [\-\-] \fIpattern\fR [\fIfiles\fR \.\.\.] 9 | . 10 | .SH "DESCRIPTION" 11 | \fBzstdgrep\fR runs \fBgrep\fR(1) on files, or \fBstdin\fR if no files argument is given, after decompressing them with \fBzstdcat\fR(1)\. 12 | . 13 | .P 14 | The \fIgrep\-flags\fR and \fIpattern\fR arguments are passed on to \fBgrep\fR(1)\. If an \fB\-e\fR flag is found in the \fIgrep\-flags\fR, \fBzstdgrep\fR will not look for a \fIpattern\fR argument\. 15 | . 16 | .P 17 | Note that modern \fBgrep\fR alternatives such as \fBripgrep\fR (\fBrg\fR(1)) support \fBzstd\fR\-compressed files out of the box, and can prove better alternatives than \fBzstdgrep\fR notably for unsupported complex pattern searches\. Note though that such alternatives may also feature some minor command line differences\. 18 | . 19 | .SH "EXIT STATUS" 20 | In case of missing arguments or missing pattern, 1 will be returned, otherwise 0\. 21 | . 22 | .SH "SEE ALSO" 23 | \fBzstd\fR(1) 24 | . 25 | .SH "AUTHORS" 26 | Thomas Klausner \fIwiz@NetBSD\.org\fR 27 | -------------------------------------------------------------------------------- /zstd/programs/zstdgrep.1.md: -------------------------------------------------------------------------------- 1 | zstdgrep(1) -- print lines matching a pattern in zstandard-compressed files 2 | ============================================================================ 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | `zstdgrep` [] [--] [ ...] 8 | 9 | 10 | DESCRIPTION 11 | ----------- 12 | `zstdgrep` runs `grep`(1) on files, or `stdin` if no files argument is given, after decompressing them with `zstdcat`(1). 13 | 14 | The and arguments are passed on to `grep`(1). If an `-e` flag is found in the , `zstdgrep` will not look for a argument. 15 | 16 | Note that modern `grep` alternatives such as `ripgrep` (`rg`(1)) support `zstd`-compressed files out of the box, 17 | and can prove better alternatives than `zstdgrep` notably for unsupported complex pattern searches. 18 | Note though that such alternatives may also feature some minor command line differences. 19 | 20 | EXIT STATUS 21 | ----------- 22 | In case of missing arguments or missing pattern, 1 will be returned, otherwise 0. 23 | 24 | SEE ALSO 25 | -------- 26 | `zstd`(1) 27 | 28 | AUTHORS 29 | ------- 30 | Thomas Klausner 31 | -------------------------------------------------------------------------------- /zstd/programs/zstdless: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | zstd=${ZSTD:-zstd} 4 | 5 | # TODO: Address quirks and bugs tied to old versions of less, provide a mechanism to pass flags directly to zstd 6 | 7 | export LESSOPEN="|-${zstd} -cdfq %s" 8 | exec less "$@" 9 | -------------------------------------------------------------------------------- /zstd/programs/zstdless.1: -------------------------------------------------------------------------------- 1 | . 2 | .TH "ZSTDLESS" "1" "March 2024" "zstd 1.5.6" "User Commands" 3 | . 4 | .SH "NAME" 5 | \fBzstdless\fR \- view zstandard\-compressed files 6 | . 7 | .SH "SYNOPSIS" 8 | \fBzstdless\fR [\fIflags\fR] [\fIfile\fR \.\.\.] 9 | . 10 | .SH "DESCRIPTION" 11 | \fBzstdless\fR runs \fBless\fR(1) on files or stdin, if no \fIfile\fR argument is given, after decompressing them with \fBzstdcat\fR(1)\. 12 | . 13 | .SH "SEE ALSO" 14 | \fBzstd\fR(1) 15 | -------------------------------------------------------------------------------- /zstd/programs/zstdless.1.md: -------------------------------------------------------------------------------- 1 | zstdless(1) -- view zstandard-compressed files 2 | ============================================================================ 3 | 4 | SYNOPSIS 5 | -------- 6 | 7 | `zstdless` [] [ ...] 8 | 9 | 10 | DESCRIPTION 11 | ----------- 12 | `zstdless` runs `less`(1) on files or stdin, if no argument is given, after decompressing them with `zstdcat`(1). 13 | 14 | SEE ALSO 15 | -------- 16 | `zstd`(1) 17 | -------------------------------------------------------------------------------- /zstd/tests/.gitignore: -------------------------------------------------------------------------------- 1 | # local binary (Makefile) 2 | fullbench 3 | fullbench32 4 | fullbench-lib 5 | fuzzer 6 | fuzzer32 7 | fuzzer-dll 8 | zbufftest 9 | zbufftest32 10 | zbufftest-dll 11 | zstreamtest 12 | zstreamtest32 13 | zstreamtest_asan 14 | zstreamtest_tsan 15 | zstreamtest_ubsan 16 | zstreamtest-dll 17 | datagen 18 | paramgrill 19 | paramgrill32 20 | roundTripCrash 21 | longmatch 22 | symbols 23 | legacy 24 | decodecorpus 25 | pool 26 | poolTests 27 | invalidDictionaries 28 | checkTag 29 | zcat 30 | zstdcat 31 | tm 32 | 33 | # test artifacts 34 | dictionary 35 | grillResults.txt 36 | _* 37 | tmp* 38 | *.zst 39 | *.gz 40 | !gzip/hufts-segv.gz 41 | result 42 | out 43 | *.zstd 44 | hello* 45 | world 46 | 47 | # Tmp test directory 48 | zstdtest 49 | speedTest 50 | versionsTest 51 | namespaceTest 52 | dirTest* 53 | 54 | # fuzzer 55 | afl 56 | 57 | # Local script 58 | startSpeedTest 59 | speedTest.pid 60 | *.bat 61 | 62 | # Generic Object files 63 | *.o 64 | *.ko 65 | 66 | # Generic Executables 67 | *.exe 68 | *.out 69 | *.app 70 | 71 | # Specific exclusions 72 | !golden-decompression/*.zst 73 | -------------------------------------------------------------------------------- /zstd/tests/check_size.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # ################################################################ 3 | # Copyright (c) Meta Platforms, Inc. and affiliates. 4 | # All rights reserved. 5 | # 6 | # This source code is licensed under both the BSD-style license (found in the 7 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 8 | # in the COPYING file in the root directory of this source tree). 9 | # You may select, at your option, one of the above-listed licenses. 10 | # ################################################################ 11 | 12 | import os 13 | import subprocess 14 | import sys 15 | 16 | if len(sys.argv) != 3: 17 | print(f"Usage: {sys.argv[0]} FILE SIZE_LIMIT") 18 | sys.exit(1) 19 | 20 | file = sys.argv[1] 21 | limit = int(sys.argv[2]) 22 | 23 | if not os.path.exists(file): 24 | print(f"{file} does not exist") 25 | sys.exit(1) 26 | 27 | size = os.path.getsize(file) 28 | 29 | if size > limit: 30 | print(f"file {file} is {size} bytes, which is greater than the limit of {limit} bytes") 31 | sys.exit(1) 32 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/.gitignore: -------------------------------------------------------------------------------- 1 | !bin/ 2 | !datagen 3 | !zstdcat 4 | 5 | scratch/ 6 | bin/symlinks 7 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/args.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | println "+ zstd --blah" >&2 4 | zstd --blah 5 | println "+ zstd -xz" >&2 6 | zstd -xz 7 | println "+ zstd --adapt=min=1,maxx=2 file.txt" >&2 8 | zstd --adapt=min=1,maxx=2 file.txt 9 | println "+ zstd --train-cover=k=48,d=8,steps32 file.txt" >&2 10 | zstd --train-cover=k=48,d=8,steps32 file.txt 11 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/args.sh.exit: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/args.sh.stderr.glob: -------------------------------------------------------------------------------- 1 | + zstd --blah 2 | Incorrect parameter: --blah 3 | ... 4 | Usage: zstd * 5 | 6 | Options: 7 | ... 8 | + zstd -xz 9 | Incorrect parameter: -x 10 | ... 11 | Usage: zstd * 12 | 13 | Options: 14 | ... 15 | + zstd --adapt=min=1,maxx=2 file.txt 16 | Incorrect parameter: --adapt=min=1,maxx=2 17 | ... 18 | Usage: zstd * 19 | 20 | Options: 21 | ... 22 | + zstd --train-cover=k=48,d=8,steps32 file.txt 23 | Incorrect parameter: --train-cover=k=48,d=8,steps32 24 | ... 25 | Usage: zstd * 26 | 27 | Options: 28 | ... 29 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/help.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | println "+ zstd -h" 6 | zstd -h 7 | println "+ zstd -H" 8 | zstd -H 9 | println "+ zstd --help" 10 | zstd --help 11 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/help.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | + zstd -h 2 | Compress or decompress the INPUT file(s); reads from STDIN if INPUT is `-` or not provided. 3 | 4 | Usage: zstd *OPTIONS...* *INPUT... | -* *-o OUTPUT* 5 | 6 | Options: 7 | -o OUTPUT Write output to a single file, OUTPUT. 8 | -k, --keep Preserve INPUT file(s). *Default* 9 | --rm Remove INPUT file(s) after successful (de)compression. 10 | 11 | -# Desired compression level, where `#` is a number between 1 and 19; 12 | lower numbers provide faster compression, higher numbers yield 13 | better compression ratios. *Default: 3* 14 | 15 | -d, --decompress Perform decompression. 16 | -D DICT Use DICT as the dictionary for compression or decompression. 17 | 18 | -f, --force Disable input and output checks. Allows overwriting existing files, 19 | receiving input from the console, printing output to STDOUT, and 20 | operating on links, block devices, etc. Unrecognized formats will be 21 | passed-through through as-is. 22 | 23 | -h Display short usage and exit. 24 | -H, --help Display full help and exit. 25 | -V, --version Display the program version and exit. 26 | 27 | + zstd -H 28 | ... 29 | Advanced options: 30 | ... 31 | + zstd --help 32 | ... 33 | Advanced options: 34 | ... 35 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/memlimit.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 2 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 3 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 4 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 5 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 6 | error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed 7 | Should allow numeric parameter without suffix 8 | Should allow numeric parameter with expected suffix 9 | Should allow numeric parameter with expected suffix 10 | Should allow numeric parameter with expected suffix 11 | Should allow numeric parameter with expected suffix 12 | Should allow numeric parameter with expected suffix 13 | Should allow numeric parameter with expected suffix 14 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/memlimit.sh.stdout.exact: -------------------------------------------------------------------------------- 1 | + zstd --memory=32LB file 2 | + zstd --memory=32LiB file 3 | + zstd --memory=32A file 4 | + zstd --memory=32r82347dn83 file 5 | + zstd --memory=32asbdf file 6 | + zstd --memory=hello file 7 | + zstd --memory=1 file 8 | + zstd --memory=1K file 9 | + zstd --memory=1KB file 10 | + zstd --memory=1KiB file 11 | + zstd --memory=1M file 12 | + zstd --memory=1MB file 13 | + zstd --memory=1MiB file 14 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/output_dir.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | println "+ zstd -r * --output-dir-mirror=\"\"" 4 | zstd -r * --output-dir-mirror="" && die "Should not allow empty output dir!" 5 | println "+ zstd -r * --output-dir-flat=\"\"" 6 | zstd -r * --output-dir-flat="" && die "Should not allow empty output dir!" 7 | exit 0 8 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/output_dir.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | error: output dir cannot be empty string (did you mean to pass '.' instead?) 2 | error: output dir cannot be empty string (did you mean to pass '.' instead?) 3 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/output_dir.sh.stdout.exact: -------------------------------------------------------------------------------- 1 | + zstd -r * --output-dir-mirror="" 2 | + zstd -r * --output-dir-flat="" 3 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | zstd -V 6 | zstd --version 7 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/basic/version.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | *** Zstandard CLI (*-bit) v1.*.*, by Yann Collet *** 2 | *** Zstandard CLI (*-bit) v1.*.*, by Yann Collet *** 3 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/bin/cmp_size: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | usage() 6 | { 7 | printf "USAGE:\n\t$0 [-eq|-ne|-lt|-le|-gt|-ge] FILE1 FILE2\n" 8 | } 9 | 10 | help() 11 | { 12 | printf "Small utility to compare file sizes without printing them with set -x.\n\n" 13 | usage 14 | } 15 | 16 | case "$1" in 17 | -h) help; exit 0 ;; 18 | --help) help; exit 0 ;; 19 | esac 20 | 21 | if ! test -f $2; then 22 | printf "FILE1='%b' is not a file\n\n" "$2" 23 | usage 24 | exit 1 25 | fi 26 | 27 | if ! test -f $3; then 28 | printf "FILE2='%b' is not a file\n\n" "$3" 29 | usage 30 | exit 1 31 | fi 32 | 33 | 34 | size1=$(wc -c < $2) 35 | size2=$(wc -c < $3) 36 | 37 | case "$1" in 38 | -eq) [ "$size1" -eq "$size2" ] ;; 39 | -ne) [ "$size1" -ne "$size2" ] ;; 40 | -lt) [ "$size1" -lt "$size2" ] ;; 41 | -le) [ "$size1" -le "$size2" ] ;; 42 | -gt) [ "$size1" -gt "$size2" ] ;; 43 | -ge) [ "$size1" -ge "$size2" ] ;; 44 | esac 45 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/bin/datagen: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | "$DATAGEN_BIN" $@ 4 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/bin/die: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | println "${*}" 1>&2 4 | exit 1 5 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/bin/println: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | printf '%b\n' "${*}" 3 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/bin/unzstd: -------------------------------------------------------------------------------- 1 | zstd -------------------------------------------------------------------------------- /zstd/tests/cli-tests/bin/zstd: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | zstdname=$(basename $0) 4 | 5 | if [ -z "$EXEC_PREFIX" ]; then 6 | "$ZSTD_SYMLINK_DIR/$zstdname" $@ 7 | else 8 | $EXEC_PREFIX "$ZSTD_SYMLINK_DIR/$zstdname" $@ 9 | fi 10 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/bin/zstdcat: -------------------------------------------------------------------------------- 1 | zstd -------------------------------------------------------------------------------- /zstd/tests/cli-tests/bin/zstdgrep: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | "$ZSTDGREP_BIN" $@ 3 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/bin/zstdless: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | "$ZSTDLESS_BIN" $@ 3 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/cltools/setup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | echo "1234" > file 6 | zstd file 7 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/cltools/zstdgrep.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | println "+ good path" 6 | zstdgrep "1234" file file.zst 7 | println "+ bad path" 8 | zstdgrep "1234" bad.zst 9 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/cltools/zstdgrep.sh.exit: -------------------------------------------------------------------------------- 1 | 1 -------------------------------------------------------------------------------- /zstd/tests/cli-tests/cltools/zstdgrep.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | zstd: can't stat bad.zst : No such file or directory -- ignored 2 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/cltools/zstdgrep.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | + good path 2 | file:1234 3 | file.zst:1234 4 | + bad path 5 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/cltools/zstdless.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | println "+ good path" 6 | zstdless file.zst 7 | println "+ pass parameters" 8 | zstdless -N file.zst # This parameter does not produce line #s when piped, but still serves to test that the flag went to less and not zstd 9 | println "+ bad path" 10 | zstdless bad.zst >&2 11 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/cltools/zstdless.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | zstd: can't stat bad.zst : No such file or directory -- ignored 2 | bad.zst: No such file or directory 3 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/cltools/zstdless.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | + good path 2 | 1234 3 | + pass parameters 4 | 1234 5 | + bad path 6 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/common/format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$COMMON/platform.sh" 4 | 5 | zstd_supports_format() 6 | { 7 | zstd -h | grep > $INTOVOID -- "--format=$1" 8 | } 9 | 10 | format_extension() 11 | { 12 | if [ "$1" = "zstd" ]; then 13 | printf "zst" 14 | elif [ "$1" = "gzip" ]; then 15 | printf "gz" 16 | else 17 | printf "$1" 18 | fi 19 | } 20 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/common/mtime.sh: -------------------------------------------------------------------------------- 1 | . "$COMMON/platform.sh" 2 | 3 | MTIME="stat -c %Y" 4 | case "$UNAME" in 5 | Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;; 6 | esac 7 | 8 | assertSameMTime() { 9 | MT1=$($MTIME "$1") 10 | MT2=$($MTIME "$2") 11 | echo MTIME $MT1 $MT2 12 | [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)" 13 | } 14 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/common/permissions.sh: -------------------------------------------------------------------------------- 1 | . "$COMMON/platform.sh" 2 | 3 | GET_PERMS="stat -c %a" 4 | case "$UNAME" in 5 | Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;; 6 | esac 7 | 8 | assertFilePermissions() { 9 | STAT1=$($GET_PERMS "$1") 10 | STAT2=$2 11 | [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)" 12 | } 13 | 14 | assertSamePermissions() { 15 | STAT1=$($GET_PERMS "$1") 16 | STAT2=$($GET_PERMS "$2") 17 | [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)" 18 | } 19 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/common/platform.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | UNAME=$(uname) 4 | 5 | isWindows=false 6 | INTOVOID="/dev/null" 7 | case "$UNAME" in 8 | GNU) DEVDEVICE="/dev/random" ;; 9 | *) DEVDEVICE="/dev/zero" ;; 10 | esac 11 | case "$OS" in 12 | Windows*) 13 | isWindows=true 14 | INTOVOID="NUL" 15 | DEVDEVICE="NUL" 16 | ;; 17 | esac 18 | 19 | case "$UNAME" in 20 | Darwin) MD5SUM="md5 -r" ;; 21 | FreeBSD) MD5SUM="gmd5sum" ;; 22 | NetBSD) MD5SUM="md5 -n" ;; 23 | OpenBSD) MD5SUM="md5" ;; 24 | *) MD5SUM="md5sum" ;; 25 | esac 26 | 27 | DIFF="diff" 28 | case "$UNAME" in 29 | SunOS) DIFF="gdiff" ;; 30 | esac 31 | 32 | if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled' 33 | then 34 | hasMT="" 35 | else 36 | hasMT="true" 37 | fi 38 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/adapt.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test --adapt 6 | zstd -f file --adapt -c | zstd -t 7 | 8 | datagen -g100M > file100M 9 | 10 | # Pick parameters to force fast adaptation, even on slow systems 11 | zstd --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression" 12 | 13 | # Adaption still happens with --no-progress 14 | zstd --no-progress --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression" 15 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/basic.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Uncomment the set -v line for debugging 6 | # set -v 7 | 8 | # Test compression flags and check that they work 9 | zstd file ; zstd -t file.zst 10 | zstd -f file ; zstd -t file.zst 11 | zstd -f -z file ; zstd -t file.zst 12 | zstd -f -k file ; zstd -t file.zst 13 | zstd -f -C file ; zstd -t file.zst 14 | zstd -f --check file ; zstd -t file.zst 15 | zstd -f --no-check file ; zstd -t file.zst 16 | zstd -f -- file ; zstd -t file.zst 17 | 18 | # Test output file compression 19 | zstd -o file-out.zst ; zstd -t file-out.zst 20 | zstd -fo file-out.zst; zstd -t file-out.zst 21 | 22 | # Test compression to stdout 23 | zstd -c file | zstd -t 24 | zstd --stdout file | zstd -t 25 | println bob | zstd | zstd -t 26 | 27 | # Test keeping input file when compressing to stdout in gzip mode 28 | if $(command -v $ZSTD_SYMLINK_DIR/gzip); then 29 | $ZSTD_SYMLINK_DIR/gzip -c file | zstd -t ; test -f file 30 | $ZSTD_SYMLINK_DIR/gzip --stdout file | zstd -t ; test -f file 31 | fi 32 | 33 | # Test --rm 34 | cp file file-rm 35 | zstd --rm file-rm; zstd -t file-rm.zst 36 | test ! -f file-rm 37 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/compress-literals.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test --[no-]compress-literals 6 | zstd file --no-compress-literals -1 -c | zstd -t 7 | zstd file --no-compress-literals -19 -c | zstd -t 8 | zstd file --no-compress-literals --fast=1 -c | zstd -t 9 | zstd file --compress-literals -1 -c | zstd -t 10 | zstd file --compress-literals --fast=1 -c | zstd -t 11 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/format.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$COMMON/format.sh" 4 | 5 | set -e 6 | 7 | # Test --format 8 | zstd --format=zstd file -f 9 | zstd -t file.zst 10 | for format in "gzip" "lz4" "xz" "lzma"; do 11 | if zstd_supports_format $format; then 12 | zstd --format=$format file 13 | zstd -t file.$(format_extension $format) 14 | zstd -c --format=$format file | zstd -t --format=$format 15 | fi 16 | done 17 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/golden.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-compression/" 6 | cp -r "$GOLDEN_DIR" golden/ 7 | 8 | zstd -rf golden/ --output-dir-mirror golden-compressed/ 9 | zstd -r -t golden-compressed/ 10 | 11 | zstd --target-compressed-block-size=1024 -rf golden/ --output-dir-mirror golden-compressed/ 12 | zstd -r -t golden-compressed/ 13 | 14 | # PR #3517 block splitter corruption test 15 | zstd -rf -19 --zstd=mml=7 golden/ --output-dir-mirror golden-compressed/ 16 | zstd -r -t golden-compressed/ -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/gzip-compat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Uncomment the set -v line for debugging 6 | # set -v 7 | 8 | # Test gzip specific compression option 9 | if $(command -v $ZSTD_SYMLINK_DIR/gzip); then 10 | $ZSTD_SYMLINK_DIR/gzip --fast file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz 11 | $ZSTD_SYMLINK_DIR/gzip --best file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz 12 | 13 | # Test -n / --no-name: do not embed original filename in archive 14 | $ZSTD_SYMLINK_DIR/gzip -n file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz 15 | $ZSTD_SYMLINK_DIR/gzip --no-name file ; grep -qv file file.gz ; $ZSTD_SYMLINK_DIR/gzip -d file.gz 16 | $ZSTD_SYMLINK_DIR/gzip -c --no-name file | grep -qv file 17 | fi 18 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/long-distance-matcher.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test --long 6 | zstd -f file --long ; zstd -t file.zst 7 | zstd -f file --long=20; zstd -t file.zst 8 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/multi-threaded.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test multi-threaded flags 6 | zstd --single-thread file -f -q ; zstd -t file.zst 7 | zstd -T2 -f file -q ; zstd -t file.zst 8 | zstd --rsyncable -f file -q ; zstd -t file.zst 9 | zstd -T0 -f file -q ; zstd -t file.zst 10 | zstd -T0 --auto-threads=logical -f file -q ; zstd -t file.zst 11 | zstd -T0 --auto-threads=physical -f file -q ; zstd -t file.zst 12 | 13 | # multi-thread decompression warning test 14 | zstd -T0 -f file -q ; zstd -t file.zst; zstd -T0 -d file.zst -o file3 15 | zstd -T0 -f file -q ; zstd -t file.zst; zstd -T2 -d file.zst -o file4 16 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/multi-threaded.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | file.zst : 65537 bytes 2 | file.zst : 65537 bytes 3 | file.zst : 65537 bytes 4 | file.zst : 65537 bytes 5 | file.zst : 65537 bytes 6 | file.zst : 65537 bytes 7 | file.zst : 65537 bytes 8 | file.zst : 65537 bytes 9 | file.zst : 65537 bytes 10 | Warning : decompression does not support multi-threading 11 | file.zst : 65537 bytes 12 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/multiple-files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # setup 5 | echo "file1" > file1 6 | echo "file2" > file2 7 | 8 | echo "Test zstd ./file1 - file2" 9 | rm -f ./file*.zst 10 | echo "stdin" | zstd ./file1 - ./file2 | zstd -d 11 | cat file1.zst | zstd -d 12 | cat file2.zst | zstd -d 13 | 14 | echo "Test zstd -d ./file1.zst - file2.zst" 15 | rm ./file1 ./file2 16 | echo "stdin" | zstd - | zstd -d ./file1.zst - file2.zst 17 | cat file1 18 | cat file2 19 | 20 | echo "zstd -d ./file1.zst - file2.zst -c" 21 | echo "stdin" | zstd | zstd -d ./file1.zst - file2.zst -c 22 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/multiple-files.sh.stdout.exact: -------------------------------------------------------------------------------- 1 | Test zstd ./file1 - file2 2 | stdin 3 | file1 4 | file2 5 | Test zstd -d ./file1.zst - file2.zst 6 | stdin 7 | file1 8 | file2 9 | zstd -d ./file1.zst - file2.zst -c 10 | file1 11 | stdin 12 | file2 13 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/row-match-finder.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test --[no-]row-match-finder 6 | zstd file -7f --row-match-finder 7 | zstd file -7f --no-row-match-finder 8 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/setup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen > file 6 | datagen > file0 7 | datagen > file1 8 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/stream-size.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | # Test stream size & hint 6 | datagen -g7654 | zstd --stream-size=7654 | zstd -t 7 | datagen -g7654 | zstd --size-hint=7000 | zstd -t 8 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/verbose-wlog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | . "$COMMON/platform.sh" 6 | 7 | zstd < file -vv -19 -o file.19.zst 8 | zstd -vv -l file.19.zst 9 | 10 | zstd < file -vv -19 --long -o file.19.long.zst 11 | zstd -vv -l file.19.long.zst 12 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/verbose-wlog.sh.stderr.glob: -------------------------------------------------------------------------------- 1 | ... 2 | *wlog=23* 3 | ... 4 | *wlog=27* 5 | ... 6 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/verbose-wlog.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | ... 2 | *Window Size: 8388608 B* 3 | ... 4 | *Window Size: 134217728 B* 5 | ... 6 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/window-resize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | datagen -g1G > file 3 | zstd --long=30 -1 --single-thread --no-content-size -f file 4 | zstd -l -v file.zst 5 | 6 | # We want to ignore stderr (its outputting "*** zstd command line interface 7 | # 64-bits v1.5.3, by Yann Collet ***") 8 | 9 | rm file file.zst 10 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/window-resize.sh.stderr.ignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/cli-tests/compression/window-resize.sh.stderr.ignore -------------------------------------------------------------------------------- /zstd/tests/cli-tests/compression/window-resize.sh.stdout.glob: -------------------------------------------------------------------------------- 1 | ... 2 | Window Size: 1.000 GiB (1073741824 B) 3 | ... 4 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/decompression/detectErrors.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression-errors/" 6 | 7 | for file in "$GOLDEN_DIR"/*; do 8 | zstd -t $file && die "should have detected an error" 9 | done 10 | exit 0 11 | 12 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/decompression/golden.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-decompression/" 6 | 7 | zstd -r -t "$GOLDEN_DIR" 8 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/decompression/pass-through.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | . "$COMMON/platform.sh" 6 | 7 | echo "" > 1 8 | echo "2" > 2 9 | echo "23" > 3 10 | echo "234" > 4 11 | echo "some data" > file 12 | 13 | println "+ passthrough enabled" 14 | 15 | zstd file 16 | 17 | # Test short files 18 | zstd -dc --pass-through 1 2 3 4 19 | 20 | # Test *cat symlinks 21 | zstdcat file 22 | "$ZSTD_SYMLINK_DIR/zcat" file 23 | "$ZSTD_SYMLINK_DIR/gzcat" file 24 | 25 | # Test multiple files with mix of compressed & not 26 | zstdcat file file.zst 27 | zstdcat file.zst file 28 | 29 | # Test --pass-through 30 | zstd -dc --pass-through file 31 | zstd -d --pass-through file -o pass-through-file 32 | 33 | # Test legacy implicit passthrough with -fc 34 | zstd -dcf file 35 | zstd -dcf file file.zst 36 | zstd -df < file 37 | zstd -dcf < file file.zst - 38 | zstd -dcf < file.zst file - 39 | 40 | $DIFF file pass-through-file 41 | 42 | println "+ passthrough disabled" 43 | 44 | # Test *cat 45 | zstdcat --no-pass-through file && die "should fail" 46 | "$ZSTD_SYMLINK_DIR/zcat" --no-pass-through file && die "should fail" 47 | "$ZSTD_SYMLINK_DIR/gzcat" --no-pass-through file && die "should fail" 48 | # Test zstd without implicit passthrough 49 | zstd -d file -o no-pass-through-file && die "should fail" 50 | zstd -d < file && die "should fail" 51 | 52 | # Test legacy implicit passthrough with -fc 53 | zstd --no-pass-through -dcf file && die "should fail" 54 | zstd --no-pass-through -dcf file file.zst && die "should fail" 55 | zstd --no-pass-through -df < file && die "should fail" 56 | zstd --no-pass-through -dcf < file file.zst - && die "should fail" 57 | zstd --no-pass-through -dcf < file.zst file - && die "should fail" ||: 58 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/decompression/pass-through.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | file :230.00% ( 10 B => 23 B, file.zst) 2 | zstd: file: unsupported format 3 | zstd: file: unsupported format 4 | zstd: file: unsupported format 5 | zstd: file: unsupported format 6 | zstd: /*stdin*\: unsupported format 7 | zstd: file: unsupported format 8 | zstd: file: unsupported format 9 | zstd: /*stdin*\: unsupported format 10 | zstd: /*stdin*\: unsupported format 11 | zstd: file: unsupported format 12 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/decompression/pass-through.sh.stdout.exact: -------------------------------------------------------------------------------- 1 | + passthrough enabled 2 | 3 | 2 4 | 23 5 | 234 6 | some data 7 | some data 8 | some data 9 | some data 10 | some data 11 | some data 12 | some data 13 | some data 14 | some data 15 | some data 16 | some data 17 | some data 18 | some data 19 | some data 20 | some data 21 | some data 22 | + passthrough disabled 23 | some data 24 | some data 25 | some data 26 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/dict-builder/empty-input.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | for i in $(seq 50); do 4 | datagen -s$i > file$i 5 | done 6 | touch empty 7 | 8 | set -v 9 | zstd -q --train empty file* 10 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/dict-builder/empty-input.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | zstd -q --train empty file* 2 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/dict-builder/no-inputs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -v 3 | zstd --train 4 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/dict-builder/no-inputs.sh.exit: -------------------------------------------------------------------------------- 1 | 14 2 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | zstd --train 2 | ! Warning : nb of samples too low for proper processing ! 3 | ! Please provide _one file per sample_. 4 | ! Alternatively, split files into fixed-size blocks representative of samples, with -B# 5 | Error 14 : nb of samples too low 6 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$COMMON/platform.sh" 4 | 5 | set -e 6 | 7 | if [ false ]; then 8 | for seed in $(seq 100); do 9 | datagen -g1000 -s$seed > file$seed 10 | done 11 | 12 | zstd --train -r . -o dict0 -qq 13 | 14 | for seed in $(seq 101 200); do 15 | datagen -g1000 -s$seed > file$seed 16 | done 17 | 18 | zstd --train -r . -o dict1 -qq 19 | 20 | [ "$($MD5SUM < dict0)" != "$($MD5SUM < dict1)" ] || die "dictionaries must not match" 21 | 22 | datagen -g1000 -s0 > file0 23 | fi 24 | 25 | set -v 26 | zstd files/0 -D dicts/0 -q 27 | zstd -t files/0.zst -D dicts/0 28 | zstd -t files/0.zst -D dicts/1 && die "Must fail" ||: 29 | zstd -t files/0.zst && die "Must fail" ||: 30 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | zstd files/0 -D dicts/0 -q 2 | zstd -t files/0.zst -D dicts/0 3 | files/0.zst : 1000 bytes 4 | zstd -t files/0.zst -D dicts/1 && die "Must fail" ||: 5 | files/0.zst : Decoding error (36) : Dictionary mismatch 6 | zstd -t files/0.zst && die "Must fail" ||: 7 | files/0.zst : Decoding error (36) : Dictionary mismatch 8 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/dictionaries/golden.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | GOLDEN_COMP_DIR="$ZSTD_REPO_DIR/tests/golden-compression/" 6 | GOLDEN_DICT_DIR="$ZSTD_REPO_DIR/tests/golden-dictionaries/" 7 | 8 | zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" "$GOLDEN_COMP_DIR/http" -o http.zst 9 | zstd -D "$GOLDEN_DICT_DIR/http-dict-missing-symbols" -t http.zst 10 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/dictionaries/setup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cp -r ../files . 6 | cp -r ../dicts . 7 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/dictionaries/setup_once: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | . "$COMMON/platform.sh" 6 | 7 | 8 | mkdir files/ dicts/ 9 | 10 | for seed in $(seq 50); do 11 | datagen -g1000 -s$seed > files/$seed 12 | done 13 | 14 | zstd --train -r files -o dicts/0 -qq 15 | 16 | for seed in $(seq 51 100); do 17 | datagen -g1000 -s$seed > files/$seed 18 | done 19 | 20 | zstd --train -r files -o dicts/1 -qq 21 | 22 | cmp dicts/0 dicts/1 && die "dictionaries must not match!" 23 | 24 | datagen -g1000 > files/0 25 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-handling/directory-mirror.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # setup 5 | mkdir -p src/.hidden src/dir 6 | mkdir mid dst 7 | 8 | echo "file1" > src/file1 9 | echo "file2" > src/.file2 10 | echo "file3" > src/.hidden/.file3 11 | echo "file4" > src/dir/.file4 12 | 13 | # relative paths 14 | zstd -q -r --output-dir-mirror mid/ src/ 15 | zstd -q -d -r --output-dir-mirror dst/ mid/src/ 16 | 17 | diff --brief --recursive --new-file src/ dst/mid/src/ 18 | 19 | # reset 20 | rm -rf mid dst 21 | mkdir mid dst 22 | 23 | # from inside the directory 24 | (cd src; zstd -q -r --output-dir-mirror ../mid/ ./) 25 | (cd mid; zstd -q -d -r --output-dir-mirror ../dst/ ./) 26 | 27 | diff --brief --recursive --new-file src/ dst/ 28 | 29 | # reset 30 | rm -rf mid dst 31 | mkdir mid dst 32 | 33 | # absolute paths 34 | export BASE_PATH="$(pwd)" 35 | 36 | zstd -q -r --output-dir-mirror mid/ "${BASE_PATH}/src/" 37 | zstd -q -d -r --output-dir-mirror dst/ "${BASE_PATH}/mid/${BASE_PATH}/src/" 38 | 39 | diff --brief --recursive --new-file src/ "dst/${BASE_PATH}/mid/${BASE_PATH}/src/" 40 | 41 | # reset 42 | rm -rf mid dst 43 | mkdir mid dst 44 | 45 | # dots 46 | zstd -q -r --output-dir-mirror mid/ ./src/./ 47 | zstd -q -d -r --output-dir-mirror dst/ ./mid/./src/./ 48 | 49 | diff --brief --recursive --new-file src/ dst/mid/src/ 50 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-handling/directory-mirror.sh.stderr.exact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/cli-tests/file-handling/directory-mirror.sh.stderr.exact -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-handling/directory-mirror.sh.stdout.exact: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/cli-tests/file-handling/directory-mirror.sh.stdout.exact -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # motivated by issue #3523 4 | 5 | datagen > file 6 | mkdir out 7 | chmod 000 out 8 | 9 | zstd file -q --trace-file-stat -o out/file.zst 10 | zstd -tq out/file.zst 11 | 12 | chmod 777 out 13 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/compress-file-to-dir-without-write-perm.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isLink(file) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(2) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_getFileSize(file) 6 | Trace:FileStat: > UTIL_stat(-1, file) 7 | Trace:FileStat: < 1 8 | Trace:FileStat: < 65537 9 | Trace:FileStat: > UTIL_stat(-1, file) 10 | Trace:FileStat: < 1 11 | Trace:FileStat: > UTIL_isDirectoryStat() 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_stat(-1, file) 14 | Trace:FileStat: < 1 15 | Trace:FileStat: > UTIL_isSameFile(file, out/file.zst) 16 | Trace:FileStat: > UTIL_stat(-1, file) 17 | Trace:FileStat: < 1 18 | Trace:FileStat: > UTIL_stat(-1, out/file.zst) 19 | Trace:FileStat: < 0 20 | Trace:FileStat: < 0 21 | Trace:FileStat: > UTIL_isRegularFile(out/file.zst) 22 | Trace:FileStat: > UTIL_stat(-1, out/file.zst) 23 | Trace:FileStat: < 0 24 | Trace:FileStat: < 0 25 | zstd: out/file.zst: Permission denied 26 | zstd: can't stat out/file.zst : Permission denied -- ignored 27 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/compress-file-to-file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen > file 6 | chmod 642 file 7 | 8 | zstd file -q --trace-file-stat -o file.zst 9 | zstd -tq file.zst 10 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/compress-file-to-file.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isLink(file) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(2) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_getFileSize(file) 6 | Trace:FileStat: > UTIL_stat(-1, file) 7 | Trace:FileStat: < 1 8 | Trace:FileStat: < 65537 9 | Trace:FileStat: > UTIL_stat(-1, file) 10 | Trace:FileStat: < 1 11 | Trace:FileStat: > UTIL_isDirectoryStat() 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_stat(-1, file) 14 | Trace:FileStat: < 1 15 | Trace:FileStat: > UTIL_isSameFile(file, file.zst) 16 | Trace:FileStat: > UTIL_stat(-1, file) 17 | Trace:FileStat: < 1 18 | Trace:FileStat: > UTIL_stat(-1, file.zst) 19 | Trace:FileStat: < 0 20 | Trace:FileStat: < 0 21 | Trace:FileStat: > UTIL_isRegularFile(file.zst) 22 | Trace:FileStat: > UTIL_stat(-1, file.zst) 23 | Trace:FileStat: < 0 24 | Trace:FileStat: < 0 25 | Trace:FileStat: > UTIL_isRegularFile(file.zst) 26 | Trace:FileStat: > UTIL_stat(-1, file.zst) 27 | Trace:FileStat: < 1 28 | Trace:FileStat: < 1 29 | Trace:FileStat: > UTIL_getFileSize(file) 30 | Trace:FileStat: > UTIL_stat(-1, file) 31 | Trace:FileStat: < 1 32 | Trace:FileStat: < 65537 33 | Trace:FileStat: > UTIL_setFileStat(4, file.zst) 34 | Trace:FileStat: > UTIL_stat(4, file.zst) 35 | Trace:FileStat: < 1 36 | Trace:FileStat: > UTIL_chmod(file.zst, 0642) 37 | Trace:FileStat: > fchmod 38 | Trace:FileStat: < 0 39 | Trace:FileStat: < 0 40 | Trace:FileStat: < 0 41 | Trace:FileStat: > UTIL_utime(file.zst) 42 | Trace:FileStat: < 0 43 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen > file 6 | 7 | zstd file -cq --trace-file-stat > file.zst 8 | zstd -tq file.zst 9 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/compress-file-to-stdout.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isLink(file) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(1) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isConsole(2) 6 | Trace:FileStat: < 0 7 | Trace:FileStat: > UTIL_getFileSize(file) 8 | Trace:FileStat: > UTIL_stat(-1, file) 9 | Trace:FileStat: < 1 10 | Trace:FileStat: < 65537 11 | Trace:FileStat: > UTIL_stat(-1, file) 12 | Trace:FileStat: < 1 13 | Trace:FileStat: > UTIL_isDirectoryStat() 14 | Trace:FileStat: < 0 15 | Trace:FileStat: > UTIL_stat(-1, file) 16 | Trace:FileStat: < 1 17 | Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) 18 | Trace:FileStat: > UTIL_stat(-1, /*stdout*\) 19 | Trace:FileStat: < 0 20 | Trace:FileStat: < 0 21 | Trace:FileStat: > UTIL_getFileSize(file) 22 | Trace:FileStat: > UTIL_stat(-1, file) 23 | Trace:FileStat: < 1 24 | Trace:FileStat: < 65537 25 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen > file 6 | 7 | zstd < file -q --trace-file-stat -o file.zst 8 | zstd -tq file.zst 9 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/compress-stdin-to-file.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isConsole(0) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(2) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_getFileSize(/*stdin*\) 6 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 7 | Trace:FileStat: < 0 8 | Trace:FileStat: < -1 9 | Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file.zst) 10 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 11 | Trace:FileStat: < 0 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_isRegularFile(file.zst) 14 | Trace:FileStat: > UTIL_stat(-1, file.zst) 15 | Trace:FileStat: < 0 16 | Trace:FileStat: < 0 17 | Trace:FileStat: > UTIL_isRegularFile(file.zst) 18 | Trace:FileStat: > UTIL_stat(-1, file.zst) 19 | Trace:FileStat: < 1 20 | Trace:FileStat: < 1 21 | Trace:FileStat: > UTIL_getFileSize(/*stdin*\) 22 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 23 | Trace:FileStat: < 0 24 | Trace:FileStat: < -1 25 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen > file 6 | 7 | zstd < file -cq --trace-file-stat > file.zst 8 | zstd -tq file.zst 9 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/compress-stdin-to-stdout.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isConsole(0) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(1) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isConsole(2) 6 | Trace:FileStat: < 0 7 | Trace:FileStat: > UTIL_getFileSize(/*stdin*\) 8 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 9 | Trace:FileStat: < 0 10 | Trace:FileStat: < -1 11 | Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) 12 | Trace:FileStat: > UTIL_stat(-1, /*stdout*\) 13 | Trace:FileStat: < 0 14 | Trace:FileStat: < 0 15 | Trace:FileStat: > UTIL_getFileSize(/*stdin*\) 16 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 17 | Trace:FileStat: < 0 18 | Trace:FileStat: < -1 19 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen | zstd -q > file.zst 6 | chmod 642 file.zst 7 | 8 | zstd -dq --trace-file-stat file.zst 9 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/decompress-file-to-file.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isLink(file.zst) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(1) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isConsole(2) 6 | Trace:FileStat: < 0 7 | Trace:FileStat: > UTIL_isDirectory(file.zst) 8 | Trace:FileStat: > UTIL_stat(-1, file.zst) 9 | Trace:FileStat: < 1 10 | Trace:FileStat: > UTIL_isDirectoryStat() 11 | Trace:FileStat: < 0 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_stat(-1, file.zst) 14 | Trace:FileStat: < 1 15 | Trace:FileStat: > UTIL_isSameFile(file.zst, file) 16 | Trace:FileStat: > UTIL_stat(-1, file.zst) 17 | Trace:FileStat: < 1 18 | Trace:FileStat: > UTIL_stat(-1, file) 19 | Trace:FileStat: < 0 20 | Trace:FileStat: < 0 21 | Trace:FileStat: > UTIL_isRegularFile(file) 22 | Trace:FileStat: > UTIL_stat(-1, file) 23 | Trace:FileStat: < 0 24 | Trace:FileStat: < 0 25 | Trace:FileStat: > UTIL_isRegularFile(file) 26 | Trace:FileStat: > UTIL_stat(-1, file) 27 | Trace:FileStat: < 1 28 | Trace:FileStat: < 1 29 | Trace:FileStat: > UTIL_setFileStat(4, file) 30 | Trace:FileStat: > UTIL_stat(4, file) 31 | Trace:FileStat: < 1 32 | Trace:FileStat: > UTIL_chmod(file, 0642) 33 | Trace:FileStat: > fchmod 34 | Trace:FileStat: < 0 35 | Trace:FileStat: < 0 36 | Trace:FileStat: < 0 37 | Trace:FileStat: > UTIL_utime(file) 38 | Trace:FileStat: < 0 39 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen | zstd -q > file.zst 6 | 7 | zstd -dcq --trace-file-stat file.zst > file 8 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/decompress-file-to-stdout.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isLink(file.zst) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(1) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isConsole(2) 6 | Trace:FileStat: < 0 7 | Trace:FileStat: > UTIL_isDirectory(file.zst) 8 | Trace:FileStat: > UTIL_stat(-1, file.zst) 9 | Trace:FileStat: < 1 10 | Trace:FileStat: > UTIL_isDirectoryStat() 11 | Trace:FileStat: < 0 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_stat(-1, file.zst) 14 | Trace:FileStat: < 1 15 | Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) 16 | Trace:FileStat: > UTIL_stat(-1, /*stdout*\) 17 | Trace:FileStat: < 0 18 | Trace:FileStat: < 0 19 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen | zstd -q > file.zst 6 | 7 | zstd -dcq --trace-file-stat < file.zst -o file 8 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/decompress-stdin-to-file.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isConsole(0) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(2) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isDirectory(/*stdin*\) 6 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 7 | Trace:FileStat: < 0 8 | Trace:FileStat: < 0 9 | Trace:FileStat: > UTIL_isSameFile(/*stdin*\, file) 10 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 11 | Trace:FileStat: < 0 12 | Trace:FileStat: < 0 13 | Trace:FileStat: > UTIL_isRegularFile(file) 14 | Trace:FileStat: > UTIL_stat(-1, file) 15 | Trace:FileStat: < 0 16 | Trace:FileStat: < 0 17 | Trace:FileStat: > UTIL_isRegularFile(file) 18 | Trace:FileStat: > UTIL_stat(-1, file) 19 | Trace:FileStat: < 1 20 | Trace:FileStat: < 1 21 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | datagen | zstd -q > file.zst 6 | 7 | zstd -dcq --trace-file-stat < file.zst > file 8 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/file-stat/decompress-stdin-to-stdout.sh.stderr.exact: -------------------------------------------------------------------------------- 1 | Trace:FileStat: > UTIL_isConsole(0) 2 | Trace:FileStat: < 0 3 | Trace:FileStat: > UTIL_isConsole(1) 4 | Trace:FileStat: < 0 5 | Trace:FileStat: > UTIL_isConsole(2) 6 | Trace:FileStat: < 0 7 | Trace:FileStat: > UTIL_isDirectory(/*stdin*\) 8 | Trace:FileStat: > UTIL_stat(-1, /*stdin*\) 9 | Trace:FileStat: < 0 10 | Trace:FileStat: < 0 11 | Trace:FileStat: > UTIL_isRegularFile(/*stdout*\) 12 | Trace:FileStat: > UTIL_stat(-1, /*stdout*\) 13 | Trace:FileStat: < 0 14 | Trace:FileStat: < 0 15 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/progress/no-progress.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #!/bin/sh 4 | 5 | . "$COMMON/platform.sh" 6 | 7 | set -e 8 | 9 | echo hello > hello 10 | echo world > world 11 | 12 | zstd -q hello world 13 | 14 | println >&2 "Tests cases where progress information should not be printed" 15 | 16 | for args in \ 17 | "" \ 18 | "--fake-stderr-is-console -q" \ 19 | "--fake-stderr-is-console -qq --progress" \ 20 | "--no-progress --fake-stderr-is-console" \ 21 | "--no-progress --fake-stderr-is-console -v" 22 | do 23 | println >&2 "args = $args" 24 | println >&2 "compress file to file" 25 | zstd $args -f hello 26 | println >&2 "compress pipe to pipe" 27 | zstd $args < hello > $INTOVOID 28 | println >&2 "compress pipe to file" 29 | zstd $args < hello -fo hello.zst 30 | println >&2 "compress file to pipe" 31 | zstd $args hello -c > $INTOVOID 32 | println >&2 "compress 2 files" 33 | zstd $args -f hello world 34 | 35 | println >&2 "decompress file to file" 36 | zstd $args -d -f hello.zst 37 | println >&2 "decompress pipe to pipe" 38 | zstd $args -d < hello.zst > $INTOVOID 39 | println >&2 "decompress pipe to file" 40 | zstd $args -d < hello.zst -fo hello 41 | println >&2 "decompress file to pipe" 42 | zstd $args -d hello.zst -c > $INTOVOID 43 | println >&2 "decompress 2 files" 44 | zstd $args -d -f hello.zst world.zst 45 | println >&2 "" 46 | done 47 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/progress/progress.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | . "$COMMON/platform.sh" 4 | 5 | set -e 6 | 7 | println >&2 "Tests cases where progress information should be printed" 8 | 9 | echo hello > hello 10 | echo world > world 11 | 12 | zstd -q hello world 13 | 14 | for args in \ 15 | "--progress" \ 16 | "--fake-stderr-is-console" \ 17 | "--progress --fake-stderr-is-console -q"; do 18 | println >&2 "args = $args" 19 | println >&2 "compress file to file" 20 | zstd $args -f hello 21 | println >&2 "compress pipe to pipe" 22 | zstd $args < hello > $INTOVOID 23 | println >&2 "compress pipe to file" 24 | zstd $args < hello -fo hello.zst 25 | println >&2 "compress file to pipe" 26 | zstd $args hello -c > $INTOVOID 27 | println >&2 "compress 2 files" 28 | zstd $args -f hello world 29 | 30 | println >&2 "decompress file to file" 31 | zstd $args -d -f hello.zst 32 | println >&2 "decompress pipe to pipe" 33 | zstd $args -d < hello.zst > $INTOVOID 34 | println >&2 "decompress pipe to file" 35 | zstd $args -d < hello.zst -fo hello 36 | println >&2 "decompress file to pipe" 37 | zstd $args -d hello.zst -c > $INTOVOID 38 | println >&2 "decompress 2 files" 39 | zstd $args -d -f hello.zst world.zst 40 | println >&2 "" 41 | done 42 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/progress/progress.sh.stderr.glob: -------------------------------------------------------------------------------- 1 | Tests cases where progress information should be printed 2 | args = --progress 3 | compress file to file 4 | *Read:*hello*hello.zst* 5 | compress pipe to pipe 6 | *Read:*stdin*stdout* 7 | compress pipe to file 8 | *Read:*stdin*hello.zst* 9 | compress file to pipe 10 | *Read:*hello*stdout* 11 | compress 2 files 12 | *Read*2 files compressed* 13 | decompress file to file 14 | *hello.zst*hello.zst* 15 | decompress pipe to pipe 16 | *stdin*stdin* 17 | decompress pipe to file 18 | *stdin*stdin* 19 | decompress file to pipe 20 | *hello.zst*hello.zst* 21 | decompress 2 files 22 | *hello.zst*2 files decompressed* 23 | 24 | args = --fake-stderr-is-console 25 | compress file to file 26 | *Read:*hello*hello.zst* 27 | compress pipe to pipe 28 | compress pipe to file 29 | *Read:*stdin*hello.zst* 30 | compress file to pipe 31 | compress 2 files 32 | *Read*2 files compressed* 33 | decompress file to file 34 | *hello.zst*hello.zst* 35 | decompress pipe to pipe 36 | decompress pipe to file 37 | *stdin*stdin* 38 | decompress file to pipe 39 | decompress 2 files 40 | *hello.zst*2 files decompressed* 41 | 42 | args = --progress --fake-stderr-is-console -q 43 | compress file to file 44 | *Read:*hello*hello.zst* 45 | compress pipe to pipe 46 | *Read:*stdin*stdout* 47 | compress pipe to file 48 | *Read:*stdin*hello.zst* 49 | compress file to pipe 50 | *Read:*hello*stdout* 51 | compress 2 files 52 | *Read*2 files compressed* 53 | decompress file to file 54 | *hello.zst*hello.zst* 55 | decompress pipe to pipe 56 | *stdin*stdin* 57 | decompress pipe to file 58 | *stdin*stdin* 59 | decompress file to pipe 60 | *hello.zst*hello.zst* 61 | decompress 2 files 62 | *hello.zst*2 files decompressed* 63 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/zstd-symlinks/setup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | println "hello" > hello 5 | println "world" > world 6 | zstd hello world 7 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # Test zstdcat symlink in bin/ 5 | zstdcat hello.zst 6 | zstdcat hello.zst world 7 | zstdcat hello world.zst 8 | zstdcat hello.zst world.zst 9 | 10 | # Test local zstdcat symlink 11 | ln -s $(which zstd) ./zstdcat 12 | ./zstdcat hello.zst 13 | -------------------------------------------------------------------------------- /zstd/tests/cli-tests/zstd-symlinks/zstdcat.sh.stdout.exact: -------------------------------------------------------------------------------- 1 | hello 2 | hello 3 | world 4 | hello 5 | world 6 | hello 7 | world 8 | hello 9 | -------------------------------------------------------------------------------- /zstd/tests/dict-files/zero-weight-dict: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/dict-files/zero-weight-dict -------------------------------------------------------------------------------- /zstd/tests/external_matchfinder.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Yann Collet, Meta Platforms, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef EXTERNAL_MATCHFINDER 12 | #define EXTERNAL_MATCHFINDER 13 | 14 | #define ZSTD_STATIC_LINKING_ONLY 15 | #include "zstd.h" 16 | 17 | /* See external_matchfinder.c for details on each test case */ 18 | typedef enum { 19 | EMF_ZERO_SEQS = 0, 20 | EMF_ONE_BIG_SEQ = 1, 21 | EMF_LOTS_OF_SEQS = 2, 22 | EMF_BIG_ERROR = 3, 23 | EMF_SMALL_ERROR = 4, 24 | EMF_INVALID_OFFSET = 5, 25 | EMF_INVALID_MATCHLEN = 6, 26 | EMF_INVALID_LITLEN = 7, 27 | EMF_INVALID_LAST_LITS = 8 28 | } EMF_testCase; 29 | 30 | size_t zstreamSequenceProducer( 31 | void* sequenceProducerState, 32 | ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, 33 | const void* src, size_t srcSize, 34 | const void* dict, size_t dictSize, 35 | int compressionLevel, 36 | size_t windowSize 37 | ); 38 | 39 | #endif /* EXTERNAL_MATCHFINDER */ 40 | -------------------------------------------------------------------------------- /zstd/tests/fuzz/.gitignore: -------------------------------------------------------------------------------- 1 | # test artefacts 2 | corpora 3 | block_decompress 4 | block_round_trip 5 | dictionary_decompress 6 | dictionary_loader 7 | dictionary_round_trip 8 | dictionary_stream_round_trip 9 | raw_dictionary_round_trip 10 | simple_compress 11 | simple_decompress 12 | simple_round_trip 13 | stream_decompress 14 | stream_round_trip 15 | zstd_frame_info 16 | decompress_dstSize_tooSmall 17 | fse_read_ncount 18 | sequence_compression_api 19 | seekable_roundtrip 20 | huf_decompress 21 | huf_round_trip 22 | fuzz-*.log 23 | rt_lib_* 24 | d_lib_* 25 | crash-* 26 | 27 | # misc 28 | trace 29 | tmp* 30 | -------------------------------------------------------------------------------- /zstd/tests/fuzz/fuzz_helpers.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | #include "fuzz_helpers.h" 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | void* FUZZ_malloc(size_t size) 17 | { 18 | if (size > 0) { 19 | void* const mem = malloc(size); 20 | FUZZ_ASSERT(mem); 21 | return mem; 22 | } 23 | return NULL; 24 | } 25 | 26 | void* FUZZ_malloc_rand(size_t size, FUZZ_dataProducer_t *producer) 27 | { 28 | if (size > 0) { 29 | void* const mem = malloc(size); 30 | FUZZ_ASSERT(mem); 31 | return mem; 32 | } else { 33 | uintptr_t ptr = 0; 34 | /* Add +- 1M 50% of the time */ 35 | if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) 36 | FUZZ_dataProducer_int32Range(producer, -1000000, 1000000); 37 | return (void*)ptr; 38 | } 39 | 40 | } 41 | 42 | int FUZZ_memcmp(void const* lhs, void const* rhs, size_t size) 43 | { 44 | if (size == 0) { 45 | return 0; 46 | } 47 | return memcmp(lhs, rhs, size); 48 | } 49 | -------------------------------------------------------------------------------- /zstd/tests/fuzz/seq_prod_fuzz_example/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) Yann Collet, Meta Platforms, Inc. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under both the BSD-style license (found in the 5 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 6 | # in the COPYING file in the root directory of this source tree). 7 | # You may select, at your option, one of the above-listed licenses. 8 | 9 | CC = clang 10 | CFLAGS = -g -fno-omit-frame-pointer -fsanitize=undefined,address,fuzzer -I../ -I../../../lib/ 11 | 12 | .PHONY: default 13 | default: example_seq_prod.o 14 | 15 | example_seq_prod.o: example_seq_prod.c 16 | $(CC) -c $(CFLAGS) $^ -o $@ 17 | -------------------------------------------------------------------------------- /zstd/tests/fuzz/seq_prod_fuzz_example/README.md: -------------------------------------------------------------------------------- 1 | # Fuzzing a Custom Sequence Producer Plugin 2 | This directory contains example code for using a custom sequence producer in the zstd fuzzers. 3 | 4 | You can build and run the code in this directory using these commands: 5 | ``` 6 | $ make corpora 7 | $ make -C seq_prod_fuzz_example/ 8 | $ python3 ./fuzz.py build all --enable-fuzzer --enable-asan --enable-ubsan --cc clang --cxx clang++ --custom-seq-prod=seq_prod_fuzz_example/example_seq_prod.o 9 | $ python3 ./fuzz.py libfuzzer simple_round_trip 10 | ``` 11 | 12 | See `../fuzz_third_party_seq_prod.h` and `../README.md` for more information on zstd fuzzing. 13 | -------------------------------------------------------------------------------- /zstd/tests/fuzz/seq_prod_fuzz_example/example_seq_prod.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Yann Collet, Meta Platforms, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #include "fuzz_third_party_seq_prod.h" 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | _Thread_local size_t threadLocalState; 18 | 19 | size_t FUZZ_seqProdSetup(void) { 20 | threadLocalState = 0; 21 | return 0; 22 | } 23 | 24 | size_t FUZZ_seqProdTearDown(void) { 25 | return 0; 26 | } 27 | 28 | void* FUZZ_createSeqProdState(void) { 29 | return calloc(1, sizeof(size_t)); 30 | } 31 | 32 | size_t FUZZ_freeSeqProdState(void* state) { 33 | free(state); 34 | return 0; 35 | } 36 | 37 | size_t FUZZ_thirdPartySeqProd( 38 | void* sequenceProducerState, 39 | ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, 40 | const void* src, size_t srcSize, 41 | const void* dict, size_t dictSize, 42 | int compressionLevel, 43 | size_t windowSize 44 | ) { 45 | /* Try to catch unsafe use of the shared state */ 46 | size_t* const sharedStatePtr = (size_t*)sequenceProducerState; 47 | assert(*sharedStatePtr == threadLocalState); 48 | (*sharedStatePtr)++; threadLocalState++; 49 | 50 | /* Check that fallback is enabled when FUZZ_THIRD_PARTY_SEQ_PROD is defined */ 51 | return ZSTD_SEQUENCE_PRODUCER_ERROR; 52 | } 53 | -------------------------------------------------------------------------------- /zstd/tests/fuzz/zstd_frame_info.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /** 12 | * This fuzz target fuzzes all of the helper functions that consume compressed 13 | * input. 14 | */ 15 | 16 | #include 17 | #include 18 | #include 19 | #include "fuzz_helpers.h" 20 | #include "zstd_helpers.h" 21 | 22 | int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) 23 | { 24 | ZSTD_frameHeader zfh; 25 | if (size == 0) { 26 | src = NULL; 27 | } 28 | /* You can fuzz any helper functions here that are fast, and take zstd 29 | * compressed data as input. E.g. don't expect the input to be a dictionary, 30 | * so don't fuzz ZSTD_getDictID_fromDict(). 31 | */ 32 | ZSTD_getFrameContentSize(src, size); 33 | ZSTD_getDecompressedSize(src, size); 34 | ZSTD_findFrameCompressedSize(src, size); 35 | ZSTD_getDictID_fromFrame(src, size); 36 | ZSTD_findDecompressedSize(src, size); 37 | ZSTD_decompressBound(src, size); 38 | ZSTD_frameHeaderSize(src, size); 39 | ZSTD_isFrame(src, size); 40 | ZSTD_getFrameHeader(&zfh, src, size); 41 | ZSTD_getFrameHeader_advanced(&zfh, src, size, ZSTD_f_zstd1); 42 | return 0; 43 | } 44 | -------------------------------------------------------------------------------- /zstd/tests/golden-compression/PR-3517-block-splitter-corruption-test: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/golden-compression/PR-3517-block-splitter-corruption-test -------------------------------------------------------------------------------- /zstd/tests/golden-compression/http: -------------------------------------------------------------------------------- 1 | ads60.vertamedia.comhttp://ads60.vertamedia.com/ops/43C53990160C658B/46991http://ads60.vertamedia.com/ve/43C53990160C658B/53http://ads60.vertamedia.com/ve/43C53990160C658B/54http://ads60.vertamedia.com/ve/43C53990160C658B/55http://ads60.vertamedia.com/ve/43C53990160C658B/56http://ads60.vertamedia.com/ve/43C53990160C658B/57http://ads60.vertamedia.com/ve/43C53990160C658B/66http://ads60.vertamedia.com/ve/43C53990160C658B/71 -------------------------------------------------------------------------------- /zstd/tests/golden-compression/huffman-compressed-larger: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/golden-compression/huffman-compressed-larger -------------------------------------------------------------------------------- /zstd/tests/golden-compression/large-literal-and-match-lengths: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/golden-compression/large-literal-and-match-lengths -------------------------------------------------------------------------------- /zstd/tests/golden-decompression-errors/.gitignore: -------------------------------------------------------------------------------- 1 | !*.zst 2 | -------------------------------------------------------------------------------- /zstd/tests/golden-decompression-errors/off0.bin.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/golden-decompression-errors/off0.bin.zst -------------------------------------------------------------------------------- /zstd/tests/golden-decompression-errors/zeroSeq_extraneous.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/golden-decompression-errors/zeroSeq_extraneous.zst -------------------------------------------------------------------------------- /zstd/tests/golden-decompression/block-128k.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/golden-decompression/block-128k.zst -------------------------------------------------------------------------------- /zstd/tests/golden-decompression/empty-block.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/golden-decompression/empty-block.zst -------------------------------------------------------------------------------- /zstd/tests/golden-decompression/rle-first-block.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/golden-decompression/rle-first-block.zst -------------------------------------------------------------------------------- /zstd/tests/golden-decompression/zeroSeq_2B.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/golden-decompression/zeroSeq_2B.zst -------------------------------------------------------------------------------- /zstd/tests/golden-dictionaries/http-dict-missing-symbols: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/golden-dictionaries/http-dict-missing-symbols -------------------------------------------------------------------------------- /zstd/tests/gzip/gzip-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Test the obsolescent GZIP environment variable. 3 | 4 | # Copyright 2015-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | #echo PATH=$PATH 23 | #gzip --version 24 | 25 | echo a >exp || framework_failure_ 26 | gzip in || framework_failure_ 27 | 28 | fail=0 29 | GZIP=-qv gzip -d out 2>err || fail=1 30 | compare exp out || fail=1 31 | 32 | for badopt in -- -c --stdout -d --decompress -f --force -h --help -k --keep \ 33 | -l --list -L --license -r --recursive -Sxxx --suffix=xxx '--suffix xxx' \ 34 | -t --test -V --version 35 | do 36 | GZIP=$badopt gzip -d out 2>err && fail=1 37 | done 38 | 39 | for goodopt in -n --no-name -N --name -q --quiet -v --verbose \ 40 | -1 --fast -2 -3 -4 -5 -6 -7 -8 -9 --best 41 | do 42 | GZIP=$goodopt gzip -d out 2>err || fail=1 43 | compare exp out || fail=1 44 | done 45 | 46 | Exit $fail 47 | -------------------------------------------------------------------------------- /zstd/tests/gzip/helin-segv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Before gzip-1.4, gzip -d would segfault on some inputs. 3 | 4 | # Copyright (C) 2010-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | # This test case was provided by Aki Helin. 23 | printf '\037\235\220\0\0\0\304' > helin.gz || framework_failure_ 24 | printf '\0\0' > exp || framework_failure_ 25 | 26 | fail=0 27 | 28 | gzip -dc helin.gz > out || fail=1 29 | compare exp out || fail=1 30 | 31 | Exit $fail 32 | -------------------------------------------------------------------------------- /zstd/tests/gzip/hufts-segv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/valyala/gozstd/d0e6ac601c125c5b719a0725259f0f4b5fbb903e/zstd/tests/gzip/hufts-segv.gz -------------------------------------------------------------------------------- /zstd/tests/gzip/hufts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Exercise a bug whereby an invalid input could make gzip -d misbehave. 3 | 4 | # Copyright (C) 2009-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | printf '\n...: invalid compressed data--format violated\n' > exp \ 23 | || framework_failure_ 24 | 25 | fail=0 26 | gzip -dc "$abs_srcdir/hufts-segv.gz" > out 2> err 27 | test $? = 1 || fail=1 28 | 29 | compare /dev/null out || fail=1 30 | 31 | sed 's/.*hufts-segv.gz: /...: /' err > k; mv k err || fail=1 32 | compare exp err || fail=1 33 | 34 | Exit $fail 35 | -------------------------------------------------------------------------------- /zstd/tests/gzip/init.cfg: -------------------------------------------------------------------------------- 1 | # This file is sourced by init.sh, *before* its initialization. 2 | 3 | # This goes hand in hand with the "exec 9>&2;" in Makefile.am's 4 | # TESTS_ENVIRONMENT definition. 5 | stderr_fileno_=9 6 | -------------------------------------------------------------------------------- /zstd/tests/gzip/keep.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Exercise the --keep option. 3 | 4 | # Copyright (C) 2013-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | echo fooooooooo > in || framework_failure_ 23 | cp in orig || framework_failure_ 24 | 25 | fail=0 26 | 27 | # Compress and decompress both with and without --keep. 28 | for k in --keep ''; do 29 | # With --keep, the source must be retained, otherwise, it must be removed. 30 | case $k in --keep) op='||' ;; *) op='&&' ;; esac 31 | 32 | gzip $k in || fail=1 33 | eval "test -f in $op fail=1" 34 | test -f in.gz || fail=1 35 | rm -f in || fail=1 36 | 37 | gzip -d $k in.gz || fail=1 38 | eval "test -f in.gz $op fail=1" 39 | test -f in || fail=1 40 | compare in orig || fail=1 41 | rm -f in.gz || fail=1 42 | done 43 | 44 | cp orig in || framework_failure_ 45 | log=$(gzip -kv in 2>&1) || fail=1 46 | case $log in 47 | *'created in.gz'*) ;; 48 | *) fail=1;; 49 | esac 50 | 51 | Exit $fail 52 | -------------------------------------------------------------------------------- /zstd/tests/gzip/list.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Exercise the --list option. 3 | 4 | # Copyright 2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | echo zoology zucchini > in || framework_failure_ 23 | cp in orig || framework_failure_ 24 | 25 | gzip -l in && fail=1 26 | gzip -9 in || fail=1 27 | gzip -l in.gz >out1 || fail=1 28 | gzip -l in.gz | cat >out2 || fail=1 29 | compare out1 out2 || fail=1 30 | 31 | Exit $fail 32 | -------------------------------------------------------------------------------- /zstd/tests/gzip/memcpy-abuse.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Before gzip-1.4, this the use of memcpy in inflate_codes could 3 | # mistakenly operate on overlapping regions. Exercise that code. 4 | 5 | # Copyright (C) 2010-2016 Free Software Foundation, Inc. 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # limit so don't run it by default. 20 | 21 | . "${srcdir=.}/init.sh"; path_prepend_ . 22 | 23 | # The input must be larger than 32KiB and slightly 24 | # less uniform than e.g., all zeros. 25 | printf wxy%032767d 0 | tee in | gzip > in.gz || framework_failure_ 26 | 27 | fail=0 28 | 29 | # Before the fix, this would call memcpy with overlapping regions. 30 | gzip -dc in.gz > out || fail=1 31 | 32 | compare in out || fail=1 33 | 34 | Exit $fail 35 | -------------------------------------------------------------------------------- /zstd/tests/gzip/null-suffix-clobber.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Before gzip-1.5, gzip -d -S '' k.gz would delete F.gz and not create "F" 3 | 4 | # Copyright (C) 2010-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | printf anything | gzip > F.gz || framework_failure_ 23 | echo y > yes || framework_failure_ 24 | echo "gzip: invalid suffix ''" > expected-err || framework_failure_ 25 | 26 | fail=0 27 | 28 | gzip ---presume-input-tty -d -S '' F.gz < yes > out 2>err && fail=1 29 | 30 | compare /dev/null out || fail=1 31 | compare expected-err err || fail=1 32 | 33 | test -f F.gz || fail=1 34 | 35 | Exit $fail 36 | -------------------------------------------------------------------------------- /zstd/tests/gzip/stdin.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Ensure that gzip interprets "-" as stdin. 3 | 4 | # Copyright (C) 2009-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | printf a | gzip > in || framework_failure_ 23 | printf aaa > exp || framework_failure_ 24 | 25 | fail=0 26 | gzip -dc in - in < in > out 2>err || fail=1 27 | 28 | compare exp out || fail=1 29 | compare /dev/null err || fail=1 30 | 31 | Exit $fail 32 | -------------------------------------------------------------------------------- /zstd/tests/gzip/trailing-nul.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # gzip accepts trailing NUL bytes; don't fail if there is exactly one. 3 | # Before gzip-1.4, this would fail. 4 | 5 | # Copyright (C) 2009-2016 Free Software Foundation, Inc. 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # limit so don't run it by default. 20 | 21 | . "${srcdir=.}/init.sh"; path_prepend_ . 22 | 23 | (echo 0 | gzip; printf '\0') > 0.gz || framework_failure_ 24 | (echo 00 | gzip; printf '\0\0') > 00.gz || framework_failure_ 25 | (echo 1 | gzip; printf '\1') > 1.gz || framework_failure_ 26 | 27 | fail=0 28 | 29 | for i in 0 00 1; do 30 | gzip -d $i.gz; ret=$? 31 | test $ret -eq $i || fail=1 32 | test $ret = 1 && continue 33 | echo $i > exp || fail=1 34 | compare exp $i || fail=1 35 | done 36 | 37 | Exit $fail 38 | -------------------------------------------------------------------------------- /zstd/tests/gzip/unpack-invalid.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # gzip should report invalid 'unpack' input when uncompressing. 3 | # With gzip-1.5, it would output invalid data instead. 4 | 5 | # Copyright (C) 2012-2016 Free Software Foundation, Inc. 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # limit so don't run it by default. 20 | 21 | . "${srcdir=.}/init.sh"; path_prepend_ . 22 | 23 | for input in \ 24 | '\037\036\000\000\037\213\010\000\000\000\000\000\002\003\036\000\000\000\002\003\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\037\000\302\240\037\213\010\000\000\000\000\000\002\003\355\301' \ 25 | '\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\076\366\017\370\036\016\030\000\000\000\000\000\000\000\000\000\034\010\105\140\104\025\020\047\000\000\037\036\016\030\000\000\000'; do 26 | 27 | printf "$input" >in || framework_failure_ 28 | 29 | if gzip -d out 2>err; then 30 | fail=1 31 | else 32 | fail=0 33 | fi 34 | done 35 | 36 | Exit $fail 37 | -------------------------------------------------------------------------------- /zstd/tests/gzip/z-suffix.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Check that -Sz works. 3 | 4 | # Copyright 2014-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | printf anything > F && cp F G || framework_failure_ 23 | gzip -Sz F || fail=1 24 | test ! -f F || fail=1 25 | test -f Fz || fail=1 26 | gzip -dSz F || fail=1 27 | test ! -f Fz || fail=1 28 | compare F G || fail\1 29 | 30 | Exit $fail 31 | -------------------------------------------------------------------------------- /zstd/tests/gzip/zdiff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Exercise zdiff with two compressed inputs. 3 | # Before gzip-1.4, this would fail. 4 | 5 | # Copyright (C) 2009-2016 Free Software Foundation, Inc. 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # limit so don't run it by default. 20 | 21 | . "${srcdir=.}/init.sh"; path_prepend_ . 22 | 23 | echo a > a || framework_failure_ 24 | echo b > b || framework_failure_ 25 | gzip a b || framework_failure_ 26 | 27 | cat < exp 28 | 1c1 29 | < a 30 | --- 31 | > b 32 | EOF 33 | 34 | fail=0 35 | zdiff a.gz b.gz > out 2>&1 36 | test $? = 1 || fail=1 37 | 38 | compare exp out || fail=1 39 | 40 | rm -f out 41 | # expect success, for equal files 42 | zdiff a.gz a.gz > out 2> err || fail=1 43 | # expect no output 44 | test -s out && fail=1 45 | # expect no stderr 46 | test -s err && fail=1 47 | 48 | Exit $fail 49 | -------------------------------------------------------------------------------- /zstd/tests/gzip/zgrep-context.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Ensure that zgrep -15 works. Before gzip-1.5, it would fail. 3 | 4 | # Copyright (C) 2012-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | # A limited replacement for seq: handle 1 or 2 args; increment must be 1 23 | seq() 24 | { 25 | case $# in 26 | 1) start=1 final=$1;; 27 | 2) start=$1 final=$2;; 28 | *) echo you lose 1>&2; exit 1;; 29 | esac 30 | awk 'BEGIN{for(i='$start';i<='$final';i++) print i}' < /dev/null 31 | } 32 | 33 | seq 40 > in || framework_failure_ 34 | gzip < in > in.gz || framework_failure_ 35 | seq 2 32 > exp || framework_failure_ 36 | 37 | : ${GREP=grep} 38 | $GREP -15 17 - < in > out && compare exp out || { 39 | echo >&2 "$0: $GREP does not support context options; skipping this test" 40 | exit 77 41 | } 42 | 43 | fail=0 44 | zgrep -15 17 - < in.gz > out || fail=1 45 | compare exp out || fail=1 46 | 47 | Exit $fail 48 | -------------------------------------------------------------------------------- /zstd/tests/gzip/zgrep-f.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Ensure that zgrep -f - works like grep -f - 3 | # Before gzip-1.4, it would fail. 4 | 5 | # Copyright (C) 2009-2016 Free Software Foundation, Inc. 6 | 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see . 19 | # limit so don't run it by default. 20 | 21 | . "${srcdir=.}/init.sh"; path_prepend_ . 22 | 23 | printf 'needle\nn2\n' > n || framework_failure_ 24 | cp n haystack || framework_failure_ 25 | gzip haystack || framework_failure_ 26 | 27 | fail=0 28 | zgrep -f - haystack.gz < n > out 2>&1 || fail=1 29 | 30 | compare out n || fail=1 31 | 32 | if ${BASH_VERSION+:} false; then 33 | set +o posix 34 | # This failed with gzip 1.6. 35 | cat n n >nn || framework_failure_ 36 | eval 'zgrep -h -f <(cat n) haystack.gz haystack.gz' >out || fail=1 37 | compare out nn || fail=1 38 | fi 39 | 40 | # This failed with gzip 1.4. 41 | echo a-b | zgrep -e - > /dev/null || fail=1 42 | 43 | Exit $fail 44 | -------------------------------------------------------------------------------- /zstd/tests/gzip/znew-k.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Check that znew -K works without compress(1). 3 | 4 | # Copyright (C) 2010-2016 Free Software Foundation, Inc. 5 | 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # limit so don't run it by default. 19 | 20 | . "${srcdir=.}/init.sh"; path_prepend_ . 21 | 22 | cat <<'EOF' >compress || framework_failure_ 23 | #!/bin/sh 24 | echo >&2 'compress has been invoked' 25 | exit 1 26 | EOF 27 | chmod +x compress || framework_failure_ 28 | 29 | # Note that the basename must have a length of 6 or greater. 30 | # Otherwise, "test -f $name" below would fail. 31 | name=123456.Z 32 | 33 | printf '%1012977s' ' ' | gzip -c > $name || framework_failure_ 34 | 35 | fail=0 36 | 37 | znew -K $name || fail=1 38 | test -f $name || fail=1 39 | 40 | Exit $fail 41 | -------------------------------------------------------------------------------- /zstd/tests/loremOut.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | /* LOREM_genOut(): 12 | * Generate @size bytes of compressible data using lorem ipsum generator into 13 | * stdout. 14 | */ 15 | void LOREM_genOut(unsigned long long size, unsigned seed); 16 | -------------------------------------------------------------------------------- /zstd/tests/rateLimiter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # ################################################################ 4 | # Copyright (c) Meta Platforms, Inc. and affiliates. 5 | # All rights reserved. 6 | # 7 | # This source code is licensed under both the BSD-style license (found in the 8 | # LICENSE file in the root directory of this source tree) and the GPLv2 (found 9 | # in the COPYING file in the root directory of this source tree). 10 | # You may select, at your option, one of the above-listed licenses. 11 | # ########################################################################## 12 | 13 | # Rate limiter, replacement for pv 14 | # this rate limiter does not "catch up" after a blocking period 15 | # Limitations: 16 | # - only accepts limit speed in MB/s 17 | 18 | import sys 19 | import time 20 | 21 | MB = 1024 * 1024 22 | rate = float(sys.argv[1]) * MB 23 | start = time.time() 24 | total_read = 0 25 | 26 | # sys.stderr.close() # remove error message, for Ctrl+C 27 | 28 | try: 29 | buf = " " 30 | while len(buf): 31 | now = time.time() 32 | to_read = max(int(rate * (now - start)), 1) 33 | max_buf_size = 1 * MB 34 | to_read = min(to_read, max_buf_size) 35 | start = now 36 | 37 | buf = sys.stdin.buffer.read(to_read) 38 | sys.stdout.buffer.write(buf) 39 | 40 | except (KeyboardInterrupt, BrokenPipeError) as e: 41 | pass 42 | -------------------------------------------------------------------------------- /zstd/tests/regression/.gitignore: -------------------------------------------------------------------------------- 1 | # regression test artifacts 2 | data-cache 3 | cache 4 | test 5 | -------------------------------------------------------------------------------- /zstd/tests/regression/README.md: -------------------------------------------------------------------------------- 1 | # Regression tests 2 | 3 | The regression tests run zstd in many scenarios and ensures that the size of the compressed results doesn't change. This helps us ensure that we don't accidentally regress zstd's compression ratio. 4 | 5 | These tests get run every night by CircleCI. If the job fails you can read the diff printed by the job to ensure the change isn't a regression. If all is well you can download the `results.csv` artifact and commit the new results. Or you can rebuild it yourself following the instructions below. 6 | 7 | ## Rebuilding results.csv 8 | 9 | From the root of the zstd repo run: 10 | 11 | ``` 12 | # Build the zstd binary 13 | make clean 14 | make -j zstd 15 | 16 | # Build the regression test binary 17 | cd tests/regression 18 | make clean 19 | make -j test 20 | 21 | # Run the regression test 22 | ./test --cache data-cache --zstd ../../zstd --output results.csv 23 | 24 | # Check results.csv to ensure the new results are okay 25 | git diff 26 | 27 | # Then submit the PR 28 | ``` 29 | -------------------------------------------------------------------------------- /zstd/tests/regression/levels.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #ifndef LEVEL 12 | # error LEVEL(x) must be defined 13 | #endif 14 | #ifndef FAST_LEVEL 15 | # error FAST_LEVEL(x) must be defined 16 | #endif 17 | #ifndef ROW_LEVEL 18 | # error ROW_LEVEL(x, y) must be defined 19 | #endif 20 | 21 | /** 22 | * The levels are chosen to trigger every strategy in every source size, 23 | * as well as some fast levels and the default level. 24 | * If you change the compression levels, you should probably update these. 25 | */ 26 | 27 | FAST_LEVEL(5) 28 | 29 | FAST_LEVEL(3) 30 | 31 | FAST_LEVEL(1) 32 | LEVEL(0) 33 | LEVEL(1) 34 | 35 | LEVEL(3) 36 | LEVEL(4) 37 | /* ROW_LEVEL triggers the row hash (force enabled and disabled) with different 38 | * dictionary strategies, and 16/32/64 row entries based on the level/searchLog. 39 | * 1 == enabled, 2 == disabled. 40 | */ 41 | ROW_LEVEL(5, 1) 42 | ROW_LEVEL(5, 2) /* 16-entry rows */ 43 | LEVEL(5) 44 | LEVEL(6) 45 | ROW_LEVEL(7, 1) 46 | ROW_LEVEL(7, 2) /* 16-entry rows */ 47 | LEVEL(7) 48 | 49 | LEVEL(9) 50 | 51 | ROW_LEVEL(11, 1) 52 | ROW_LEVEL(11, 2) /* 32-entry rows */ 53 | ROW_LEVEL(12, 1) 54 | ROW_LEVEL(12, 2) /* 64-entry rows */ 55 | LEVEL(13) 56 | 57 | LEVEL(16) 58 | 59 | LEVEL(19) 60 | -------------------------------------------------------------------------------- /zstd/tests/regression/result.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under both the BSD-style license (found in the 6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 | * in the COPYING file in the root directory of this source tree). 8 | * You may select, at your option, one of the above-listed licenses. 9 | */ 10 | 11 | #include "result.h" 12 | 13 | char const* result_get_error_string(result_t result) { 14 | switch (result_get_error(result)) { 15 | case result_error_ok: 16 | return "okay"; 17 | case result_error_skip: 18 | return "skip"; 19 | case result_error_system_error: 20 | return "system error"; 21 | case result_error_compression_error: 22 | return "compression error"; 23 | case result_error_decompression_error: 24 | return "decompression error"; 25 | case result_error_round_trip_error: 26 | return "round trip error"; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /zstd/zlibWrapper/.gitignore: -------------------------------------------------------------------------------- 1 | # object artifacts 2 | *.o 3 | 4 | # Default result files 5 | _* 6 | example 7 | example_zstd.* 8 | example_gz.* 9 | fitblk 10 | fitblk_zstd.* 11 | zwrapbench 12 | foo.gz 13 | 14 | minigzip 15 | minigzip_zstd 16 | example 17 | example_zstd 18 | fitblk 19 | fitblk_zstd 20 | zwrapbench 21 | 22 | # Misc files 23 | *.bat 24 | *.zip 25 | *.txt 26 | 27 | # Directories 28 | minizip/ 29 | -------------------------------------------------------------------------------- /zstd/zlibWrapper/BUCK: -------------------------------------------------------------------------------- 1 | cxx_library( 2 | name='zlib_wrapper', 3 | visibility=['PUBLIC'], 4 | exported_linker_flags=['-lz'], 5 | header_namespace='', 6 | exported_headers=['zstd_zlibwrapper.h'], 7 | headers=[ 8 | 'gzcompatibility.h', 9 | 'gzguts.h', 10 | ], 11 | srcs=glob(['*.c']), 12 | deps=[ 13 | '//lib:zstd', 14 | '//lib:zstd_common', 15 | ], 16 | ) 17 | 18 | cxx_binary( 19 | name='minigzip', 20 | srcs=['examples/minigzip.c'], 21 | deps=[':zlib_wrapper'], 22 | ) 23 | -------------------------------------------------------------------------------- /zstd/zlibWrapper/gzclose.c: -------------------------------------------------------------------------------- 1 | /* gzclose.c contains minimal changes required to be compiled with zlibWrapper: 2 | * - gz_statep was converted to union to work with -Wstrict-aliasing=1 */ 3 | 4 | /* gzclose.c -- zlib gzclose() function 5 | * Copyright (C) 2004, 2010 Mark Adler 6 | * For conditions of distribution and use, see https://www.zlib.net/zlib_license.html 7 | */ 8 | 9 | #include "gzguts.h" 10 | 11 | /* gzclose() is in a separate file so that it is linked in only if it is used. 12 | That way the other gzclose functions can be used instead to avoid linking in 13 | unneeded compression or decompression routines. */ 14 | int ZEXPORT gzclose(gzFile file) { 15 | #ifndef NO_GZCOMPRESS 16 | gz_statep state; 17 | 18 | if (file == NULL) 19 | return Z_STREAM_ERROR; 20 | state.file = file; 21 | 22 | return state.state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file); 23 | #else 24 | return gzclose_r(file); 25 | #endif 26 | } 27 | --------------------------------------------------------------------------------