├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── codeql-analysis.yml │ ├── go.yml │ └── release.yml ├── .gitignore ├── .goreleaser.yml ├── LICENSE ├── README.md ├── SECURITY.md ├── compressible.go ├── compressible_test.go ├── dict ├── README.md ├── builder.go └── cmd │ └── builddict │ └── main.go ├── flate ├── _gen │ └── gen_inflate.go ├── deflate.go ├── deflate_test.go ├── dict_decoder.go ├── dict_decoder_test.go ├── fast_encoder.go ├── flate_test.go ├── fuzz_test.go ├── huffman_bit_writer.go ├── huffman_bit_writer_test.go ├── huffman_code.go ├── huffman_sortByFreq.go ├── huffman_sortByLiteral.go ├── inflate.go ├── inflate_gen.go ├── inflate_test.go ├── level1.go ├── level2.go ├── level3.go ├── level4.go ├── level5.go ├── level6.go ├── matchlen_generic.go ├── reader_test.go ├── regmask_amd64.go ├── regmask_other.go ├── stateless.go ├── testdata │ ├── fuzz │ │ ├── FuzzEncoding.zip │ │ └── encode-raw-corpus.zip │ ├── huffman-null-max.dyn.expect │ ├── huffman-null-max.dyn.expect-noinput │ ├── huffman-null-max.golden │ ├── huffman-null-max.in │ ├── huffman-null-max.sync.expect │ ├── huffman-null-max.sync.expect-noinput │ ├── huffman-null-max.wb.expect │ ├── huffman-null-max.wb.expect-noinput │ ├── huffman-pi.dyn.expect │ ├── huffman-pi.dyn.expect-noinput │ ├── huffman-pi.golden │ ├── huffman-pi.in │ ├── huffman-pi.sync.expect │ ├── huffman-pi.sync.expect-noinput │ ├── huffman-pi.wb.expect │ ├── huffman-pi.wb.expect-noinput │ ├── huffman-rand-1k.dyn.expect │ ├── huffman-rand-1k.dyn.expect-noinput │ ├── huffman-rand-1k.golden │ ├── huffman-rand-1k.in │ ├── huffman-rand-1k.sync.expect │ ├── huffman-rand-1k.sync.expect-noinput │ ├── huffman-rand-1k.wb.expect │ ├── huffman-rand-1k.wb.expect-noinput │ ├── huffman-rand-limit.dyn.expect │ ├── huffman-rand-limit.dyn.expect-noinput │ ├── huffman-rand-limit.golden │ ├── huffman-rand-limit.in │ ├── huffman-rand-limit.sync.expect │ ├── huffman-rand-limit.sync.expect-noinput │ ├── huffman-rand-limit.wb.expect │ ├── huffman-rand-limit.wb.expect-noinput │ ├── huffman-rand-max.golden │ ├── huffman-rand-max.in │ ├── huffman-shifts.dyn.expect │ ├── huffman-shifts.dyn.expect-noinput │ ├── huffman-shifts.golden │ ├── huffman-shifts.in │ ├── huffman-shifts.sync.expect │ ├── huffman-shifts.sync.expect-noinput │ ├── huffman-shifts.wb.expect │ ├── huffman-shifts.wb.expect-noinput │ ├── huffman-text-shift.dyn.expect │ ├── huffman-text-shift.dyn.expect-noinput │ ├── huffman-text-shift.golden │ ├── huffman-text-shift.in │ ├── huffman-text-shift.sync.expect │ ├── huffman-text-shift.sync.expect-noinput │ ├── huffman-text-shift.wb.expect │ ├── huffman-text-shift.wb.expect-noinput │ ├── huffman-text.dyn.expect │ ├── huffman-text.dyn.expect-noinput │ ├── huffman-text.golden │ ├── huffman-text.in │ ├── huffman-text.sync.expect │ ├── huffman-text.sync.expect-noinput │ ├── huffman-text.wb.expect │ ├── huffman-text.wb.expect-noinput │ ├── huffman-zero.dyn.expect │ ├── huffman-zero.dyn.expect-noinput │ ├── huffman-zero.golden │ ├── huffman-zero.in │ ├── huffman-zero.sync.expect │ ├── huffman-zero.sync.expect-noinput │ ├── huffman-zero.wb.expect │ ├── huffman-zero.wb.expect-noinput │ ├── null-long-match.dyn.expect-noinput │ ├── null-long-match.sync.expect-noinput │ ├── null-long-match.wb.expect-noinput │ ├── partial-block │ ├── regression.zip │ └── tokens.bin ├── token.go ├── token_test.go └── writer_test.go ├── fse ├── README.md ├── bitreader.go ├── bitwriter.go ├── bytereader.go ├── compress.go ├── decompress.go ├── fse.go ├── fse_test.go ├── fuzz_test.go └── testdata │ ├── fse_compress.zip │ └── fse_decompress.zip ├── gen.sh ├── go.mod ├── go.sum ├── gzhttp ├── LICENSE ├── README.md ├── asserts_test.go ├── compress.go ├── compress_test.go ├── examples_test.go ├── testdata │ └── benchmark.json ├── transport.go ├── transport_test.go └── writer │ ├── gzkp │ ├── gzkp.go │ └── gzkp_test.go │ ├── gzstd │ ├── stdlib.go │ └── stdlib_test.go │ └── interface.go ├── gzip ├── example_test.go ├── gunzip.go ├── gunzip_test.go ├── gzip.go ├── gzip_test.go └── testdata │ ├── issue6550.gz │ └── test.json ├── huff0 ├── .gitignore ├── README.md ├── _generate │ ├── gen.go │ ├── go.mod │ └── go.sum ├── bitreader.go ├── bitwriter.go ├── compress.go ├── compress_test.go ├── decompress.go ├── decompress_amd64.go ├── decompress_amd64.s ├── decompress_generic.go ├── decompress_test.go ├── fuzz_test.go ├── huff0.go └── testdata │ ├── decompress1x_regression.zip │ ├── fse_compress.zip │ ├── huff0_decompress1x.zip │ └── regression.zip ├── internal ├── cpuinfo │ ├── cpuinfo.go │ ├── cpuinfo_amd64.go │ └── cpuinfo_amd64.s ├── fuzz │ └── helpers.go ├── godebug │ └── godebug.go ├── le │ ├── le.go │ ├── unsafe_disabled.go │ └── unsafe_enabled.go ├── lz4ref │ ├── LICENSE │ ├── block.go │ └── errors.go ├── race │ ├── norace.go │ └── race.go └── snapref │ ├── LICENSE │ ├── decode.go │ ├── decode_other.go │ ├── encode.go │ ├── encode_other.go │ └── snappy.go ├── ossfuzz ├── cmd │ └── setup_dicts.go └── ossfuzz.sh ├── s2 ├── .gitignore ├── LICENSE ├── README.md ├── _generate │ ├── cleanup.go │ ├── gen.go │ ├── go.mod │ └── go.sum ├── cmd │ ├── _s2sx │ │ ├── _unpack │ │ │ └── main.go │ │ ├── gen_other.go │ │ ├── gen_windows.go │ │ ├── gensfx.cmd │ │ ├── gensfx.sh │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── internal │ │ ├── filepathx │ │ │ ├── LICENSE │ │ │ ├── README.md │ │ │ ├── filepathx.go │ │ │ └── filepathx_test.go │ │ └── readahead │ │ │ ├── LICENSE │ │ │ ├── README.md │ │ │ └── reader.go │ ├── s2c │ │ └── main.go │ └── s2d │ │ └── main.go ├── decode.go ├── decode_amd64.s ├── decode_arm64.s ├── decode_asm.go ├── decode_other.go ├── decode_test.go ├── dict.go ├── dict_test.go ├── encode.go ├── encode_all.go ├── encode_amd64.go ├── encode_best.go ├── encode_better.go ├── encode_go.go ├── encode_test.go ├── encodeblock_amd64.go ├── encodeblock_amd64.s ├── examples_test.go ├── fuzz_test.go ├── index.go ├── index_test.go ├── lz4convert.go ├── lz4convert_test.go ├── lz4sconvert.go ├── lz4sconvert_test.go ├── reader.go ├── reader_test.go ├── s2.go ├── s2_test.go ├── testdata │ ├── 4f9e1a0da7915a3d69632f5613ed78bc998a8a23.zst │ ├── Mark.Twain-Tom.Sawyer.txt │ ├── Mark.Twain-Tom.Sawyer.txt.rawsnappy │ ├── dec-block-regressions.zip │ ├── enc_regressions.zip │ ├── fuzz │ │ ├── FuzzLZ4Block.zip │ │ ├── block-corpus-enc.zip │ │ ├── block-corpus-raw.zip │ │ ├── dict-corpus-oss.zip │ │ └── lz4-convert-corpus-raw.zip │ ├── random │ └── s2-dict.bin.gz ├── writer.go └── writer_test.go ├── s2sx.mod ├── s2sx.sum ├── snappy ├── .gitignore ├── AUTHORS ├── CONTRIBUTORS ├── LICENSE ├── README.md ├── decode.go ├── encode.go ├── snappy.go ├── snappy_test.go ├── testdata │ ├── Isaac.Newton-Opticks.txt │ └── Isaac.Newton-Opticks.txt.rawsnappy └── xerial │ ├── LICENSE │ ├── README.md │ ├── fuzz_test.go │ ├── testdata │ ├── FuzzDecode_raw.zip │ ├── FuzzDecoder.zip │ └── block-corpus-raw.zip │ ├── xerial.go │ └── xerial_test.go ├── testdata ├── Mark.Twain-Tom.Sawyer.txt ├── case1.bin ├── case2.bin ├── case3.bin ├── crash1.bin ├── crash2.bin ├── crash3.bin ├── crash4.bin ├── crash5.bin ├── dec-crash6.bin ├── dec-hang1.bin ├── dec-hang2.bin ├── dec-hang3.bin ├── dec-symlen1.bin ├── e.txt ├── endnonzero.bin ├── endzerobits.bin ├── fse-artifact3.bin ├── gettysburg.txt ├── html.txt ├── normcount2.bin ├── pi.txt ├── pngdata.bin └── sharnd.out ├── zip ├── example_test.go ├── fuzz_test.go ├── internal │ └── obscuretestdata │ │ └── obscuretestdata.go ├── reader.go ├── reader_test.go ├── register.go ├── struct.go ├── testdata │ ├── FuzzReader-enc.zip │ ├── FuzzReader-raw.zip │ ├── comment-truncated.zip │ ├── crc32-not-streamed.zip │ ├── dd.zip │ ├── dupdir.zip │ ├── go-no-datadesc-sig.zip │ ├── go-no-datadesc-sig.zip.base64 │ ├── go-with-datadesc-sig.zip │ ├── gophercolor16x16.png │ ├── readme.notzip │ ├── readme.zip │ ├── subdir.zip │ ├── symlink.zip │ ├── test-badbase.zip │ ├── test-baddirsz.zip │ ├── test-prefix.zip │ ├── test-trailing-junk.zip │ ├── test.zip │ ├── time-22738.zip │ ├── time-7zip.zip │ ├── time-go.zip │ ├── time-infozip.zip │ ├── time-osx.zip │ ├── time-win7.zip │ ├── time-winrar.zip │ ├── time-winzip.zip │ ├── unix.zip │ ├── utf8-7zip.zip │ ├── utf8-infozip.zip │ ├── utf8-osx.zip │ ├── utf8-winrar.zip │ ├── utf8-winzip.zip │ ├── winxp.zip │ ├── zip64-2.zip │ └── zip64.zip ├── writer.go ├── writer_test.go └── zip_test.go ├── zlib ├── example_test.go ├── reader.go ├── reader_test.go ├── writer.go └── writer_test.go └── zstd ├── README.md ├── _generate ├── gen.go ├── gen_fse.go ├── go.mod └── go.sum ├── bitreader.go ├── bitwriter.go ├── blockdec.go ├── blockenc.go ├── blocktype_string.go ├── bytebuf.go ├── bytereader.go ├── decodeheader.go ├── decodeheader_test.go ├── decoder.go ├── decoder_options.go ├── decoder_test.go ├── dict.go ├── dict_test.go ├── enc_base.go ├── enc_best.go ├── enc_better.go ├── enc_dfast.go ├── enc_fast.go ├── encoder.go ├── encoder_options.go ├── encoder_options_test.go ├── encoder_test.go ├── example_test.go ├── framedec.go ├── frameenc.go ├── fse_decoder.go ├── fse_decoder_amd64.go ├── fse_decoder_amd64.s ├── fse_decoder_generic.go ├── fse_encoder.go ├── fse_predefined.go ├── fuzz_test.go ├── hash.go ├── history.go ├── internal └── xxhash │ ├── LICENSE.txt │ ├── README.md │ ├── xxhash.go │ ├── xxhash_amd64.s │ ├── xxhash_arm64.s │ ├── xxhash_asm.go │ ├── xxhash_other.go │ ├── xxhash_safe.go │ └── xxhash_test.go ├── matchlen_amd64.go ├── matchlen_amd64.s ├── matchlen_generic.go ├── race_enabled_test.go ├── seqdec.go ├── seqdec_amd64.go ├── seqdec_amd64.s ├── seqdec_amd64_test.go ├── seqdec_generic.go ├── seqdec_test.go ├── seqenc.go ├── snappy.go ├── snappy_test.go ├── testdata ├── bad.zip ├── benchdecoder.zip ├── comp-crashers.zip ├── decode-regression.zip ├── decoder.zip ├── delta │ ├── source.txt │ ├── target.txt │ └── target.txt.zst ├── dict-tests-small.zip ├── fuzz │ ├── decode-corpus-encoded.zip │ ├── decode-corpus-raw.zip │ ├── decode-oss.zip │ ├── encode-corpus-encoded.zip │ └── encode-corpus-raw.zip ├── good.zip ├── headers-want.json.zst ├── headers.zip ├── large.zip ├── regression.zip ├── seqs-want.zip ├── seqs.zip ├── xml.zst ├── z000028 └── z000028.zst ├── zip.go ├── zip_test.go ├── zstd.go └── zstd_test.go /.gitattributes: -------------------------------------------------------------------------------- 1 | * -text 2 | *.bin -text -diff 3 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | groups: 8 | github-actions: 9 | patterns: 10 | - "*" 11 | 12 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "master" ] 20 | schedule: 21 | - cron: '31 1 * * 5' 22 | 23 | permissions: read-all 24 | 25 | jobs: 26 | analyze: 27 | name: Analyze 28 | runs-on: ubuntu-latest 29 | permissions: 30 | actions: read 31 | contents: read 32 | security-events: write 33 | 34 | strategy: 35 | fail-fast: false 36 | matrix: 37 | language: [ 'go' ] 38 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 39 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 40 | 41 | steps: 42 | - name: Checkout repository 43 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 44 | 45 | # Initializes the CodeQL tools for scanning. 46 | - name: Initialize CodeQL 47 | uses: github/codeql-action/init@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 48 | with: 49 | languages: ${{ matrix.language }} 50 | # If you wish to specify custom queries, you can do so here or in a config file. 51 | # By default, queries listed here will override any specified in a config file. 52 | # Prefix the list here with "+" to use these queries and those in the config file. 53 | 54 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 55 | # queries: security-extended,security-and-quality 56 | 57 | 58 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 59 | # If this step fails, then you should remove it and run the build manually (see below) 60 | - name: Autobuild 61 | uses: github/codeql-action/autobuild@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 62 | 63 | # ℹ️ Command-line programs to run using the OS shell. 64 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 65 | 66 | # If the Autobuild fails above, remove it and uncomment the following three lines. 67 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 68 | 69 | # - run: | 70 | # echo "Run, Build Application using script" 71 | # ./location_of_script_within_repo/buildscript.sh 72 | 73 | - name: Perform CodeQL Analysis 74 | uses: github/codeql-action/analyze@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 75 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | permissions: 9 | contents: write # for goreleaser/goreleaser-action to create a GitHub release 10 | 11 | jobs: 12 | goreleaser: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - 16 | name: Checkout 17 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 18 | with: 19 | fetch-depth: 0 20 | - 21 | name: Set up Go 22 | uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.2.0 23 | with: 24 | go-version: 1.24.x 25 | - 26 | name: Run GoReleaser 27 | uses: goreleaser/goreleaser-action@9c156ee8a17a598857849441385a2041ef570552 # v6.3.0 28 | with: 29 | version: 2.3.2 30 | args: release --clean 31 | env: 32 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 33 | CGO_ENABLED: 0 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | /s2/cmd/_s2sx/sfx-exe 26 | 27 | # Linux perf files 28 | perf.data 29 | perf.data.old 30 | 31 | # gdb history 32 | .gdb_history 33 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | before: 4 | hooks: 5 | - ./gen.sh 6 | 7 | builds: 8 | - 9 | id: "s2c" 10 | binary: s2c 11 | main: ./s2/cmd/s2c/main.go 12 | flags: 13 | - -trimpath 14 | env: 15 | - CGO_ENABLED=0 16 | goos: 17 | - aix 18 | - linux 19 | - freebsd 20 | - netbsd 21 | - windows 22 | - darwin 23 | goarch: 24 | - 386 25 | - amd64 26 | - arm 27 | - arm64 28 | - ppc64 29 | - ppc64le 30 | - mips64 31 | - mips64le 32 | goarm: 33 | - 7 34 | - 35 | id: "s2d" 36 | binary: s2d 37 | main: ./s2/cmd/s2d/main.go 38 | flags: 39 | - -trimpath 40 | env: 41 | - CGO_ENABLED=0 42 | goos: 43 | - aix 44 | - linux 45 | - freebsd 46 | - netbsd 47 | - windows 48 | - darwin 49 | goarch: 50 | - 386 51 | - amd64 52 | - arm 53 | - arm64 54 | - ppc64 55 | - ppc64le 56 | - mips64 57 | - mips64le 58 | goarm: 59 | - 7 60 | - 61 | id: "s2sx" 62 | binary: s2sx 63 | main: ./s2/cmd/_s2sx/main.go 64 | flags: 65 | - -modfile=s2sx.mod 66 | - -trimpath 67 | env: 68 | - CGO_ENABLED=0 69 | goos: 70 | - aix 71 | - linux 72 | - freebsd 73 | - netbsd 74 | - windows 75 | - darwin 76 | goarch: 77 | - 386 78 | - amd64 79 | - arm 80 | - arm64 81 | - ppc64 82 | - ppc64le 83 | - mips64 84 | - mips64le 85 | goarm: 86 | - 7 87 | 88 | archives: 89 | - 90 | id: s2-binaries 91 | name_template: "s2-{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" 92 | format_overrides: 93 | - goos: windows 94 | format: zip 95 | files: 96 | - unpack/* 97 | - s2/LICENSE 98 | - s2/README.md 99 | checksum: 100 | name_template: 'checksums.txt' 101 | snapshot: 102 | version_template: "{{ .Tag }}-next" 103 | changelog: 104 | sort: asc 105 | filters: 106 | exclude: 107 | - '^doc:' 108 | - '^docs:' 109 | - '^test:' 110 | - '^tests:' 111 | - '^Update\sREADME.md' 112 | 113 | nfpms: 114 | - 115 | file_name_template: "s2_package__{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" 116 | vendor: Klaus Post 117 | homepage: https://github.com/klauspost/compress 118 | maintainer: Klaus Post 119 | description: S2 Compression Tool 120 | license: BSD 3-Clause 121 | formats: 122 | - deb 123 | - rpm 124 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Security updates are applied only to the latest release. 6 | 7 | ## Vulnerability Definition 8 | 9 | A security vulnerability is a bug that with certain input triggers a crash or an infinite loop. Most calls will have varying execution time and only in rare cases will slow operation be considered a security vulnerability. 10 | 11 | Corrupted output generally is not considered a security vulnerability, unless independent operations are able to affect each other. Note that not all functionality is re-entrant and safe to use concurrently. 12 | 13 | Out-of-memory crashes only applies if the en/decoder uses an abnormal amount of memory, with appropriate options applied, to limit maximum window size, concurrency, etc. However, if you are in doubt you are welcome to file a security issue. 14 | 15 | It is assumed that all callers are trusted, meaning internal data exposed through reflection or inspection of returned data structures is not considered a vulnerability. 16 | 17 | Vulnerabilities resulting from compiler/assembler errors should be reported upstream. Depending on the severity this package may or may not implement a workaround. 18 | 19 | ## Reporting a Vulnerability 20 | 21 | If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released. 22 | 23 | Please disclose it at [security advisory](https://github.com/klauspost/compress/security/advisories/new). If possible please provide a minimal reproducer. If the issue only applies to a single platform, it would be helpful to provide access to that. 24 | 25 | This project is maintained by a team of volunteers on a reasonable-effort basis. As such, vulnerabilities will be disclosed in a best effort base. 26 | -------------------------------------------------------------------------------- /compressible.go: -------------------------------------------------------------------------------- 1 | package compress 2 | 3 | import "math" 4 | 5 | // Estimate returns a normalized compressibility estimate of block b. 6 | // Values close to zero are likely uncompressible. 7 | // Values above 0.1 are likely to be compressible. 8 | // Values above 0.5 are very compressible. 9 | // Very small lengths will return 0. 10 | func Estimate(b []byte) float64 { 11 | if len(b) < 16 { 12 | return 0 13 | } 14 | 15 | // Correctly predicted order 1 16 | hits := 0 17 | lastMatch := false 18 | var o1 [256]byte 19 | var hist [256]int 20 | c1 := byte(0) 21 | for _, c := range b { 22 | if c == o1[c1] { 23 | // We only count a hit if there was two correct predictions in a row. 24 | if lastMatch { 25 | hits++ 26 | } 27 | lastMatch = true 28 | } else { 29 | lastMatch = false 30 | } 31 | o1[c1] = c 32 | c1 = c 33 | hist[c]++ 34 | } 35 | 36 | // Use x^0.6 to give better spread 37 | prediction := math.Pow(float64(hits)/float64(len(b)), 0.6) 38 | 39 | // Calculate histogram distribution 40 | variance := float64(0) 41 | avg := float64(len(b)) / 256 42 | 43 | for _, v := range hist { 44 | Δ := float64(v) - avg 45 | variance += Δ * Δ 46 | } 47 | 48 | stddev := math.Sqrt(float64(variance)) / float64(len(b)) 49 | exp := math.Sqrt(1 / float64(len(b))) 50 | 51 | // Subtract expected stddev 52 | stddev -= exp 53 | if stddev < 0 { 54 | stddev = 0 55 | } 56 | stddev *= 1 + exp 57 | 58 | // Use x^0.4 to give better spread 59 | entropy := math.Pow(stddev, 0.4) 60 | 61 | // 50/50 weight between prediction and histogram distribution 62 | return math.Pow((prediction+entropy)/2, 0.9) 63 | } 64 | 65 | // ShannonEntropyBits returns the number of bits minimum required to represent 66 | // an entropy encoding of the input bytes. 67 | // https://en.wiktionary.org/wiki/Shannon_entropy 68 | func ShannonEntropyBits(b []byte) int { 69 | if len(b) == 0 { 70 | return 0 71 | } 72 | var hist [256]int 73 | for _, c := range b { 74 | hist[c]++ 75 | } 76 | shannon := float64(0) 77 | invTotal := 1.0 / float64(len(b)) 78 | for _, v := range hist[:] { 79 | if v > 0 { 80 | n := float64(v) 81 | shannon += math.Ceil(-math.Log2(n*invTotal) * n) 82 | } 83 | } 84 | return int(math.Ceil(shannon)) 85 | } 86 | -------------------------------------------------------------------------------- /dict/cmd/builddict/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023+ Klaus Post. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "fmt" 10 | "io" 11 | "log" 12 | "os" 13 | "path/filepath" 14 | "runtime/debug" 15 | 16 | "github.com/klauspost/compress/dict" 17 | "github.com/klauspost/compress/zstd" 18 | ) 19 | 20 | var ( 21 | wantLenFlag = flag.Int("len", 112<<10, "Specify custom output size") 22 | wantHashBytes = flag.Int("hash", 6, "Hash bytes match length. Minimum match length.") 23 | wantMaxBytes = flag.Int("max", 32<<10, "Max input length to index per input file") 24 | wantOutput = flag.String("o", "dictionary.bin", "Output name") 25 | wantFormat = flag.String("format", "zstd", `Output type. "zstd" "s2" or "raw"`) 26 | wantZstdID = flag.Uint("dictID", 0, "Zstd dictionary ID. Default (0) will be random") 27 | wantZstdCompat = flag.Bool("zcompat", true, "Generate dictionary compatible with zstd 1.5.5 and older") 28 | wantZstdLevel = flag.Int("zlevel", 0, "Zstd compression level. 0-4") 29 | quiet = flag.Bool("q", false, "Do not print progress") 30 | ) 31 | 32 | func main() { 33 | flag.Parse() 34 | debug.SetGCPercent(25) 35 | o := dict.Options{ 36 | MaxDictSize: *wantLenFlag, 37 | HashBytes: *wantHashBytes, 38 | Output: os.Stdout, 39 | ZstdDictID: uint32(*wantZstdID), 40 | ZstdDictCompat: *wantZstdCompat, 41 | ZstdLevel: zstd.EncoderLevel(*wantZstdLevel), 42 | } 43 | if *wantOutput == "" || *quiet { 44 | o.Output = nil 45 | } 46 | var input [][]byte 47 | base := flag.Arg(0) 48 | if base == "" { 49 | flag.Usage() 50 | log.Fatal("no path with files specified") 51 | } 52 | 53 | // Index ALL hashes in all files. 54 | err := filepath.Walk(base, func(path string, info os.FileInfo, err error) error { 55 | if err != nil { 56 | return err 57 | } 58 | if info.IsDir() { 59 | return nil 60 | } 61 | 62 | f, err := os.Open(path) 63 | if err != nil { 64 | log.Print(err) 65 | return nil 66 | } 67 | defer f.Close() 68 | b, err := io.ReadAll(io.LimitReader(f, int64(*wantMaxBytes))) 69 | if len(b) < 8 { 70 | return nil 71 | } 72 | if len(b) == 0 { 73 | return nil 74 | } 75 | input = append(input, b) 76 | if !*quiet { 77 | fmt.Print("\r"+info.Name(), " read...") 78 | } 79 | return nil 80 | }) 81 | if err != nil { 82 | log.Fatal(err) 83 | } 84 | if len(input) == 0 { 85 | log.Fatal("no files read") 86 | } 87 | var out []byte 88 | switch *wantFormat { 89 | case "zstd": 90 | out, err = dict.BuildZstdDict(input, o) 91 | case "s2": 92 | out, err = dict.BuildS2Dict(input, o) 93 | case "raw": 94 | out, err = dict.BuildRawDict(input, o) 95 | default: 96 | err = fmt.Errorf("unknown format %q", *wantFormat) 97 | } 98 | if err != nil { 99 | log.Fatal(err) 100 | } 101 | if *wantOutput != "" { 102 | err = os.WriteFile(*wantOutput, out, 0666) 103 | if err != nil { 104 | log.Fatal(err) 105 | } 106 | } else { 107 | _, err = os.Stdout.Write(out) 108 | if err != nil { 109 | log.Fatal(err) 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /flate/matchlen_generic.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019+ Klaus Post. All rights reserved. 2 | // License information can be found in the LICENSE file. 3 | 4 | package flate 5 | 6 | import ( 7 | "math/bits" 8 | 9 | "github.com/klauspost/compress/internal/le" 10 | ) 11 | 12 | // matchLen returns the maximum common prefix length of a and b. 13 | // a must be the shortest of the two. 14 | func matchLen(a, b []byte) (n int) { 15 | left := len(a) 16 | for left >= 8 { 17 | diff := le.Load64(a, n) ^ le.Load64(b, n) 18 | if diff != 0 { 19 | return n + bits.TrailingZeros64(diff)>>3 20 | } 21 | n += 8 22 | left -= 8 23 | } 24 | 25 | a = a[n:] 26 | b = b[n:] 27 | for i := range a { 28 | if a[i] != b[i] { 29 | break 30 | } 31 | n++ 32 | } 33 | return n 34 | } 35 | -------------------------------------------------------------------------------- /flate/regmask_amd64.go: -------------------------------------------------------------------------------- 1 | package flate 2 | 3 | const ( 4 | // Masks for shifts with register sizes of the shift value. 5 | // This can be used to work around the x86 design of shifting by mod register size. 6 | // It can be used when a variable shift is always smaller than the register size. 7 | 8 | // reg8SizeMaskX - shift value is 8 bits, shifted is X 9 | reg8SizeMask8 = 7 10 | reg8SizeMask16 = 15 11 | reg8SizeMask32 = 31 12 | reg8SizeMask64 = 63 13 | 14 | // reg16SizeMaskX - shift value is 16 bits, shifted is X 15 | reg16SizeMask8 = reg8SizeMask8 16 | reg16SizeMask16 = reg8SizeMask16 17 | reg16SizeMask32 = reg8SizeMask32 18 | reg16SizeMask64 = reg8SizeMask64 19 | 20 | // reg32SizeMaskX - shift value is 32 bits, shifted is X 21 | reg32SizeMask8 = reg8SizeMask8 22 | reg32SizeMask16 = reg8SizeMask16 23 | reg32SizeMask32 = reg8SizeMask32 24 | reg32SizeMask64 = reg8SizeMask64 25 | 26 | // reg64SizeMaskX - shift value is 64 bits, shifted is X 27 | reg64SizeMask8 = reg8SizeMask8 28 | reg64SizeMask16 = reg8SizeMask16 29 | reg64SizeMask32 = reg8SizeMask32 30 | reg64SizeMask64 = reg8SizeMask64 31 | 32 | // regSizeMaskUintX - shift value is uint, shifted is X 33 | regSizeMaskUint8 = reg8SizeMask8 34 | regSizeMaskUint16 = reg8SizeMask16 35 | regSizeMaskUint32 = reg8SizeMask32 36 | regSizeMaskUint64 = reg8SizeMask64 37 | ) 38 | -------------------------------------------------------------------------------- /flate/regmask_other.go: -------------------------------------------------------------------------------- 1 | //go:build !amd64 2 | // +build !amd64 3 | 4 | package flate 5 | 6 | const ( 7 | // Masks for shifts with register sizes of the shift value. 8 | // This can be used to work around the x86 design of shifting by mod register size. 9 | // It can be used when a variable shift is always smaller than the register size. 10 | 11 | // reg8SizeMaskX - shift value is 8 bits, shifted is X 12 | reg8SizeMask8 = 0xff 13 | reg8SizeMask16 = 0xff 14 | reg8SizeMask32 = 0xff 15 | reg8SizeMask64 = 0xff 16 | 17 | // reg16SizeMaskX - shift value is 16 bits, shifted is X 18 | reg16SizeMask8 = 0xffff 19 | reg16SizeMask16 = 0xffff 20 | reg16SizeMask32 = 0xffff 21 | reg16SizeMask64 = 0xffff 22 | 23 | // reg32SizeMaskX - shift value is 32 bits, shifted is X 24 | reg32SizeMask8 = 0xffffffff 25 | reg32SizeMask16 = 0xffffffff 26 | reg32SizeMask32 = 0xffffffff 27 | reg32SizeMask64 = 0xffffffff 28 | 29 | // reg64SizeMaskX - shift value is 64 bits, shifted is X 30 | reg64SizeMask8 = 0xffffffffffffffff 31 | reg64SizeMask16 = 0xffffffffffffffff 32 | reg64SizeMask32 = 0xffffffffffffffff 33 | reg64SizeMask64 = 0xffffffffffffffff 34 | 35 | // regSizeMaskUintX - shift value is uint, shifted is X 36 | regSizeMaskUint8 = ^uint(0) 37 | regSizeMaskUint16 = ^uint(0) 38 | regSizeMaskUint32 = ^uint(0) 39 | regSizeMaskUint64 = ^uint(0) 40 | ) 41 | -------------------------------------------------------------------------------- /flate/testdata/fuzz/FuzzEncoding.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/fuzz/FuzzEncoding.zip -------------------------------------------------------------------------------- /flate/testdata/fuzz/encode-raw-corpus.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/fuzz/encode-raw-corpus.zip -------------------------------------------------------------------------------- /flate/testdata/huffman-null-max.dyn.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-null-max.dyn.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-null-max.dyn.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-null-max.dyn.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-null-max.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-null-max.golden -------------------------------------------------------------------------------- /flate/testdata/huffman-null-max.sync.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-null-max.sync.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-null-max.sync.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-null-max.sync.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-null-max.wb.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-null-max.wb.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-null-max.wb.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-null-max.wb.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-pi.dyn.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-pi.dyn.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-pi.dyn.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-pi.dyn.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-pi.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-pi.golden -------------------------------------------------------------------------------- /flate/testdata/huffman-pi.in: -------------------------------------------------------------------------------- 1 | 3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180 -------------------------------------------------------------------------------- /flate/testdata/huffman-pi.sync.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-pi.sync.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-pi.sync.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-pi.sync.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-pi.wb.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-pi.wb.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-pi.wb.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-pi.wb.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-1k.dyn.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-1k.dyn.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-1k.dyn.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-1k.dyn.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-1k.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-1k.golden -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-1k.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-1k.in -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-1k.sync.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-1k.sync.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-1k.sync.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-1k.sync.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-1k.wb.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-1k.wb.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-1k.wb.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-1k.wb.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-limit.dyn.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-limit.dyn.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-limit.dyn.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-limit.dyn.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-limit.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-limit.golden -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-limit.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-limit.in -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-limit.sync.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-limit.sync.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-limit.sync.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-limit.sync.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-limit.wb.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-limit.wb.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-limit.wb.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-limit.wb.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-max.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-max.golden -------------------------------------------------------------------------------- /flate/testdata/huffman-rand-max.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-rand-max.in -------------------------------------------------------------------------------- /flate/testdata/huffman-shifts.dyn.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-shifts.dyn.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-shifts.dyn.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-shifts.dyn.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-shifts.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-shifts.golden -------------------------------------------------------------------------------- /flate/testdata/huffman-shifts.sync.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-shifts.sync.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-shifts.sync.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-shifts.sync.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-shifts.wb.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-shifts.wb.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-shifts.wb.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-shifts.wb.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-text-shift.dyn.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text-shift.dyn.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-text-shift.dyn.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text-shift.dyn.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-text-shift.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text-shift.golden -------------------------------------------------------------------------------- /flate/testdata/huffman-text-shift.in: -------------------------------------------------------------------------------- 1 | //Copyright2009ThGoAuthor.Allrightrrvd. 2 | //UofthiourccodigovrndbyBSD-tyl 3 | //licnthtcnbfoundinthLICENSEfil. 4 | 5 | pckgmin 6 | 7 | import"o" 8 | 9 | funcmin(){ 10 | vrb=mk([]byt,65535) 11 | f,_:=o.Crt("huffmn-null-mx.in") 12 | f.Writ(b) 13 | } 14 | ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?" -------------------------------------------------------------------------------- /flate/testdata/huffman-text-shift.sync.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text-shift.sync.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-text-shift.sync.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text-shift.sync.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-text-shift.wb.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text-shift.wb.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-text-shift.wb.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text-shift.wb.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-text.dyn.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text.dyn.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-text.dyn.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text.dyn.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-text.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text.golden -------------------------------------------------------------------------------- /flate/testdata/huffman-text.in: -------------------------------------------------------------------------------- 1 | // Copyright 2009 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import "os" 8 | 9 | func main() { 10 | var b = make([]byte, 65535) 11 | f, _ := os.Create("huffman-null-max.in") 12 | f.Write(b) 13 | } 14 | -------------------------------------------------------------------------------- /flate/testdata/huffman-text.sync.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text.sync.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-text.sync.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text.sync.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-text.wb.expect: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text.wb.expect -------------------------------------------------------------------------------- /flate/testdata/huffman-text.wb.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-text.wb.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/huffman-zero.dyn.expect: -------------------------------------------------------------------------------- 1 | 2$ -------------------------------------------------------------------------------- /flate/testdata/huffman-zero.dyn.expect-noinput: -------------------------------------------------------------------------------- 1 | 2$ -------------------------------------------------------------------------------- /flate/testdata/huffman-zero.golden: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/huffman-zero.golden -------------------------------------------------------------------------------- /flate/testdata/huffman-zero.in: -------------------------------------------------------------------------------- 1 | 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 -------------------------------------------------------------------------------- /flate/testdata/huffman-zero.sync.expect: -------------------------------------------------------------------------------- 1 | 2$ -------------------------------------------------------------------------------- /flate/testdata/huffman-zero.sync.expect-noinput: -------------------------------------------------------------------------------- 1 | 2$ -------------------------------------------------------------------------------- /flate/testdata/huffman-zero.wb.expect: -------------------------------------------------------------------------------- 1 | 2$ -------------------------------------------------------------------------------- /flate/testdata/huffman-zero.wb.expect-noinput: -------------------------------------------------------------------------------- 1 | 2$ -------------------------------------------------------------------------------- /flate/testdata/null-long-match.dyn.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/null-long-match.dyn.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/null-long-match.sync.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/null-long-match.sync.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/null-long-match.wb.expect-noinput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/null-long-match.wb.expect-noinput -------------------------------------------------------------------------------- /flate/testdata/partial-block: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/partial-block -------------------------------------------------------------------------------- /flate/testdata/regression.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/regression.zip -------------------------------------------------------------------------------- /flate/testdata/tokens.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/flate/testdata/tokens.bin -------------------------------------------------------------------------------- /flate/token_test.go: -------------------------------------------------------------------------------- 1 | package flate 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | type testFatal interface { 10 | Fatal(args ...interface{}) 11 | } 12 | 13 | // loadTestTokens will load test tokens. 14 | // First block from enwik9, varint encoded. 15 | func loadTestTokens(t testFatal) *tokens { 16 | b, err := os.ReadFile("testdata/tokens.bin") 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | var tokens tokens 21 | err = tokens.FromVarInt(b) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | return &tokens 26 | } 27 | 28 | func Test_tokens_EstimatedBits(t *testing.T) { 29 | tok := loadTestTokens(t) 30 | // The estimated size, update if method changes. 31 | const expect = 221057 32 | n := tok.EstimatedBits() 33 | var buf bytes.Buffer 34 | wr := newHuffmanBitWriter(&buf) 35 | wr.writeBlockDynamic(tok, true, nil, true) 36 | if wr.err != nil { 37 | t.Fatal(wr.err) 38 | } 39 | wr.flush() 40 | t.Log("got:", n, "actual:", buf.Len()*8, "(header not part of estimate)") 41 | if n != expect { 42 | t.Error("want:", expect, "bits, got:", n) 43 | } 44 | } 45 | 46 | func Benchmark_tokens_EstimatedBits(b *testing.B) { 47 | tok := loadTestTokens(b) 48 | b.ResetTimer() 49 | // One "byte", one token iteration. 50 | b.SetBytes(1) 51 | for i := 0; i < b.N; i++ { 52 | _ = tok.EstimatedBits() 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /fse/bitreader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Klaus Post. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | // Based on work Copyright (c) 2013, Yann Collet, released under BSD License. 5 | 6 | package fse 7 | 8 | import ( 9 | "encoding/binary" 10 | "errors" 11 | "io" 12 | ) 13 | 14 | // bitReader reads a bitstream in reverse. 15 | // The last set bit indicates the start of the stream and is used 16 | // for aligning the input. 17 | type bitReader struct { 18 | in []byte 19 | off uint // next byte to read is at in[off - 1] 20 | value uint64 21 | bitsRead uint8 22 | } 23 | 24 | // init initializes and resets the bit reader. 25 | func (b *bitReader) init(in []byte) error { 26 | if len(in) < 1 { 27 | return errors.New("corrupt stream: too short") 28 | } 29 | b.in = in 30 | b.off = uint(len(in)) 31 | // The highest bit of the last byte indicates where to start 32 | v := in[len(in)-1] 33 | if v == 0 { 34 | return errors.New("corrupt stream, did not find end of stream") 35 | } 36 | b.bitsRead = 64 37 | b.value = 0 38 | if len(in) >= 8 { 39 | b.fillFastStart() 40 | } else { 41 | b.fill() 42 | b.fill() 43 | } 44 | b.bitsRead += 8 - uint8(highBits(uint32(v))) 45 | return nil 46 | } 47 | 48 | // getBits will return n bits. n can be 0. 49 | func (b *bitReader) getBits(n uint8) uint16 { 50 | if n == 0 || b.bitsRead >= 64 { 51 | return 0 52 | } 53 | return b.getBitsFast(n) 54 | } 55 | 56 | // getBitsFast requires that at least one bit is requested every time. 57 | // There are no checks if the buffer is filled. 58 | func (b *bitReader) getBitsFast(n uint8) uint16 { 59 | const regMask = 64 - 1 60 | v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) 61 | b.bitsRead += n 62 | return v 63 | } 64 | 65 | // fillFast() will make sure at least 32 bits are available. 66 | // There must be at least 4 bytes available. 67 | func (b *bitReader) fillFast() { 68 | if b.bitsRead < 32 { 69 | return 70 | } 71 | // 2 bounds checks. 72 | v := b.in[b.off-4:] 73 | v = v[:4] 74 | low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) 75 | b.value = (b.value << 32) | uint64(low) 76 | b.bitsRead -= 32 77 | b.off -= 4 78 | } 79 | 80 | // fill() will make sure at least 32 bits are available. 81 | func (b *bitReader) fill() { 82 | if b.bitsRead < 32 { 83 | return 84 | } 85 | if b.off > 4 { 86 | v := b.in[b.off-4:] 87 | v = v[:4] 88 | low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) 89 | b.value = (b.value << 32) | uint64(low) 90 | b.bitsRead -= 32 91 | b.off -= 4 92 | return 93 | } 94 | for b.off > 0 { 95 | b.value = (b.value << 8) | uint64(b.in[b.off-1]) 96 | b.bitsRead -= 8 97 | b.off-- 98 | } 99 | } 100 | 101 | // fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read. 102 | func (b *bitReader) fillFastStart() { 103 | // Do single re-slice to avoid bounds checks. 104 | b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) 105 | b.bitsRead = 0 106 | b.off -= 8 107 | } 108 | 109 | // finished returns true if all bits have been read from the bit stream. 110 | func (b *bitReader) finished() bool { 111 | return b.bitsRead >= 64 && b.off == 0 112 | } 113 | 114 | // close the bitstream and returns an error if out-of-buffer reads occurred. 115 | func (b *bitReader) close() error { 116 | // Release reference. 117 | b.in = nil 118 | if b.bitsRead > 64 { 119 | return io.ErrUnexpectedEOF 120 | } 121 | return nil 122 | } 123 | -------------------------------------------------------------------------------- /fse/bytereader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Klaus Post. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | // Based on work Copyright (c) 2013, Yann Collet, released under BSD License. 5 | 6 | package fse 7 | 8 | // byteReader provides a byte reader that reads 9 | // little endian values from a byte stream. 10 | // The input stream is manually advanced. 11 | // The reader performs no bounds checks. 12 | type byteReader struct { 13 | b []byte 14 | off int 15 | } 16 | 17 | // init will initialize the reader and set the input. 18 | func (b *byteReader) init(in []byte) { 19 | b.b = in 20 | b.off = 0 21 | } 22 | 23 | // advance the stream b n bytes. 24 | func (b *byteReader) advance(n uint) { 25 | b.off += int(n) 26 | } 27 | 28 | // Uint32 returns a little endian uint32 starting at current offset. 29 | func (b byteReader) Uint32() uint32 { 30 | b2 := b.b[b.off:] 31 | b2 = b2[:4] 32 | v3 := uint32(b2[3]) 33 | v2 := uint32(b2[2]) 34 | v1 := uint32(b2[1]) 35 | v0 := uint32(b2[0]) 36 | return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) 37 | } 38 | 39 | // unread returns the unread portion of the input. 40 | func (b byteReader) unread() []byte { 41 | return b.b[b.off:] 42 | } 43 | 44 | // remain will return the number of bytes remaining. 45 | func (b byteReader) remain() int { 46 | return len(b.b) - b.off 47 | } 48 | -------------------------------------------------------------------------------- /fse/fuzz_test.go: -------------------------------------------------------------------------------- 1 | package fse 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/klauspost/compress/internal/fuzz" 9 | ) 10 | 11 | func FuzzCompress(f *testing.F) { 12 | fuzz.AddFromZip(f, "testdata/fse_compress.zip", fuzz.TypeGoFuzz, false) 13 | f.Fuzz(func(t *testing.T, buf0 []byte) { 14 | var s, s2 Scratch 15 | b, err := Compress(buf0, &s) 16 | if err != nil || b == nil { 17 | return 18 | } 19 | err = s.validateNorm() 20 | if err != nil { 21 | return 22 | } 23 | //Decompress 24 | got, err := Decompress(b, &s2) 25 | if err != nil || len(got) == 0 { 26 | return 27 | } 28 | if !bytes.Equal(buf0, got) { 29 | t.Fatal(fmt.Sprintln("FuzzCompress output mismatch\n", len(got), "org: \n", len(buf0))) 30 | } 31 | }) 32 | } 33 | 34 | func FuzzDecompress(f *testing.F) { 35 | // Input is mixed, but TypeGoFuzz will fall back to raw input. 36 | fuzz.AddFromZip(f, "testdata/fse_decompress.zip", fuzz.TypeGoFuzz, false) 37 | f.Fuzz(func(t *testing.T, buf0 []byte) { 38 | var s2 Scratch 39 | s2.DecompressLimit = 128 << 10 40 | //Decompress 41 | got, err := Decompress(buf0, &s2) 42 | if err != nil || len(got) == 0 { 43 | return 44 | } 45 | }) 46 | } 47 | -------------------------------------------------------------------------------- /fse/testdata/fse_compress.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/fse/testdata/fse_compress.zip -------------------------------------------------------------------------------- /fse/testdata/fse_decompress.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/fse/testdata/fse_decompress.zip -------------------------------------------------------------------------------- /gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd s2/cmd/_s2sx/ || exit 1 4 | go generate . 5 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/klauspost/compress 2 | 3 | go 1.22 4 | 5 | retract ( 6 | // https://github.com/klauspost/compress/pull/503 7 | v1.14.3 8 | v1.14.2 9 | v1.14.1 10 | ) 11 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/go.sum -------------------------------------------------------------------------------- /gzhttp/asserts_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 Klaus Post. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package gzhttp 6 | 7 | import ( 8 | "reflect" 9 | "testing" 10 | ) 11 | 12 | func assertEqual(t testing.TB, want, got interface{}) { 13 | t.Helper() 14 | if !reflect.DeepEqual(want, got) { 15 | t.Fatalf("want %#v, got %#v", want, got) 16 | } 17 | } 18 | 19 | func assertNotEqual(t testing.TB, want, got interface{}) { 20 | t.Helper() 21 | if reflect.DeepEqual(want, got) { 22 | t.Fatalf("did not want %#v, got %#v", want, got) 23 | } 24 | } 25 | 26 | func assertNil(t testing.TB, object interface{}) { 27 | if isNil(object) { 28 | return 29 | } 30 | t.Helper() 31 | t.Fatalf("Expected value to be nil.") 32 | } 33 | 34 | func assertNotNil(t testing.TB, object interface{}) { 35 | if !isNil(object) { 36 | return 37 | } 38 | t.Helper() 39 | t.Fatalf("Expected value not to be nil.") 40 | } 41 | 42 | // isNil checks if a specified object is nil or not, without Failing. 43 | func isNil(object interface{}) bool { 44 | if object == nil { 45 | return true 46 | } 47 | 48 | value := reflect.ValueOf(object) 49 | kind := value.Kind() 50 | isNilableKind := containsKind( 51 | []reflect.Kind{ 52 | reflect.Chan, reflect.Func, 53 | reflect.Interface, reflect.Map, 54 | reflect.Ptr, reflect.Slice}, 55 | kind) 56 | 57 | if isNilableKind && value.IsNil() { 58 | return true 59 | } 60 | 61 | return false 62 | } 63 | 64 | // containsKind checks if a specified kind in the slice of kinds. 65 | func containsKind(kinds []reflect.Kind, kind reflect.Kind) bool { 66 | for i := 0; i < len(kinds); i++ { 67 | if kind == kinds[i] { 68 | return true 69 | } 70 | } 71 | 72 | return false 73 | } 74 | -------------------------------------------------------------------------------- /gzhttp/examples_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2021 Klaus Post. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package gzhttp_test 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "log" 12 | "net/http" 13 | "net/http/httptest" 14 | 15 | "github.com/klauspost/compress/gzhttp" 16 | "github.com/klauspost/compress/gzip" 17 | ) 18 | 19 | func ExampleTransport() { 20 | // Get a client. 21 | client := http.Client{ 22 | // Wrap the transport: 23 | Transport: gzhttp.Transport(http.DefaultTransport), 24 | } 25 | 26 | resp, err := client.Get("https://google.com") 27 | if err != nil { 28 | fmt.Println(err) 29 | return 30 | } 31 | defer resp.Body.Close() 32 | 33 | body, _ := ioutil.ReadAll(resp.Body) 34 | fmt.Println("body:", string(body)) 35 | } 36 | 37 | func ExampleNewWrapper() { 38 | handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 39 | w.Header().Set("Content-Type", "text/plain") 40 | io.WriteString(w, "Hello, World, Welcome to the jungle...") 41 | }) 42 | handler2 := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 43 | io.WriteString(w, "Hello, Another World.................") 44 | }) 45 | 46 | // Create a reusable wrapper with custom options. 47 | wrapper, err := gzhttp.NewWrapper(gzhttp.MinSize(20), gzhttp.CompressionLevel(gzip.BestSpeed)) 48 | if err != nil { 49 | log.Fatalln(err) 50 | } 51 | server := http.NewServeMux() 52 | server.Handle("/a", wrapper(handler)) 53 | server.Handle("/b", wrapper(handler2)) 54 | 55 | test := httptest.NewServer(server) 56 | defer test.Close() 57 | 58 | resp, err := http.Get(test.URL + "/a") 59 | if err != nil { 60 | log.Fatalln(err) 61 | } 62 | content, _ := ioutil.ReadAll(resp.Body) 63 | fmt.Println(string(content)) 64 | 65 | resp, err = http.Get(test.URL + "/b") 66 | if err != nil { 67 | log.Fatalln(err) 68 | } 69 | content, _ = ioutil.ReadAll(resp.Body) 70 | fmt.Println(string(content)) 71 | // Output: 72 | // Hello, World, Welcome to the jungle... 73 | // Hello, Another World................. 74 | } 75 | -------------------------------------------------------------------------------- /gzhttp/writer/gzkp/gzkp.go: -------------------------------------------------------------------------------- 1 | // package gzkp provides gzip compression through github.com/klauspost/compress/gzip. 2 | 3 | package gzkp 4 | 5 | import ( 6 | "io" 7 | "sync" 8 | 9 | "github.com/klauspost/compress/gzhttp/writer" 10 | "github.com/klauspost/compress/gzip" 11 | ) 12 | 13 | // gzipWriterPools stores a sync.Pool for each compression level for reuse of 14 | // gzip.Writers. Use poolIndex to covert a compression level to an index into 15 | // gzipWriterPools. 16 | var gzipWriterPools [gzip.BestCompression - gzip.StatelessCompression + 1]*sync.Pool 17 | 18 | func init() { 19 | for i := gzip.StatelessCompression; i <= gzip.BestCompression; i++ { 20 | addLevelPool(i) 21 | } 22 | } 23 | 24 | // poolIndex maps a compression level to its index into gzipWriterPools. It 25 | // assumes that level is a valid gzip compression level. 26 | func poolIndex(level int) int { 27 | if level > gzip.BestCompression { 28 | level = gzip.BestCompression 29 | } 30 | if level < gzip.StatelessCompression { 31 | level = gzip.BestSpeed 32 | } 33 | return level - gzip.StatelessCompression 34 | } 35 | 36 | func addLevelPool(level int) { 37 | gzipWriterPools[poolIndex(level)] = &sync.Pool{ 38 | New: func() interface{} { 39 | // NewWriterLevel only returns error on a bad level, we are guaranteeing 40 | // that this will be a valid level so it is okay to ignore the returned 41 | // error. 42 | w, _ := gzip.NewWriterLevel(nil, level) 43 | return w 44 | }, 45 | } 46 | } 47 | 48 | type pooledWriter struct { 49 | *gzip.Writer 50 | index int 51 | } 52 | 53 | func (pw *pooledWriter) Close() error { 54 | err := pw.Writer.Close() 55 | gzipWriterPools[pw.index].Put(pw.Writer) 56 | pw.Writer = nil 57 | return err 58 | } 59 | 60 | func NewWriter(w io.Writer, level int) writer.GzipWriter { 61 | index := poolIndex(level) 62 | gzw := gzipWriterPools[index].Get().(*gzip.Writer) 63 | gzw.Reset(w) 64 | return &pooledWriter{ 65 | Writer: gzw, 66 | index: index, 67 | } 68 | } 69 | 70 | // SetHeader will override the gzip header on pw. 71 | func (pw *pooledWriter) SetHeader(h writer.Header) { 72 | pw.Name = h.Name 73 | pw.Extra = h.Extra 74 | pw.Comment = h.Comment 75 | pw.ModTime = h.ModTime 76 | pw.OS = h.OS 77 | } 78 | 79 | func Levels() (min, max int) { 80 | return gzip.StatelessCompression, gzip.BestCompression 81 | } 82 | 83 | func ImplementationInfo() string { 84 | return "klauspost/compress/gzip" 85 | } 86 | -------------------------------------------------------------------------------- /gzhttp/writer/gzkp/gzkp_test.go: -------------------------------------------------------------------------------- 1 | package gzkp 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "testing" 7 | ) 8 | 9 | func TestGzipDoubleClose(t *testing.T) { 10 | // reset the pool for the default compression so we can make sure duplicates 11 | // aren't added back by double close 12 | addLevelPool(gzip.DefaultCompression) 13 | 14 | w := bytes.NewBufferString("") 15 | writer := NewWriter(w, gzip.DefaultCompression) 16 | writer.Close() 17 | 18 | // the second close shouldn't have added the same writer 19 | // so we pull out 2 writers from the pool and make sure they're different 20 | w1 := gzipWriterPools[poolIndex(gzip.DefaultCompression)].Get() 21 | w2 := gzipWriterPools[poolIndex(gzip.DefaultCompression)].Get() 22 | 23 | if w1 == w2 { 24 | t.Fatal("got same writer") 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /gzhttp/writer/gzstd/stdlib.go: -------------------------------------------------------------------------------- 1 | // package gzstd provides gzip compression through the standard library. 2 | 3 | package gzstd 4 | 5 | import ( 6 | "compress/gzip" 7 | "io" 8 | "sync" 9 | 10 | "github.com/klauspost/compress/gzhttp/writer" 11 | ) 12 | 13 | // gzipWriterPools stores a sync.Pool for each compression level for reuse of 14 | // gzip.Writers. Use poolIndex to covert a compression level to an index into 15 | // gzipWriterPools. 16 | var gzipWriterPools [gzip.BestCompression - gzip.HuffmanOnly + 1]*sync.Pool 17 | 18 | func init() { 19 | for i := gzip.HuffmanOnly; i <= gzip.BestCompression; i++ { 20 | addLevelPool(i) 21 | } 22 | } 23 | 24 | // poolIndex maps a compression level to its index into gzipWriterPools. It 25 | // assumes that level is a valid gzip compression level. 26 | func poolIndex(level int) int { 27 | if level > gzip.BestCompression { 28 | level = gzip.BestCompression 29 | } 30 | if level < gzip.HuffmanOnly { 31 | level = gzip.BestSpeed 32 | } 33 | return level - gzip.HuffmanOnly 34 | } 35 | 36 | func addLevelPool(level int) { 37 | gzipWriterPools[poolIndex(level)] = &sync.Pool{ 38 | New: func() interface{} { 39 | // NewWriterLevel only returns error on a bad level, we are guaranteeing 40 | // that this will be a valid level so it is okay to ignore the returned 41 | // error. 42 | w, _ := gzip.NewWriterLevel(nil, level) 43 | return w 44 | }, 45 | } 46 | } 47 | 48 | type pooledWriter struct { 49 | *gzip.Writer 50 | index int 51 | } 52 | 53 | func (pw *pooledWriter) Close() error { 54 | err := pw.Writer.Close() 55 | gzipWriterPools[pw.index].Put(pw.Writer) 56 | pw.Writer = nil 57 | return err 58 | } 59 | 60 | func NewWriter(w io.Writer, level int) writer.GzipWriter { 61 | index := poolIndex(level) 62 | gzw := gzipWriterPools[index].Get().(*gzip.Writer) 63 | gzw.Reset(w) 64 | return &pooledWriter{ 65 | Writer: gzw, 66 | index: index, 67 | } 68 | } 69 | 70 | // SetHeader will override the gzip header on pw. 71 | func (pw *pooledWriter) SetHeader(h writer.Header) { 72 | pw.Name = h.Name 73 | pw.Extra = h.Extra 74 | pw.Comment = h.Comment 75 | pw.ModTime = h.ModTime 76 | pw.OS = h.OS 77 | } 78 | 79 | func Levels() (min, max int) { 80 | return gzip.HuffmanOnly, gzip.BestCompression 81 | } 82 | 83 | func ImplementationInfo() string { 84 | return "compress/gzip" 85 | } 86 | -------------------------------------------------------------------------------- /gzhttp/writer/gzstd/stdlib_test.go: -------------------------------------------------------------------------------- 1 | package gzstd 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "testing" 7 | ) 8 | 9 | func TestGzipDoubleClose(t *testing.T) { 10 | // reset the pool for the default compression so we can make sure duplicates 11 | // aren't added back by double close 12 | addLevelPool(gzip.DefaultCompression) 13 | 14 | w := bytes.NewBufferString("") 15 | writer := NewWriter(w, gzip.DefaultCompression) 16 | writer.Close() 17 | 18 | // the second close shouldn't have added the same writer 19 | // so we pull out 2 writers from the pool and make sure they're different 20 | w1 := gzipWriterPools[poolIndex(gzip.DefaultCompression)].Get() 21 | w2 := gzipWriterPools[poolIndex(gzip.DefaultCompression)].Get() 22 | 23 | if w1 == w2 { 24 | t.Fatal("got same writer") 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /gzhttp/writer/interface.go: -------------------------------------------------------------------------------- 1 | package writer 2 | 3 | import ( 4 | "io" 5 | "time" 6 | ) 7 | 8 | // GzipWriter implements the functions needed for compressing content. 9 | type GzipWriter interface { 10 | Write(p []byte) (int, error) 11 | Close() error 12 | Flush() error 13 | } 14 | 15 | // GzipWriterExt implements the functions needed for compressing content 16 | // and optional extensions. 17 | type GzipWriterExt interface { 18 | GzipWriter 19 | 20 | // SetHeader will populate header fields with non-nil values in h. 21 | SetHeader(h Header) 22 | } 23 | 24 | // Header is a gzip header. 25 | type Header struct { 26 | Comment string // comment 27 | Extra []byte // "extra data" 28 | ModTime time.Time // modification time 29 | Name string // file name 30 | OS byte // operating system type 31 | } 32 | 33 | // GzipWriterFactory contains the information needed for custom gzip implementations. 34 | type GzipWriterFactory struct { 35 | // Must return the minimum and maximum supported level. 36 | Levels func() (min, max int) 37 | 38 | // New must return a new GzipWriter. 39 | // level will always be within the return limits above. 40 | New func(writer io.Writer, level int) GzipWriter 41 | } 42 | -------------------------------------------------------------------------------- /gzip/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package gzip_test 6 | 7 | import ( 8 | "bytes" 9 | "compress/gzip" 10 | "fmt" 11 | "io" 12 | "log" 13 | "os" 14 | "time" 15 | ) 16 | 17 | func Example_writerReader() { 18 | var buf bytes.Buffer 19 | zw := gzip.NewWriter(&buf) 20 | 21 | // Setting the Header fields is optional. 22 | zw.Name = "a-new-hope.txt" 23 | zw.Comment = "an epic space opera by George Lucas" 24 | zw.ModTime = time.Date(1977, time.May, 25, 0, 0, 0, 0, time.UTC) 25 | 26 | _, err := zw.Write([]byte("A long time ago in a galaxy far, far away...")) 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | 31 | if err := zw.Close(); err != nil { 32 | log.Fatal(err) 33 | } 34 | 35 | zr, err := gzip.NewReader(&buf) 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | 40 | fmt.Printf("Name: %s\nComment: %s\nModTime: %s\n\n", zr.Name, zr.Comment, zr.ModTime.UTC()) 41 | 42 | if _, err := io.Copy(os.Stdout, zr); err != nil { 43 | log.Fatal(err) 44 | } 45 | 46 | if err := zr.Close(); err != nil { 47 | log.Fatal(err) 48 | } 49 | 50 | // Output: 51 | // Name: a-new-hope.txt 52 | // Comment: an epic space opera by George Lucas 53 | // ModTime: 1977-05-25 00:00:00 +0000 UTC 54 | // 55 | // A long time ago in a galaxy far, far away... 56 | } 57 | 58 | func ExampleReader_Multistream() { 59 | var buf bytes.Buffer 60 | zw := gzip.NewWriter(&buf) 61 | 62 | var files = []struct { 63 | name string 64 | comment string 65 | modTime time.Time 66 | data string 67 | }{ 68 | {"file-1.txt", "file-header-1", time.Date(2006, time.February, 1, 3, 4, 5, 0, time.UTC), "Hello Gophers - 1"}, 69 | {"file-2.txt", "file-header-2", time.Date(2007, time.March, 2, 4, 5, 6, 1, time.UTC), "Hello Gophers - 2"}, 70 | } 71 | 72 | for _, file := range files { 73 | zw.Name = file.name 74 | zw.Comment = file.comment 75 | zw.ModTime = file.modTime 76 | 77 | if _, err := zw.Write([]byte(file.data)); err != nil { 78 | log.Fatal(err) 79 | } 80 | 81 | if err := zw.Close(); err != nil { 82 | log.Fatal(err) 83 | } 84 | 85 | zw.Reset(&buf) 86 | } 87 | 88 | zr, err := gzip.NewReader(&buf) 89 | if err != nil { 90 | log.Fatal(err) 91 | } 92 | 93 | for { 94 | zr.Multistream(false) 95 | fmt.Printf("Name: %s\nComment: %s\nModTime: %s\n\n", zr.Name, zr.Comment, zr.ModTime.UTC()) 96 | 97 | if _, err := io.Copy(os.Stdout, zr); err != nil { 98 | log.Fatal(err) 99 | } 100 | 101 | fmt.Print("\n\n") 102 | 103 | err = zr.Reset(&buf) 104 | if err == io.EOF { 105 | break 106 | } 107 | if err != nil { 108 | log.Fatal(err) 109 | } 110 | } 111 | 112 | if err := zr.Close(); err != nil { 113 | log.Fatal(err) 114 | } 115 | 116 | // Output: 117 | // Name: file-1.txt 118 | // Comment: file-header-1 119 | // ModTime: 2006-02-01 03:04:05 +0000 UTC 120 | // 121 | // Hello Gophers - 1 122 | // 123 | // Name: file-2.txt 124 | // Comment: file-header-2 125 | // ModTime: 2007-03-02 04:05:06 +0000 UTC 126 | // 127 | // Hello Gophers - 2 128 | } 129 | -------------------------------------------------------------------------------- /gzip/testdata/issue6550.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/gzip/testdata/issue6550.gz -------------------------------------------------------------------------------- /huff0/.gitignore: -------------------------------------------------------------------------------- 1 | /huff0-fuzz.zip 2 | -------------------------------------------------------------------------------- /huff0/_generate/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/klauspost/compress/s2/_generate 2 | 3 | go 1.22.0 4 | 5 | toolchain go1.22.4 6 | 7 | require ( 8 | github.com/klauspost/compress v1.15.15 9 | github.com/mmcloughlin/avo v0.6.0 10 | ) 11 | 12 | require ( 13 | golang.org/x/mod v0.21.0 // indirect 14 | golang.org/x/sync v0.8.0 // indirect 15 | golang.org/x/tools v0.25.0 // indirect 16 | ) 17 | 18 | replace github.com/klauspost/compress => ../.. 19 | -------------------------------------------------------------------------------- /huff0/_generate/go.sum: -------------------------------------------------------------------------------- 1 | github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY= 2 | github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8= 3 | golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= 4 | golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= 5 | golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= 6 | golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 7 | golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= 8 | golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= 9 | -------------------------------------------------------------------------------- /huff0/bitwriter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Klaus Post. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | // Based on work Copyright (c) 2013, Yann Collet, released under BSD License. 5 | 6 | package huff0 7 | 8 | // bitWriter will write bits. 9 | // First bit will be LSB of the first byte of output. 10 | type bitWriter struct { 11 | bitContainer uint64 12 | nBits uint8 13 | out []byte 14 | } 15 | 16 | // addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated. 17 | // It will not check if there is space for them, so the caller must ensure that it has flushed recently. 18 | func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { 19 | b.bitContainer |= uint64(value) << (b.nBits & 63) 20 | b.nBits += bits 21 | } 22 | 23 | // encSymbol will add up to 16 bits. value may not contain more set bits than indicated. 24 | // It will not check if there is space for them, so the caller must ensure that it has flushed recently. 25 | func (b *bitWriter) encSymbol(ct cTable, symbol byte) { 26 | enc := ct[symbol] 27 | b.bitContainer |= uint64(enc.val) << (b.nBits & 63) 28 | if false { 29 | if enc.nBits == 0 { 30 | panic("nbits 0") 31 | } 32 | } 33 | b.nBits += enc.nBits 34 | } 35 | 36 | // encTwoSymbols will add up to 32 bits. value may not contain more set bits than indicated. 37 | // It will not check if there is space for them, so the caller must ensure that it has flushed recently. 38 | func (b *bitWriter) encTwoSymbols(ct cTable, av, bv byte) { 39 | encA := ct[av] 40 | encB := ct[bv] 41 | sh := b.nBits & 63 42 | combined := uint64(encA.val) | (uint64(encB.val) << (encA.nBits & 63)) 43 | b.bitContainer |= combined << sh 44 | if false { 45 | if encA.nBits == 0 { 46 | panic("nbitsA 0") 47 | } 48 | if encB.nBits == 0 { 49 | panic("nbitsB 0") 50 | } 51 | } 52 | b.nBits += encA.nBits + encB.nBits 53 | } 54 | 55 | // encFourSymbols adds up to 32 bits from four symbols. 56 | // It will not check if there is space for them, 57 | // so the caller must ensure that b has been flushed recently. 58 | func (b *bitWriter) encFourSymbols(encA, encB, encC, encD cTableEntry) { 59 | bitsA := encA.nBits 60 | bitsB := bitsA + encB.nBits 61 | bitsC := bitsB + encC.nBits 62 | bitsD := bitsC + encD.nBits 63 | combined := uint64(encA.val) | 64 | (uint64(encB.val) << (bitsA & 63)) | 65 | (uint64(encC.val) << (bitsB & 63)) | 66 | (uint64(encD.val) << (bitsC & 63)) 67 | b.bitContainer |= combined << (b.nBits & 63) 68 | b.nBits += bitsD 69 | } 70 | 71 | // flush32 will flush out, so there are at least 32 bits available for writing. 72 | func (b *bitWriter) flush32() { 73 | if b.nBits < 32 { 74 | return 75 | } 76 | b.out = append(b.out, 77 | byte(b.bitContainer), 78 | byte(b.bitContainer>>8), 79 | byte(b.bitContainer>>16), 80 | byte(b.bitContainer>>24)) 81 | b.nBits -= 32 82 | b.bitContainer >>= 32 83 | } 84 | 85 | // flushAlign will flush remaining full bytes and align to next byte boundary. 86 | func (b *bitWriter) flushAlign() { 87 | nbBytes := (b.nBits + 7) >> 3 88 | for i := uint8(0); i < nbBytes; i++ { 89 | b.out = append(b.out, byte(b.bitContainer>>(i*8))) 90 | } 91 | b.nBits = 0 92 | b.bitContainer = 0 93 | } 94 | 95 | // close will write the alignment bit and write the final byte(s) 96 | // to the output. 97 | func (b *bitWriter) close() { 98 | // End mark 99 | b.addBits16Clean(1, 1) 100 | // flush until next byte. 101 | b.flushAlign() 102 | } 103 | -------------------------------------------------------------------------------- /huff0/fuzz_test.go: -------------------------------------------------------------------------------- 1 | package huff0 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/klauspost/compress/internal/fuzz" 9 | ) 10 | 11 | func FuzzCompress(f *testing.F) { 12 | fuzz.AddFromZip(f, "testdata/fse_compress.zip", fuzz.TypeRaw, false) 13 | f.Fuzz(func(t *testing.T, buf0 []byte) { 14 | //use of Compress1X 15 | var s Scratch 16 | if len(buf0) > BlockSizeMax { 17 | buf0 = buf0[:BlockSizeMax] 18 | } 19 | EstimateSizes(buf0, &s) 20 | b, re, err := Compress1X(buf0, &s) 21 | s.validateTable(s.cTable) 22 | s.canUseTable(s.cTable) 23 | if err != nil || b == nil { 24 | return 25 | } 26 | 27 | min := s.minSize(len(buf0)) 28 | 29 | if len(s.OutData) < min { 30 | t.Errorf("FuzzCompress: output data length (%d) below shannon limit (%d)", len(s.OutData), min) 31 | } 32 | if len(s.OutTable) == 0 { 33 | t.Error("FuzzCompress: got no table definition") 34 | } 35 | if re { 36 | t.Error("FuzzCompress: claimed to have re-used.") 37 | } 38 | if len(s.OutData) == 0 { 39 | t.Error("FuzzCompress: got no data output") 40 | } 41 | 42 | dec, remain, err := ReadTable(b, nil) 43 | 44 | //use of Decompress1X 45 | out, err := dec.Decompress1X(remain) 46 | if err != nil || len(out) == 0 { 47 | return 48 | } 49 | if !bytes.Equal(out, buf0) { 50 | t.Fatal(fmt.Sprintln("FuzzCompressX1 output mismatch\n", len(out), "org: \n", len(buf0))) 51 | } 52 | 53 | //use of Compress4X 54 | s.Reuse = ReusePolicyAllow 55 | b, reUsed, err := Compress4X(buf0, &s) 56 | if err != nil || b == nil { 57 | return 58 | } 59 | remain = b 60 | if !reUsed { 61 | dec, remain, err = ReadTable(b, dec) 62 | if err != nil { 63 | return 64 | } 65 | } 66 | //use of Decompress4X 67 | out, err = dec.Decompress4X(remain, len(buf0)) 68 | if err != nil || out == nil { 69 | return 70 | } 71 | if !bytes.Equal(out, buf0) { 72 | t.Fatal(fmt.Sprintln("FuzzCompressX4 output mismatch: ", len(out), ", org: ", len(buf0))) 73 | } 74 | }) 75 | } 76 | 77 | func FuzzDecompress1x(f *testing.F) { 78 | fuzz.AddFromZip(f, "testdata/huff0_decompress1x.zip", fuzz.TypeRaw, false) 79 | f.Fuzz(func(t *testing.T, buf0 []byte) { 80 | var s Scratch 81 | _, remain, err := ReadTable(buf0, &s) 82 | if err != nil { 83 | return 84 | } 85 | out, err := s.Decompress1X(remain) 86 | if err != nil || out == nil { 87 | return 88 | } 89 | }) 90 | } 91 | -------------------------------------------------------------------------------- /huff0/testdata/decompress1x_regression.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/huff0/testdata/decompress1x_regression.zip -------------------------------------------------------------------------------- /huff0/testdata/fse_compress.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/huff0/testdata/fse_compress.zip -------------------------------------------------------------------------------- /huff0/testdata/huff0_decompress1x.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/huff0/testdata/huff0_decompress1x.zip -------------------------------------------------------------------------------- /huff0/testdata/regression.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/huff0/testdata/regression.zip -------------------------------------------------------------------------------- /internal/cpuinfo/cpuinfo.go: -------------------------------------------------------------------------------- 1 | // Package cpuinfo gives runtime info about the current CPU. 2 | // 3 | // This is a very limited module meant for use internally 4 | // in this project. For more versatile solution check 5 | // https://github.com/klauspost/cpuid. 6 | package cpuinfo 7 | 8 | // HasBMI1 checks whether an x86 CPU supports the BMI1 extension. 9 | func HasBMI1() bool { 10 | return hasBMI1 11 | } 12 | 13 | // HasBMI2 checks whether an x86 CPU supports the BMI2 extension. 14 | func HasBMI2() bool { 15 | return hasBMI2 16 | } 17 | 18 | // DisableBMI2 will disable BMI2, for testing purposes. 19 | // Call returned function to restore previous state. 20 | func DisableBMI2() func() { 21 | old := hasBMI2 22 | hasBMI2 = false 23 | return func() { 24 | hasBMI2 = old 25 | } 26 | } 27 | 28 | // HasBMI checks whether an x86 CPU supports both BMI1 and BMI2 extensions. 29 | func HasBMI() bool { 30 | return HasBMI1() && HasBMI2() 31 | } 32 | 33 | var hasBMI1 bool 34 | var hasBMI2 bool 35 | -------------------------------------------------------------------------------- /internal/cpuinfo/cpuinfo_amd64.go: -------------------------------------------------------------------------------- 1 | //go:build amd64 && !appengine && !noasm && gc 2 | // +build amd64,!appengine,!noasm,gc 3 | 4 | package cpuinfo 5 | 6 | // go:noescape 7 | func x86extensions() (bmi1, bmi2 bool) 8 | 9 | func init() { 10 | hasBMI1, hasBMI2 = x86extensions() 11 | } 12 | -------------------------------------------------------------------------------- /internal/cpuinfo/cpuinfo_amd64.s: -------------------------------------------------------------------------------- 1 | // +build !appengine 2 | // +build gc 3 | // +build !noasm 4 | 5 | #include "textflag.h" 6 | #include "funcdata.h" 7 | #include "go_asm.h" 8 | 9 | TEXT ·x86extensions(SB), NOSPLIT, $0 10 | // 1. determine max EAX value 11 | XORQ AX, AX 12 | CPUID 13 | 14 | CMPQ AX, $7 15 | JB unsupported 16 | 17 | // 2. EAX = 7, ECX = 0 --- see Table 3-8 "Information Returned by CPUID Instruction" 18 | MOVQ $7, AX 19 | MOVQ $0, CX 20 | CPUID 21 | 22 | BTQ $3, BX // bit 3 = BMI1 23 | SETCS AL 24 | 25 | BTQ $8, BX // bit 8 = BMI2 26 | SETCS AH 27 | 28 | MOVB AL, bmi1+0(FP) 29 | MOVB AH, bmi2+1(FP) 30 | RET 31 | 32 | unsupported: 33 | XORQ AX, AX 34 | MOVB AL, bmi1+0(FP) 35 | MOVB AL, bmi2+1(FP) 36 | RET 37 | -------------------------------------------------------------------------------- /internal/godebug/godebug.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package godebug makes the simplified settings in the $GODEBUG environment variable 6 | // available to packages. 7 | // Needed since internal/godebug is not available here. 8 | package godebug 9 | 10 | import "os" 11 | 12 | func Get(key string) string { 13 | s := os.Getenv("GODEBUG") 14 | if s == "" { 15 | return "" 16 | } 17 | // Scan the string backward so that later settings are used 18 | // and earlier settings are ignored. 19 | // Note that a forward scan would cause cached values 20 | // to temporarily use the ignored value before being 21 | // updated to the "correct" one. 22 | end := len(s) 23 | eq := -1 24 | for i := end - 1; i >= -1; i-- { 25 | if i == -1 || s[i] == ',' { 26 | if eq >= 0 { 27 | name, arg := s[i+1:eq], s[eq+1:end] 28 | if name == key { 29 | for j := 0; j < len(arg); j++ { 30 | if arg[j] == '#' { 31 | return arg[:j] 32 | } 33 | } 34 | return arg 35 | } 36 | } 37 | eq = -1 38 | end = i 39 | } else if s[i] == '=' { 40 | eq = i 41 | } 42 | } 43 | return "" 44 | } 45 | -------------------------------------------------------------------------------- /internal/le/le.go: -------------------------------------------------------------------------------- 1 | package le 2 | 3 | type Indexer interface { 4 | int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 5 | } 6 | -------------------------------------------------------------------------------- /internal/le/unsafe_disabled.go: -------------------------------------------------------------------------------- 1 | //go:build !(amd64 || arm64 || ppc64le || riscv64) || nounsafe || purego || appengine 2 | 3 | package le 4 | 5 | import ( 6 | "encoding/binary" 7 | ) 8 | 9 | // Load8 will load from b at index i. 10 | func Load8[I Indexer](b []byte, i I) byte { 11 | return b[i] 12 | } 13 | 14 | // Load16 will load from b at index i. 15 | func Load16[I Indexer](b []byte, i I) uint16 { 16 | return binary.LittleEndian.Uint16(b[i:]) 17 | } 18 | 19 | // Load32 will load from b at index i. 20 | func Load32[I Indexer](b []byte, i I) uint32 { 21 | return binary.LittleEndian.Uint32(b[i:]) 22 | } 23 | 24 | // Load64 will load from b at index i. 25 | func Load64[I Indexer](b []byte, i I) uint64 { 26 | return binary.LittleEndian.Uint64(b[i:]) 27 | } 28 | 29 | // Store16 will store v at b. 30 | func Store16(b []byte, v uint16) { 31 | binary.LittleEndian.PutUint16(b, v) 32 | } 33 | 34 | // Store32 will store v at b. 35 | func Store32(b []byte, v uint32) { 36 | binary.LittleEndian.PutUint32(b, v) 37 | } 38 | 39 | // Store64 will store v at b. 40 | func Store64(b []byte, v uint64) { 41 | binary.LittleEndian.PutUint64(b, v) 42 | } 43 | -------------------------------------------------------------------------------- /internal/le/unsafe_enabled.go: -------------------------------------------------------------------------------- 1 | // We enable 64 bit LE platforms: 2 | 3 | //go:build (amd64 || arm64 || ppc64le || riscv64) && !nounsafe && !purego && !appengine 4 | 5 | package le 6 | 7 | import ( 8 | "unsafe" 9 | ) 10 | 11 | // Load8 will load from b at index i. 12 | func Load8[I Indexer](b []byte, i I) byte { 13 | //return binary.LittleEndian.Uint16(b[i:]) 14 | //return *(*uint16)(unsafe.Pointer(&b[i])) 15 | return *(*byte)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) 16 | } 17 | 18 | // Load16 will load from b at index i. 19 | func Load16[I Indexer](b []byte, i I) uint16 { 20 | //return binary.LittleEndian.Uint16(b[i:]) 21 | //return *(*uint16)(unsafe.Pointer(&b[i])) 22 | return *(*uint16)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) 23 | } 24 | 25 | // Load32 will load from b at index i. 26 | func Load32[I Indexer](b []byte, i I) uint32 { 27 | //return binary.LittleEndian.Uint32(b[i:]) 28 | //return *(*uint32)(unsafe.Pointer(&b[i])) 29 | return *(*uint32)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) 30 | } 31 | 32 | // Load64 will load from b at index i. 33 | func Load64[I Indexer](b []byte, i I) uint64 { 34 | //return binary.LittleEndian.Uint64(b[i:]) 35 | //return *(*uint64)(unsafe.Pointer(&b[i])) 36 | return *(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) 37 | } 38 | 39 | // Store16 will store v at b. 40 | func Store16(b []byte, v uint16) { 41 | //binary.LittleEndian.PutUint16(b, v) 42 | *(*uint16)(unsafe.Pointer(unsafe.SliceData(b))) = v 43 | } 44 | 45 | // Store32 will store v at b. 46 | func Store32(b []byte, v uint32) { 47 | //binary.LittleEndian.PutUint32(b, v) 48 | *(*uint32)(unsafe.Pointer(unsafe.SliceData(b))) = v 49 | } 50 | 51 | // Store64 will store v at b. 52 | func Store64(b []byte, v uint64) { 53 | //binary.LittleEndian.PutUint64(b, v) 54 | *(*uint64)(unsafe.Pointer(unsafe.SliceData(b))) = v 55 | } 56 | -------------------------------------------------------------------------------- /internal/lz4ref/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Pierre Curto 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of xxHash nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -------------------------------------------------------------------------------- /internal/lz4ref/errors.go: -------------------------------------------------------------------------------- 1 | package lz4ref 2 | 3 | type Error string 4 | 5 | func (e Error) Error() string { return string(e) } 6 | 7 | const ( 8 | ErrInvalidSourceShortBuffer Error = "lz4: invalid source or destination buffer too short" 9 | ) 10 | -------------------------------------------------------------------------------- /internal/race/norace.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build !race 6 | 7 | package race 8 | 9 | func ReadSlice[T any](s []T) { 10 | } 11 | 12 | func WriteSlice[T any](s []T) { 13 | } 14 | -------------------------------------------------------------------------------- /internal/race/race.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build race 6 | 7 | package race 8 | 9 | import ( 10 | "runtime" 11 | "unsafe" 12 | ) 13 | 14 | func ReadSlice[T any](s []T) { 15 | if len(s) == 0 { 16 | return 17 | } 18 | runtime.RaceReadRange(unsafe.Pointer(&s[0]), len(s)*int(unsafe.Sizeof(s[0]))) 19 | } 20 | 21 | func WriteSlice[T any](s []T) { 22 | if len(s) == 0 { 23 | return 24 | } 25 | runtime.RaceWriteRange(unsafe.Pointer(&s[0]), len(s)*int(unsafe.Sizeof(s[0]))) 26 | } 27 | -------------------------------------------------------------------------------- /internal/snapref/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /ossfuzz/cmd/setup_dicts.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "bytes" 19 | "flag" 20 | "fmt" 21 | "io" 22 | "os" 23 | "strings" 24 | "text/template" 25 | 26 | "github.com/klauspost/compress/zip" 27 | ) 28 | 29 | var ( 30 | dictPath = flag.String("dict-path", "", "dict path") 31 | outputFile = flag.String("output-file", "", "output file") 32 | ) 33 | 34 | func main() { 35 | flag.Parse() 36 | if *dictPath == "" { 37 | panic("Need a dict path") 38 | } 39 | if *outputFile == "" { 40 | panic("Need an output file") 41 | } 42 | dicts := getFuzzDicts(*dictPath) 43 | 44 | t, err := template.New("todos").Parse(` 45 | package zstd 46 | var fuzzDicts = make([][]byte, 0) 47 | func init() { 48 | {{range $val := .}} 49 | fuzzDicts = append(fuzzDicts, {{$val}}) 50 | {{end}} 51 | } 52 | `) 53 | if err != nil { 54 | panic(err) 55 | } 56 | f, err := os.Create(*outputFile) 57 | err = t.Execute(f, dicts) 58 | if err != nil { 59 | panic(err) 60 | } 61 | f.Close() 62 | } 63 | 64 | func getFuzzDicts(path string) []string { 65 | data, err := os.ReadFile(path) 66 | if err != nil { 67 | panic(err) 68 | } 69 | zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) 70 | if err != nil { 71 | panic(err) 72 | } 73 | var dicts [][]byte 74 | for _, tt := range zr.File { 75 | if !strings.HasSuffix(tt.Name, ".dict") { 76 | continue 77 | } 78 | func() { 79 | r, err := tt.Open() 80 | if err != nil { 81 | panic(err) 82 | } 83 | defer r.Close() 84 | in, err := io.ReadAll(r) 85 | if err != nil { 86 | panic(err) 87 | } 88 | dicts = append(dicts, in) 89 | }() 90 | } 91 | stringDicts := make([]string, 0) 92 | for _, d := range dicts { 93 | stringedArray := fmt.Sprintf("%v", d) 94 | withComma := strings.Replace(stringedArray, " ", ", ", -1) 95 | withClosingBracket := strings.Replace(withComma, "]", "}", -1) 96 | withOpenBracket := strings.Replace(withClosingBracket, "[", "[]byte{", -1) 97 | stringDicts = append(stringDicts, withOpenBracket) 98 | } 99 | return stringDicts 100 | } 101 | -------------------------------------------------------------------------------- /s2/.gitignore: -------------------------------------------------------------------------------- 1 | testdata/bench 2 | 3 | # These explicitly listed benchmark data files are for an obsolete version of 4 | # snappy_test.go. 5 | testdata/alice29.txt 6 | testdata/asyoulik.txt 7 | testdata/fireworks.jpeg 8 | testdata/geo.protodata 9 | testdata/html 10 | testdata/html_x_4 11 | testdata/kppkn.gtb 12 | testdata/lcet10.txt 13 | testdata/paper-100k.pdf 14 | testdata/plrabn12.txt 15 | testdata/urls.10K 16 | -------------------------------------------------------------------------------- /s2/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. 2 | Copyright (c) 2019 Klaus Post. All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following disclaimer 12 | in the documentation and/or other materials provided with the 13 | distribution. 14 | * Neither the name of Google Inc. nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /s2/_generate/cleanup.go: -------------------------------------------------------------------------------- 1 | //go:build custom 2 | // +build custom 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | "flag" 9 | "log" 10 | "os" 11 | 12 | "github.com/klauspost/asmfmt" 13 | ) 14 | 15 | func main() { 16 | flag.Parse() 17 | args := flag.Args() 18 | for _, file := range args { 19 | data, err := os.ReadFile(file) 20 | if err != nil { 21 | log.Fatalln(err) 22 | } 23 | data = bytes.Replace(data, []byte("\t// #"), []byte("#"), -1) 24 | data, err = asmfmt.Format(bytes.NewBuffer(data)) 25 | if err != nil { 26 | log.Fatalln(err) 27 | } 28 | err = os.WriteFile(file, data, os.ModePerm) 29 | if err != nil { 30 | log.Fatalln(err) 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /s2/_generate/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/klauspost/compress/s2/_generate 2 | 3 | go 1.22.0 4 | 5 | toolchain go1.22.4 6 | 7 | require ( 8 | github.com/klauspost/asmfmt v1.3.2 9 | github.com/mmcloughlin/avo v0.6.0 10 | ) 11 | 12 | require ( 13 | golang.org/x/mod v0.21.0 // indirect 14 | golang.org/x/sync v0.8.0 // indirect 15 | golang.org/x/tools v0.25.0 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /s2/_generate/go.sum: -------------------------------------------------------------------------------- 1 | github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= 2 | github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= 3 | github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY= 4 | github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8= 5 | golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= 6 | golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= 7 | golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= 8 | golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 9 | golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= 10 | golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= 11 | -------------------------------------------------------------------------------- /s2/cmd/_s2sx/gen_other.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | // +build !windows 3 | 4 | package main 5 | 6 | //go:generate ./gensfx.sh 7 | -------------------------------------------------------------------------------- /s2/cmd/_s2sx/gen_windows.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | //go:generate .\gensfx.cmd 4 | -------------------------------------------------------------------------------- /s2/cmd/_s2sx/gensfx.cmd: -------------------------------------------------------------------------------- 1 | go build ../s2c 2 | 3 | DEL /Q sfx-exe\* 4 | 5 | SET GOOS=linux 6 | SET GOARCH=amd64 7 | SET BUILDFLAGS=-trimpath -ldflags="-s -w" 8 | go build %BUILDFLAGS% -o ./sfx-exe/%GOOS%-%GOARCH% ./_unpack/main.go 9 | SET GOARCH=arm64 10 | go build %BUILDFLAGS% -o ./sfx-exe/%GOOS%-%GOARCH% ./_unpack/main.go 11 | SET GOARCH=arm 12 | go build %BUILDFLAGS% -o ./sfx-exe/%GOOS%-%GOARCH% ./_unpack/main.go 13 | SET GOARCH=ppc64le 14 | go build %BUILDFLAGS% -o ./sfx-exe/%GOOS%-%GOARCH% ./_unpack/main.go 15 | SET GOARCH=mips64 16 | go build %BUILDFLAGS% -o ./sfx-exe/%GOOS%-%GOARCH% ./_unpack/main.go 17 | 18 | 19 | SET GOOS=darwin 20 | SET GOARCH=amd64 21 | go build %BUILDFLAGS% -o ./sfx-exe/%GOOS%-%GOARCH% ./_unpack/main.go 22 | SET GOARCH=arm64 23 | go build %BUILDFLAGS% -o ./sfx-exe/%GOOS%-%GOARCH% ./_unpack/main.go 24 | 25 | SET GOOS=windows 26 | SET GOARCH=amd64 27 | go build %BUILDFLAGS% -o ./sfx-exe/%GOOS%-%GOARCH% ./_unpack/main.go 28 | SET GOARCH=386 29 | go build %BUILDFLAGS% -o ./sfx-exe/%GOOS%-%GOARCH% ./_unpack/main.go 30 | 31 | s2c.exe -rm -slower sfx-exe\* 32 | DEL /Q s2c.exe 33 | 34 | -------------------------------------------------------------------------------- /s2/cmd/_s2sx/gensfx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | go build -o=s2c ../s2c 4 | 5 | rm -rf sfx-exe/ || true 6 | 7 | GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o ./sfx-exe/linux-amd64 ./_unpack/main.go 8 | GOOS=linux GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o ./sfx-exe/linux-arm64 ./_unpack/main.go 9 | GOOS=linux GOARCH=arm go build -trimpath -ldflags="-s -w" -o ./sfx-exe/linux-arm ./_unpack/main.go 10 | GOOS=linux GOARCH=ppc64le go build -trimpath -ldflags="-s -w" -o ./sfx-exe/linux-ppc64le ./_unpack/main.go 11 | GOOS=linux GOARCH=mips64 go build -trimpath -ldflags="-s -w" -o ./sfx-exe/linux-mips64 ./_unpack/main.go 12 | 13 | GOOS=darwin GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o ./sfx-exe/darwin-amd64 ./_unpack/main.go 14 | GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="-s -w" -o ./sfx-exe/darwin-arm64 ./_unpack/main.go 15 | 16 | GOOS=windows GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o ./sfx-exe/windows-amd64 ./_unpack/main.go 17 | GOOS=windows GOARCH=386 go build -trimpath -ldflags="-s -w" -o ./sfx-exe/windows-386 ./_unpack/main.go 18 | 19 | ./s2c -rm -slower sfx-exe/* 20 | 21 | rm s2c 22 | -------------------------------------------------------------------------------- /s2/cmd/_s2sx/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/klauspost/compress/s2/cmd/s2sx 2 | 3 | go 1.22 4 | 5 | require github.com/klauspost/compress v1.11.9 6 | 7 | replace github.com/klauspost/compress v1.11.9 => ../../.. 8 | -------------------------------------------------------------------------------- /s2/cmd/_s2sx/go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/cmd/_s2sx/go.sum -------------------------------------------------------------------------------- /s2/cmd/internal/filepathx/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 The filepathx Authors 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /s2/cmd/internal/filepathx/README.md: -------------------------------------------------------------------------------- 1 | # filepathx 2 | 3 | > A small `filepath` extension library that supports double star globbling. 4 | 5 | ## Documentation 6 | 7 | GoDoc: 8 | 9 | ## Install 10 | 11 | ```bash 12 | go get github.com/yargevad/filepathx 13 | ``` 14 | 15 | ## Usage Example 16 | 17 | You can use `a/**/*.*` to match everything under the `a` directory 18 | that contains a dot, like so: 19 | 20 | ```go 21 | package main 22 | 23 | import ( 24 | "fmt" 25 | "os" 26 | 27 | "github.com/yargevad/filepathx" 28 | ) 29 | 30 | func main() { 31 | if 2 != len(os.Args) { 32 | fmt.Println(len(os.Args), os.Args) 33 | fmt.Fprintf(os.Stderr, "Usage: go build example/find/*.go; ./find \n") 34 | os.Exit(1) 35 | return 36 | } 37 | pattern := os.Args[1] 38 | 39 | matches, err := filepathx.Glob(pattern) 40 | if err != nil { 41 | panic(err) 42 | } 43 | 44 | for _, match := range matches { 45 | fmt.Printf("MATCH: [%v]\n", match) 46 | } 47 | } 48 | ``` 49 | 50 | Given this directory structure: 51 | 52 | ```bash 53 | find a 54 | ``` 55 | 56 | ```txt 57 | a 58 | a/b 59 | a/b/c.d 60 | a/b/c.d/e.f 61 | ``` 62 | 63 | This will be the output: 64 | 65 | ```bash 66 | go build example/find/*.go 67 | ./find 'a/**/*.*' 68 | ``` 69 | 70 | ```txt 71 | MATCH: [a/b/c.d] 72 | MATCH: [a/b/c.d/e.f] 73 | ``` 74 | -------------------------------------------------------------------------------- /s2/cmd/internal/filepathx/filepathx.go: -------------------------------------------------------------------------------- 1 | // Package filepathx adds double-star globbing support to the Glob function from the core path/filepath package. 2 | // You might recognize "**" recursive globs from things like your .gitignore file, and zsh. 3 | // The "**" glob represents a recursive wildcard matching zero-or-more directory levels deep. 4 | package filepathx 5 | 6 | import ( 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | ) 11 | 12 | // Globs represents one filepath glob, with its elements joined by "**". 13 | type Globs []string 14 | 15 | // Glob adds double-star support to the core path/filepath Glob function. 16 | // It's useful when your globs might have double-stars, but you're not sure. 17 | func Glob(pattern string) ([]string, error) { 18 | if !strings.Contains(pattern, "**") { 19 | // passthru to core package if no double-star 20 | return filepath.Glob(pattern) 21 | } 22 | return Globs(strings.Split(pattern, "**")).Expand() 23 | } 24 | 25 | // Expand finds matches for the provided Globs. 26 | func (globs Globs) Expand() ([]string, error) { 27 | var matches = []string{""} // accumulate here 28 | for _, glob := range globs { 29 | var hits []string 30 | var hitMap = map[string]bool{} 31 | for _, match := range matches { 32 | paths, err := filepath.Glob(match + glob) 33 | if err != nil { 34 | return nil, err 35 | } 36 | for _, path := range paths { 37 | err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error { 38 | if err != nil { 39 | return err 40 | } 41 | // save deduped match from current iteration 42 | if _, ok := hitMap[path]; !ok { 43 | hits = append(hits, path) 44 | hitMap[path] = true 45 | } 46 | return nil 47 | }) 48 | if err != nil { 49 | return nil, err 50 | } 51 | } 52 | } 53 | matches = hits 54 | } 55 | 56 | // fix up return value for nil input 57 | if globs == nil && len(matches) > 0 && matches[0] == "" { 58 | matches = matches[1:] 59 | } 60 | 61 | return matches, nil 62 | } 63 | -------------------------------------------------------------------------------- /s2/cmd/internal/filepathx/filepathx_test.go: -------------------------------------------------------------------------------- 1 | package filepathx 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | func TestGlob_ZeroDoubleStars_oneMatch(t *testing.T) { 10 | // test passthru to vanilla path/filepath 11 | path := "./a/b/c.d/e.f" 12 | err := os.MkdirAll(path, 0755) 13 | if err != nil { 14 | t.Fatalf("os.MkdirAll: %s", err) 15 | } 16 | matches, err := Glob("./*/*/*.d") 17 | if err != nil { 18 | t.Fatalf("Glob: %s", err) 19 | } 20 | if len(matches) != 1 { 21 | t.Fatalf("got %d matches, expected 1", len(matches)) 22 | } 23 | expected := strings.Join([]string{"a", "b", "c.d"}, string(os.PathSeparator)) 24 | if matches[0] != expected { 25 | t.Fatalf("matched [%s], expected [%s]", matches[0], expected) 26 | } 27 | } 28 | 29 | func TestGlob_OneDoubleStar_oneMatch(t *testing.T) { 30 | // test a single double-star 31 | path := "./a/b/c.d/e.f" 32 | err := os.MkdirAll(path, 0755) 33 | if err != nil { 34 | t.Fatalf("os.MkdirAll: %s", err) 35 | } 36 | matches, err := Glob("./**/*.f") 37 | if err != nil { 38 | t.Fatalf("Glob: %s", err) 39 | } 40 | if len(matches) != 1 { 41 | t.Fatalf("got %d matches, expected 1", len(matches)) 42 | } 43 | expected := strings.Join([]string{"a", "b", "c.d", "e.f"}, string(os.PathSeparator)) 44 | if matches[0] != expected { 45 | t.Fatalf("matched [%s], expected [%s]", matches[0], expected) 46 | } 47 | } 48 | 49 | func TestGlob_OneDoubleStar_twoMatches(t *testing.T) { 50 | // test a single double-star 51 | path := "./a/b/c.d/e.f" 52 | err := os.MkdirAll(path, 0755) 53 | if err != nil { 54 | t.Fatalf("os.MkdirAll: %s", err) 55 | } 56 | matches, err := Glob("./a/**/*.*") 57 | if err != nil { 58 | t.Fatalf("Glob: %s", err) 59 | } 60 | if len(matches) != 2 { 61 | t.Fatalf("got %d matches, expected 2", len(matches)) 62 | } 63 | expected := []string{ 64 | strings.Join([]string{"a", "b", "c.d"}, string(os.PathSeparator)), 65 | strings.Join([]string{"a", "b", "c.d", "e.f"}, string(os.PathSeparator)), 66 | } 67 | 68 | for i, match := range matches { 69 | if match != expected[i] { 70 | t.Fatalf("matched [%s], expected [%s]", match, expected[i]) 71 | } 72 | } 73 | } 74 | 75 | func TestGlob_TwoDoubleStars_oneMatch(t *testing.T) { 76 | // test two double-stars 77 | path := "./a/b/c.d/e.f" 78 | err := os.MkdirAll(path, 0755) 79 | if err != nil { 80 | t.Fatalf("os.MkdirAll: %s", err) 81 | } 82 | matches, err := Glob("./**/b/**/*.f") 83 | if err != nil { 84 | t.Fatalf("Glob: %s", err) 85 | } 86 | if len(matches) != 1 { 87 | t.Fatalf("got %d matches, expected 1", len(matches)) 88 | } 89 | expected := strings.Join([]string{"a", "b", "c.d", "e.f"}, string(os.PathSeparator)) 90 | 91 | if matches[0] != expected { 92 | t.Fatalf("matched [%s], expected [%s]", matches[0], expected) 93 | } 94 | } 95 | 96 | func TestExpand_DirectCall_emptySlice(t *testing.T) { 97 | var empty []string 98 | matches, err := Globs(empty).Expand() 99 | if err != nil { 100 | t.Fatalf("Glob: %s", err) 101 | } 102 | if len(matches) != 0 { 103 | t.Fatalf("got %d matches, expected 0", len(matches)) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /s2/cmd/internal/readahead/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Klaus Post 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /s2/cmd/internal/readahead/README.md: -------------------------------------------------------------------------------- 1 | # readahead 2 | Asynchronous read-ahead for Go readers 3 | 4 | This package will allow you to add readhead to any reader. This means a separate goroutine will perform reads from your upstream reader, so you can request from this reader without delay. 5 | 6 | This is helpful for splitting an input stream into concurrent processing, and also helps smooth out **bursts** of input or output. 7 | 8 | This should be fully transparent, except that once an error has been returned from the Reader, it will not recover. A panic will be caught and returned as an error. 9 | 10 | The readahead object also fulfills the [`io.WriterTo`](https://golang.org/pkg/io/#WriterTo) interface, which is likely to speed up `io.Copy` and other code that use the interface. 11 | 12 | See an introduction: [An Async Read-ahead Package for Go](https://blog.klauspost.com/an-async-read-ahead-package-for-go/) 13 | 14 | [![GoDoc][1]][2] [![Build Status][3]][4] 15 | 16 | [1]: https://godoc.org/github.com/klauspost/readahead?status.svg 17 | [2]: https://godoc.org/github.com/klauspost/readahead 18 | [3]: https://travis-ci.org/klauspost/readahead.svg 19 | [4]: https://travis-ci.org/klauspost/readahead 20 | 21 | # usage 22 | 23 | To get the package use `go get -u github.com/klauspost/readahead`. 24 | 25 | Here is a simple example that does file copy. Error handling has been omitted for brevity. 26 | ```Go 27 | input, _ := os.Open("input.txt") 28 | output, _ := os.Create("output.txt") 29 | defer input.Close() 30 | defer output.Close() 31 | 32 | // Create a read-ahead Reader with default settings 33 | ra := readahead.NewReader(input) 34 | defer ra.Close() 35 | 36 | // Copy the content to our output 37 | _, _ = io.Copy(output, ra) 38 | ``` 39 | 40 | # settings 41 | 42 | You can finetune the read-ahead for your specific use case, and adjust the number of buffers and the size of each buffer. 43 | 44 | The default the size of each buffer is 1MB, and there are 4 buffers. Do not make your buffers too small since there is a small overhead for passing buffers between goroutines. Other than that you are free to experiment with buffer sizes. 45 | 46 | # contributions 47 | 48 | On this project contributions in terms of new features is limited to: 49 | 50 | * Features that are widely usable and 51 | * Features that have extensive tests 52 | 53 | This package is meant to be simple and stable, so therefore these strict requirements. 54 | 55 | # license 56 | 57 | This package is released under the MIT license. See the supplied LICENSE file for more info. 58 | -------------------------------------------------------------------------------- /s2/decode_asm.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Snappy-Go Authors. All rights reserved. 2 | // Copyright (c) 2019 Klaus Post. All rights reserved. 3 | // Use of this source code is governed by a BSD-style 4 | // license that can be found in the LICENSE file. 5 | 6 | //go:build (amd64 || arm64) && !appengine && gc && !noasm 7 | // +build amd64 arm64 8 | // +build !appengine 9 | // +build gc 10 | // +build !noasm 11 | 12 | package s2 13 | 14 | // decode has the same semantics as in decode_other.go. 15 | // 16 | //go:noescape 17 | func s2Decode(dst, src []byte) int 18 | -------------------------------------------------------------------------------- /s2/encode_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019 Klaus Post. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package s2 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "math" 11 | "testing" 12 | ) 13 | 14 | func TestEncodeHuge(t *testing.T) { 15 | if true { 16 | t.Skip("Takes too much memory") 17 | } 18 | test := func(t *testing.T, data []byte) { 19 | comp := Encode(make([]byte, MaxEncodedLen(len(data))), data) 20 | decoded, err := Decode(nil, comp) 21 | if err != nil { 22 | t.Error(err) 23 | return 24 | } 25 | if !bytes.Equal(data, decoded) { 26 | t.Error("block decoder mismatch") 27 | return 28 | } 29 | if mel := MaxEncodedLen(len(data)); len(comp) > mel { 30 | t.Error(fmt.Errorf("MaxEncodedLen Exceed: input: %d, mel: %d, got %d", len(data), mel, len(comp))) 31 | return 32 | } 33 | comp = EncodeBetter(make([]byte, MaxEncodedLen(len(data))), data) 34 | decoded, err = Decode(nil, comp) 35 | if err != nil { 36 | t.Error(err) 37 | return 38 | } 39 | if !bytes.Equal(data, decoded) { 40 | t.Error("block decoder mismatch") 41 | return 42 | } 43 | if mel := MaxEncodedLen(len(data)); len(comp) > mel { 44 | t.Error(fmt.Errorf("MaxEncodedLen Exceed: input: %d, mel: %d, got %d", len(data), mel, len(comp))) 45 | return 46 | } 47 | 48 | comp = EncodeBest(make([]byte, MaxEncodedLen(len(data))), data) 49 | decoded, err = Decode(nil, comp) 50 | if err != nil { 51 | t.Error(err) 52 | return 53 | } 54 | if !bytes.Equal(data, decoded) { 55 | t.Error("block decoder mismatch") 56 | return 57 | } 58 | if mel := MaxEncodedLen(len(data)); len(comp) > mel { 59 | t.Error(fmt.Errorf("MaxEncodedLen Exceed: input: %d, mel: %d, got %d", len(data), mel, len(comp))) 60 | return 61 | } 62 | } 63 | test(t, make([]byte, math.MaxInt32)) 64 | if math.MaxInt > math.MaxInt32 { 65 | x := int64(math.MaxInt32 + math.MaxUint16) 66 | test(t, make([]byte, x)) 67 | } 68 | test(t, make([]byte, MaxBlockSize)) 69 | } 70 | -------------------------------------------------------------------------------- /s2/lz4sconvert_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2023 Klaus Post. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package s2 6 | 7 | import ( 8 | "bytes" 9 | "encoding/binary" 10 | "path/filepath" 11 | "testing" 12 | 13 | "github.com/klauspost/compress/internal/lz4ref" 14 | "github.com/klauspost/compress/internal/snapref" 15 | ) 16 | 17 | func TestLZ4sConverter_ConvertBlock(t *testing.T) { 18 | for _, tf := range testFiles { 19 | t.Run(tf.label, func(t *testing.T) { 20 | if err := downloadBenchmarkFiles(t, tf.filename); err != nil { 21 | t.Fatalf("failed to download testdata: %s", err) 22 | } 23 | 24 | bDir := filepath.FromSlash(*benchdataDir) 25 | data := readFile(t, filepath.Join(bDir, tf.filename)) 26 | if n := tf.sizeLimit; 0 < n && n < len(data) { 27 | data = data[:n] 28 | } 29 | 30 | lz4Data := make([]byte, lz4ref.CompressBlockBound(len(data))) 31 | n, err := lz4ref.CompressBlockLZ4s(data, lz4Data) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | if n == 0 { 36 | t.Skip("incompressible") 37 | return 38 | } 39 | t.Log("input size:", len(data)) 40 | t.Log("lz4 size:", n) 41 | lz4Data = lz4Data[:n] 42 | s2Dst := make([]byte, binary.MaxVarintLen32, MaxEncodedLen(len(data))) 43 | s2Dst = s2Dst[:binary.PutUvarint(s2Dst, uint64(len(data)))] 44 | hdr := len(s2Dst) 45 | 46 | conv := LZ4sConverter{} 47 | 48 | szS := 0 49 | out, n, err := conv.ConvertBlockSnappy(s2Dst, lz4Data) 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | if n != len(data) { 54 | t.Fatalf("length mismatch: want %d, got %d", len(data), n) 55 | } 56 | szS = len(out) - hdr 57 | t.Log("lz4->snappy size:", szS) 58 | 59 | decom, err := snapref.Decode(nil, out) 60 | if err != nil { 61 | t.Fatal(err) 62 | } 63 | if !bytes.Equal(decom, data) { 64 | t.Errorf("output mismatch") 65 | } 66 | 67 | sz := 0 68 | out, n, err = conv.ConvertBlock(s2Dst, lz4Data) 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | if n != len(data) { 73 | t.Fatalf("length mismatch: want %d, got %d", len(data), n) 74 | } 75 | sz = len(out) - hdr 76 | t.Log("lz4->s2 size:", sz) 77 | 78 | decom, err = Decode(nil, out) 79 | if err != nil { 80 | t.Fatal(err) 81 | } 82 | if !bytes.Equal(decom, data) { 83 | t.Errorf("output mismatch") 84 | } 85 | 86 | out2 := Encode(s2Dst[:0], data) 87 | sz2 := len(out2) - hdr 88 | t.Log("s2 (default) size:", sz2) 89 | 90 | out2 = EncodeBetter(s2Dst[:0], data) 91 | sz3 := len(out2) - hdr 92 | t.Log("s2 (better) size:", sz3) 93 | 94 | t.Log("lz4 -> s2 bytes saved:", len(lz4Data)-sz) 95 | t.Log("lz4 -> snappy bytes saved:", len(lz4Data)-szS) 96 | t.Log("data -> s2 (default) bytes saved:", len(lz4Data)-sz2) 97 | t.Log("data -> s2 (better) bytes saved:", len(lz4Data)-sz3) 98 | t.Log("direct data -> s2 (default) compared to converted from lz4:", sz-sz2) 99 | t.Log("direct data -> s2 (better) compared to converted from lz4:", sz-sz3) 100 | }) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /s2/reader_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2019+ Klaus Post. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package s2 6 | 7 | import ( 8 | "bytes" 9 | "io" 10 | "testing" 11 | ) 12 | 13 | func TestLeadingSkippableBlock(t *testing.T) { 14 | var buf bytes.Buffer 15 | w := NewWriter(&buf) 16 | if err := w.AddSkippableBlock(0x80, []byte("skippable block")); err != nil { 17 | t.Fatalf("w.AddSkippableBlock: %v", err) 18 | } 19 | if _, err := w.Write([]byte("some data")); err != nil { 20 | t.Fatalf("w.Write: %v", err) 21 | } 22 | if err := w.Close(); err != nil { 23 | t.Fatalf("w.Close: %v", err) 24 | } 25 | r := NewReader(&buf) 26 | var sb []byte 27 | r.SkippableCB(0x80, func(sr io.Reader) error { 28 | var err error 29 | sb, err = io.ReadAll(sr) 30 | return err 31 | }) 32 | if _, err := r.Read([]byte{}); err != nil { 33 | t.Errorf("empty read failed: %v", err) 34 | } 35 | if !bytes.Equal(sb, []byte("skippable block")) { 36 | t.Errorf("didn't get correct data from skippable block: %q", string(sb)) 37 | } 38 | data, err := io.ReadAll(r) 39 | if err != nil { 40 | t.Fatalf("r.Read: %v", err) 41 | } 42 | if !bytes.Equal(data, []byte("some data")) { 43 | t.Errorf("didn't get correct compressed data: %q", string(data)) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /s2/testdata/4f9e1a0da7915a3d69632f5613ed78bc998a8a23.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/testdata/4f9e1a0da7915a3d69632f5613ed78bc998a8a23.zst -------------------------------------------------------------------------------- /s2/testdata/Mark.Twain-Tom.Sawyer.txt.rawsnappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/testdata/Mark.Twain-Tom.Sawyer.txt.rawsnappy -------------------------------------------------------------------------------- /s2/testdata/dec-block-regressions.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/testdata/dec-block-regressions.zip -------------------------------------------------------------------------------- /s2/testdata/enc_regressions.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/testdata/enc_regressions.zip -------------------------------------------------------------------------------- /s2/testdata/fuzz/FuzzLZ4Block.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/testdata/fuzz/FuzzLZ4Block.zip -------------------------------------------------------------------------------- /s2/testdata/fuzz/block-corpus-enc.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/testdata/fuzz/block-corpus-enc.zip -------------------------------------------------------------------------------- /s2/testdata/fuzz/block-corpus-raw.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/testdata/fuzz/block-corpus-raw.zip -------------------------------------------------------------------------------- /s2/testdata/fuzz/dict-corpus-oss.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/testdata/fuzz/dict-corpus-oss.zip -------------------------------------------------------------------------------- /s2/testdata/fuzz/lz4-convert-corpus-raw.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/testdata/fuzz/lz4-convert-corpus-raw.zip -------------------------------------------------------------------------------- /s2/testdata/random: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/testdata/random -------------------------------------------------------------------------------- /s2/testdata/s2-dict.bin.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2/testdata/s2-dict.bin.gz -------------------------------------------------------------------------------- /s2sx.mod: -------------------------------------------------------------------------------- 1 | module github.com/klauspost/compress 2 | 3 | go 1.22 4 | -------------------------------------------------------------------------------- /s2sx.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/s2sx.sum -------------------------------------------------------------------------------- /snappy/.gitignore: -------------------------------------------------------------------------------- 1 | cmd/snappytool/snappytool 2 | testdata/bench 3 | 4 | # These explicitly listed benchmark data files are for an obsolete version of 5 | # snappy_test.go. 6 | testdata/alice29.txt 7 | testdata/asyoulik.txt 8 | testdata/fireworks.jpeg 9 | testdata/geo.protodata 10 | testdata/html 11 | testdata/html_x_4 12 | testdata/kppkn.gtb 13 | testdata/lcet10.txt 14 | testdata/paper-100k.pdf 15 | testdata/plrabn12.txt 16 | testdata/urls.10K 17 | -------------------------------------------------------------------------------- /snappy/AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of Snappy-Go authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files. 3 | # See the latter for an explanation. 4 | 5 | # Names should be added to this file as 6 | # Name or Organization 7 | # The email address is not required for organizations. 8 | 9 | # Please keep the list sorted. 10 | 11 | Amazon.com, Inc 12 | Damian Gryski 13 | Eric Buth 14 | Google Inc. 15 | Jan Mercl <0xjnml@gmail.com> 16 | Klaus Post 17 | Rodolfo Carvalho 18 | Sebastien Binet 19 | -------------------------------------------------------------------------------- /snappy/CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # This is the official list of people who can contribute 2 | # (and typically have contributed) code to the Snappy-Go repository. 3 | # The AUTHORS file lists the copyright holders; this file 4 | # lists people. For example, Google employees are listed here 5 | # but not in AUTHORS, because Google holds the copyright. 6 | # 7 | # The submission process automatically checks to make sure 8 | # that people submitting code are listed in this file (by email address). 9 | # 10 | # Names should be added to this file only after verifying that 11 | # the individual or the individual's organization has agreed to 12 | # the appropriate Contributor License Agreement, found here: 13 | # 14 | # http://code.google.com/legal/individual-cla-v1.0.html 15 | # http://code.google.com/legal/corporate-cla-v1.0.html 16 | # 17 | # The agreement for individuals can be filled out on the web. 18 | # 19 | # When adding J Random Contributor's name to this file, 20 | # either J's name or J's organization's name should be 21 | # added to the AUTHORS file, depending on whether the 22 | # individual or corporate CLA was used. 23 | 24 | # Names should be added to this file like so: 25 | # Name 26 | 27 | # Please keep the list sorted. 28 | 29 | Alex Legg 30 | Damian Gryski 31 | Eric Buth 32 | Jan Mercl <0xjnml@gmail.com> 33 | Jonathan Swinney 34 | Kai Backman 35 | Klaus Post 36 | Marc-Antoine Ruel 37 | Nigel Tao 38 | Rob Pike 39 | Rodolfo Carvalho 40 | Russ Cox 41 | Sebastien Binet 42 | -------------------------------------------------------------------------------- /snappy/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /snappy/README.md: -------------------------------------------------------------------------------- 1 | # snappy 2 | 3 | The Snappy compression format in the Go programming language. 4 | 5 | This is a drop-in replacement for `github.com/golang/snappy`. 6 | 7 | It provides a full, compatible replacement of the Snappy package by simply changing imports. 8 | 9 | See [Snappy Compatibility](https://github.com/klauspost/compress/tree/master/s2#snappy-compatibility) in the S2 documentation. 10 | 11 | "Better" compression mode is used. For buffered streams concurrent compression is used. 12 | 13 | For more options use the [s2 package](https://pkg.go.dev/github.com/klauspost/compress/s2). 14 | 15 | # usage 16 | 17 | Replace imports `github.com/golang/snappy` with `github.com/klauspost/compress/snappy`. 18 | -------------------------------------------------------------------------------- /snappy/decode.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Snappy-Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package snappy 6 | 7 | import ( 8 | "io" 9 | 10 | "github.com/klauspost/compress/s2" 11 | ) 12 | 13 | var ( 14 | // ErrCorrupt reports that the input is invalid. 15 | ErrCorrupt = s2.ErrCorrupt 16 | // ErrTooLarge reports that the uncompressed length is too large. 17 | ErrTooLarge = s2.ErrTooLarge 18 | // ErrUnsupported reports that the input isn't supported. 19 | ErrUnsupported = s2.ErrUnsupported 20 | ) 21 | 22 | const ( 23 | // maxBlockSize is the maximum size of the input to encodeBlock. It is not 24 | // part of the wire format per se, but some parts of the encoder assume 25 | // that an offset fits into a uint16. 26 | // 27 | // Also, for the framing format (Writer type instead of Encode function), 28 | // https://github.com/google/snappy/blob/master/framing_format.txt says 29 | // that "the uncompressed data in a chunk must be no longer than 65536 30 | // bytes". 31 | maxBlockSize = 65536 32 | ) 33 | 34 | // DecodedLen returns the length of the decoded block. 35 | func DecodedLen(src []byte) (int, error) { 36 | return s2.DecodedLen(src) 37 | } 38 | 39 | // Decode returns the decoded form of src. The returned slice may be a sub- 40 | // slice of dst if dst was large enough to hold the entire decoded block. 41 | // Otherwise, a newly allocated slice will be returned. 42 | // 43 | // The dst and src must not overlap. It is valid to pass a nil dst. 44 | // 45 | // Decode handles the Snappy block format, not the Snappy stream format. 46 | func Decode(dst, src []byte) ([]byte, error) { 47 | return s2.Decode(dst, src) 48 | } 49 | 50 | // NewReader returns a new Reader that decompresses from r, using the framing 51 | // format described at 52 | // https://github.com/google/snappy/blob/master/framing_format.txt 53 | func NewReader(r io.Reader) *Reader { 54 | return s2.NewReader(r, s2.ReaderMaxBlockSize(maxBlockSize)) 55 | } 56 | 57 | // Reader is an io.Reader that can read Snappy-compressed bytes. 58 | // 59 | // Reader handles the Snappy stream format, not the Snappy block format. 60 | type Reader = s2.Reader 61 | -------------------------------------------------------------------------------- /snappy/encode.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Snappy-Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package snappy 6 | 7 | import ( 8 | "io" 9 | 10 | "github.com/klauspost/compress/s2" 11 | ) 12 | 13 | // Encode returns the encoded form of src. The returned slice may be a sub- 14 | // slice of dst if dst was large enough to hold the entire encoded block. 15 | // Otherwise, a newly allocated slice will be returned. 16 | // 17 | // The dst and src must not overlap. It is valid to pass a nil dst. 18 | // 19 | // Encode handles the Snappy block format, not the Snappy stream format. 20 | func Encode(dst, src []byte) []byte { 21 | return s2.EncodeSnappyBetter(dst, src) 22 | } 23 | 24 | // MaxEncodedLen returns the maximum length of a snappy block, given its 25 | // uncompressed length. 26 | // 27 | // It will return a negative value if srcLen is too large to encode. 28 | func MaxEncodedLen(srcLen int) int { 29 | return s2.MaxEncodedLen(srcLen) 30 | } 31 | 32 | // NewWriter returns a new Writer that compresses to w. 33 | // 34 | // The Writer returned does not buffer writes. There is no need to Flush or 35 | // Close such a Writer. 36 | // 37 | // Deprecated: the Writer returned is not suitable for many small writes, only 38 | // for few large writes. Use NewBufferedWriter instead, which is efficient 39 | // regardless of the frequency and shape of the writes, and remember to Close 40 | // that Writer when done. 41 | func NewWriter(w io.Writer) *Writer { 42 | return s2.NewWriter(w, s2.WriterSnappyCompat(), s2.WriterBetterCompression(), s2.WriterFlushOnWrite(), s2.WriterConcurrency(1)) 43 | } 44 | 45 | // NewBufferedWriter returns a new Writer that compresses to w, using the 46 | // framing format described at 47 | // https://github.com/google/snappy/blob/master/framing_format.txt 48 | // 49 | // The Writer returned buffers writes. Users must call Close to guarantee all 50 | // data has been forwarded to the underlying io.Writer. They may also call 51 | // Flush zero or more times before calling Close. 52 | func NewBufferedWriter(w io.Writer) *Writer { 53 | return s2.NewWriter(w, s2.WriterSnappyCompat(), s2.WriterBetterCompression()) 54 | } 55 | 56 | // Writer is an io.Writer that can write Snappy-compressed bytes. 57 | // 58 | // Writer handles the Snappy stream format, not the Snappy block format. 59 | type Writer = s2.Writer 60 | -------------------------------------------------------------------------------- /snappy/snappy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Snappy-Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | // Package snappy implements the Snappy compression format. It aims for very 6 | // high speeds and reasonable compression. 7 | // 8 | // There are actually two Snappy formats: block and stream. They are related, 9 | // but different: trying to decompress block-compressed data as a Snappy stream 10 | // will fail, and vice versa. The block format is the Decode and Encode 11 | // functions and the stream format is the Reader and Writer types. 12 | // 13 | // The block format, the more common case, is used when the complete size (the 14 | // number of bytes) of the original data is known upfront, at the time 15 | // compression starts. The stream format, also known as the framing format, is 16 | // for when that isn't always true. 17 | // 18 | // The canonical, C++ implementation is at https://github.com/google/snappy and 19 | // it only implements the block format. 20 | package snappy 21 | 22 | /* 23 | Each encoded block begins with the varint-encoded length of the decoded data, 24 | followed by a sequence of chunks. Chunks begin and end on byte boundaries. The 25 | first byte of each chunk is broken into its 2 least and 6 most significant bits 26 | called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag. 27 | Zero means a literal tag. All other values mean a copy tag. 28 | 29 | For literal tags: 30 | - If m < 60, the next 1 + m bytes are literal bytes. 31 | - Otherwise, let n be the little-endian unsigned integer denoted by the next 32 | m - 59 bytes. The next 1 + n bytes after that are literal bytes. 33 | 34 | For copy tags, length bytes are copied from offset bytes ago, in the style of 35 | Lempel-Ziv compression algorithms. In particular: 36 | - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12). 37 | The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10 38 | of the offset. The next byte is bits 0-7 of the offset. 39 | - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65). 40 | The length is 1 + m. The offset is the little-endian unsigned integer 41 | denoted by the next 2 bytes. 42 | - For l == 3, this tag is a legacy format that is no longer issued by most 43 | encoders. Nonetheless, the offset ranges in [0, 1<<32) and the length in 44 | [1, 65). The length is 1 + m. The offset is the little-endian unsigned 45 | integer denoted by the next 4 bytes. 46 | */ 47 | -------------------------------------------------------------------------------- /snappy/testdata/Isaac.Newton-Opticks.txt.rawsnappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/snappy/testdata/Isaac.Newton-Opticks.txt.rawsnappy -------------------------------------------------------------------------------- /snappy/xerial/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Evan Huus 4 | Copyright (c) 2023 Klaus Post 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /snappy/xerial/README.md: -------------------------------------------------------------------------------- 1 | # go-xerial-snappy 2 | 3 | Xerial-compatible Snappy framing support for golang. 4 | 5 | Packages using Xerial for snappy encoding use a framing format incompatible with 6 | basically everything else in existence. 7 | 8 | Apps that use this format include Apache Kafka (see 9 | https://github.com/dpkp/kafka-python/issues/126#issuecomment-35478921 for 10 | details). 11 | 12 | # Fork 13 | 14 | Forked from [github.com/eapache/go-xerial-snappy](https://github.com/eapache/go-xerial-snappy). 15 | 16 | Changes: 17 | 18 | * Uses [S2](https://github.com/klauspost/compress/tree/master/s2#snappy-compatibility) for better/faster compression and decompression. 19 | * Fixes 0-length roundtrips. 20 | * Adds `DecodeCapped`, which allows decompression with capped output size. 21 | * `DecodeInto` will decode directly into destination if there is space enough. 22 | * `Encode` will now encode directly into 'dst' if it has space enough. 23 | * Fixes short snappy buffers returning `ErrMalformed`. 24 | * Renames `EncodeStream` to `Encode`. 25 | * Adds `EncodeBetter` for better than default compression at ~half the speed. 26 | 27 | 28 | Comparison (before/after): 29 | 30 | ``` 31 | BenchmarkSnappyStreamEncode-32 959010 1170 ns/op 875.15 MB/s 1280 B/op 1 allocs/op 32 | BenchmarkSnappyStreamEncode-32 1000000 1107 ns/op 925.04 MB/s 0 B/op 0 allocs/op 33 | --> Output size: 913 -> 856 bytes 34 | 35 | BenchmarkSnappyStreamEncodeBetter-32 477739 2506 ns/op 408.62 MB/s 0 B/op 0 allocs/op 36 | --> Output size: 835 bytes 37 | 38 | BenchmarkSnappyStreamEncodeMassive-32 100 10596963 ns/op 966.31 MB/s 40977 B/op 1 allocs/op 39 | BenchmarkSnappyStreamEncodeMassive-32 100 10220236 ns/op 1001.93 MB/s 0 B/op 0 allocs/op 40 | --> Output size: 2365547 -> 2256991 bytes 41 | 42 | BenchmarkSnappyStreamEncodeBetterMassive-32 69 16983314 ns/op 602.94 MB/s 0 B/op 0 allocs/op 43 | --> Output size: 2011997 bytes 44 | 45 | BenchmarkSnappyStreamDecodeInto-32 1887378 639.5 ns/op 1673.19 MB/s 1088 B/op 3 allocs/op 46 | BenchmarkSnappyStreamDecodeInto-32 2707915 436.2 ns/op 2452.99 MB/s 0 B/op 0 allocs/op 47 | 48 | BenchmarkSnappyStreamDecodeIntoMassive-32 267 4559594 ns/op 2245.81 MB/s 71120 B/op 1 allocs/op 49 | BenchmarkSnappyStreamDecodeIntoMassive-32 282 4285844 ns/op 2389.26 MB/s 0 B/op 0 allocs/op 50 | ``` -------------------------------------------------------------------------------- /snappy/xerial/fuzz_test.go: -------------------------------------------------------------------------------- 1 | package xerial 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/klauspost/compress/internal/fuzz" 8 | "github.com/klauspost/compress/s2" 9 | ) 10 | 11 | func FuzzDecode(f *testing.F) { 12 | fuzz.AddFromZip(f, "testdata/FuzzDecoder.zip", fuzz.TypeGoFuzz, false) 13 | const limit = 1 << 20 14 | dst := make([]byte, 0, limit) 15 | f.Fuzz(func(t *testing.T, data []byte) { 16 | got, _ := DecodeCapped(dst[:0], data) 17 | if len(got) > cap(dst) { 18 | t.Fatalf("cap exceeded: %d > %d", len(got), cap(dst)) 19 | } 20 | }) 21 | } 22 | 23 | func FuzzEncode(f *testing.F) { 24 | fuzz.AddFromZip(f, "testdata/block-corpus-raw.zip", fuzz.TypeRaw, false) 25 | 26 | f.Fuzz(func(t *testing.T, data []byte) { 27 | encoded := Encode(make([]byte, 0, len(data)/2), data) 28 | decoded, err := Decode(encoded) 29 | if err != nil { 30 | t.Errorf("input: %+v, encoded: %+v", data, encoded) 31 | t.Fatal(err) 32 | } 33 | if !bytes.Equal(decoded, data) { 34 | t.Fatal("mismatch") 35 | } 36 | 37 | encoded = EncodeBetter(make([]byte, 0, len(data)/2), data) 38 | decoded, err = Decode(encoded) 39 | if err != nil { 40 | t.Errorf("input: %+v, encoded: %+v", data, encoded) 41 | t.Fatal(err) 42 | } 43 | if !bytes.Equal(decoded, data) { 44 | t.Fatal("mismatch") 45 | } 46 | 47 | encoded = s2.EncodeSnappy(make([]byte, 0, len(data)/2), data) 48 | decoded, err = Decode(encoded) 49 | if err != nil { 50 | t.Errorf("input: %+v, encoded: %+v", data, encoded) 51 | t.Fatal(err) 52 | } 53 | if !bytes.Equal(decoded, data) { 54 | t.Fatal("mismatch") 55 | } 56 | }) 57 | } 58 | -------------------------------------------------------------------------------- /snappy/xerial/testdata/FuzzDecode_raw.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/snappy/xerial/testdata/FuzzDecode_raw.zip -------------------------------------------------------------------------------- /snappy/xerial/testdata/FuzzDecoder.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/snappy/xerial/testdata/FuzzDecoder.zip -------------------------------------------------------------------------------- /snappy/xerial/testdata/block-corpus-raw.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/snappy/xerial/testdata/block-corpus-raw.zip -------------------------------------------------------------------------------- /testdata/case1.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/case1.bin -------------------------------------------------------------------------------- /testdata/case2.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/case2.bin -------------------------------------------------------------------------------- /testdata/case3.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/case3.bin -------------------------------------------------------------------------------- /testdata/crash1.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/crash1.bin -------------------------------------------------------------------------------- /testdata/crash2.bin: -------------------------------------------------------------------------------- 1 | 313254615470505 -------------------------------------------------------------------------------- /testdata/crash3.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/crash3.bin -------------------------------------------------------------------------------- /testdata/crash4.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/crash4.bin -------------------------------------------------------------------------------- /testdata/crash5.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/crash5.bin -------------------------------------------------------------------------------- /testdata/dec-crash6.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/dec-crash6.bin -------------------------------------------------------------------------------- /testdata/dec-hang1.bin: -------------------------------------------------------------------------------- 1 | $� -------------------------------------------------------------------------------- /testdata/dec-hang2.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/dec-hang2.bin -------------------------------------------------------------------------------- /testdata/dec-hang3.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/dec-hang3.bin -------------------------------------------------------------------------------- /testdata/dec-symlen1.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/dec-symlen1.bin -------------------------------------------------------------------------------- /testdata/endnonzero.bin: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testdata/endzerobits.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /testdata/fse-artifact3.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/fse-artifact3.bin -------------------------------------------------------------------------------- /testdata/gettysburg.txt: -------------------------------------------------------------------------------- 1 | Four score and seven years ago our fathers brought forth on 2 | this continent, a new nation, conceived in Liberty, and dedicated 3 | to the proposition that all men are created equal. 4 | Now we are engaged in a great Civil War, testing whether that 5 | nation, or any nation so conceived and so dedicated, can long 6 | endure. 7 | We are met on a great battle-field of that war. 8 | We have come to dedicate a portion of that field, as a final 9 | resting place for those who here gave their lives that that 10 | nation might live. It is altogether fitting and proper that 11 | we should do this. 12 | But, in a larger sense, we can not dedicate - we can not 13 | consecrate - we can not hallow - this ground. 14 | The brave men, living and dead, who struggled here, have 15 | consecrated it, far above our poor power to add or detract. 16 | The world will little note, nor long remember what we say here, 17 | but it can never forget what they did here. 18 | It is for us the living, rather, to be dedicated here to the 19 | unfinished work which they who fought here have thus far so 20 | nobly advanced. It is rather for us to be here dedicated to 21 | the great task remaining before us - that from these honored 22 | dead we take increased devotion to that cause for which they 23 | gave the last full measure of devotion - 24 | that we here highly resolve that these dead shall not have 25 | died in vain - that this nation, under God, shall have a new 26 | birth of freedom - and that government of the people, by the 27 | people, for the people, shall not perish from this earth. 28 | 29 | Abraham Lincoln, November 19, 1863, Gettysburg, Pennsylvania 30 | -------------------------------------------------------------------------------- /testdata/html.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/html.txt -------------------------------------------------------------------------------- /testdata/normcount2.bin: -------------------------------------------------------------------------------- 1 | 868000113000000fd9F055125272181835410155551151-0_0040Y2BW4K_0x_j4L___e__331ms__QSlz_I__ -------------------------------------------------------------------------------- /testdata/pngdata.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/pngdata.bin -------------------------------------------------------------------------------- /testdata/sharnd.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/testdata/sharnd.out -------------------------------------------------------------------------------- /zip/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package zip_test 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "io" 11 | "log" 12 | "os" 13 | 14 | "github.com/klauspost/compress/flate" 15 | "github.com/klauspost/compress/zip" 16 | ) 17 | 18 | func ExampleWriter() { 19 | // Create a buffer to write our archive to. 20 | buf := new(bytes.Buffer) 21 | 22 | // Create a new zip archive. 23 | w := zip.NewWriter(buf) 24 | 25 | // Add some files to the archive. 26 | var files = []struct { 27 | Name, Body string 28 | }{ 29 | {"readme.txt", "This archive contains some text files."}, 30 | {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"}, 31 | {"todo.txt", "Get animal handling licence.\nWrite more examples."}, 32 | } 33 | for _, file := range files { 34 | f, err := w.Create(file.Name) 35 | if err != nil { 36 | log.Fatal(err) 37 | } 38 | _, err = f.Write([]byte(file.Body)) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | } 43 | 44 | // Make sure to check the error on Close. 45 | err := w.Close() 46 | if err != nil { 47 | log.Fatal(err) 48 | } 49 | } 50 | 51 | func ExampleReader() { 52 | // Open a zip archive for reading. 53 | r, err := zip.OpenReader("testdata/readme.zip") 54 | if err != nil { 55 | log.Fatal(err) 56 | } 57 | defer r.Close() 58 | 59 | // Iterate through the files in the archive, 60 | // printing some of their contents. 61 | for _, f := range r.File { 62 | fmt.Printf("Contents of %s:\n", f.Name) 63 | rc, err := f.Open() 64 | if err != nil { 65 | log.Fatal(err) 66 | } 67 | _, err = io.CopyN(os.Stdout, rc, 68) 68 | if err != nil { 69 | log.Fatal(err) 70 | } 71 | rc.Close() 72 | fmt.Println() 73 | } 74 | // Output: 75 | // Contents of README: 76 | // This is the source code repository for the Go programming language. 77 | } 78 | 79 | func ExampleWriter_RegisterCompressor() { 80 | // Override the default Deflate compressor with a higher compression level. 81 | 82 | // Create a buffer to write our archive to. 83 | buf := new(bytes.Buffer) 84 | 85 | // Create a new zip archive. 86 | w := zip.NewWriter(buf) 87 | 88 | // Register a custom Deflate compressor. 89 | w.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) { 90 | return flate.NewWriter(out, flate.BestCompression) 91 | }) 92 | 93 | // Proceed to add files to w. 94 | } 95 | -------------------------------------------------------------------------------- /zip/fuzz_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package zip 6 | 7 | import ( 8 | "bytes" 9 | "io" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | "testing" 14 | 15 | "github.com/klauspost/compress/internal/fuzz" 16 | ) 17 | 18 | func FuzzReader(f *testing.F) { 19 | testdata, err := os.ReadDir("testdata") 20 | if err != nil { 21 | f.Fatalf("failed to read testdata directory: %s", err) 22 | } 23 | for _, de := range testdata { 24 | if de.IsDir() { 25 | continue 26 | } 27 | b, err := os.ReadFile(filepath.Join("testdata", de.Name())) 28 | if err != nil { 29 | f.Fatalf("failed to read testdata: %s", err) 30 | } 31 | f.Add(b) 32 | } 33 | fuzz.AddFromZip(f, "testdata/FuzzReader-raw.zip", fuzz.TypeRaw, testing.Short()) 34 | fuzz.AddFromZip(f, "testdata/FuzzReader-enc.zip", fuzz.TypeGoFuzz, testing.Short()) 35 | 36 | f.Fuzz(func(t *testing.T, b []byte) { 37 | r, err := NewReader(bytes.NewReader(b), int64(len(b))) 38 | if err != nil { 39 | return 40 | } 41 | 42 | type file struct { 43 | header *FileHeader 44 | content []byte 45 | } 46 | files := make([]file, 0, len(r.File)) 47 | 48 | for i, f := range r.File { 49 | fr, err := f.Open() 50 | if err != nil { 51 | continue 52 | } 53 | // No more than 1MiB per file 54 | limit := int64(1 << 20) 55 | if i < 100 { 56 | limit = 10 57 | } 58 | content, err := io.ReadAll(io.LimitReader(fr, limit)) 59 | if err != nil { 60 | continue 61 | } 62 | 63 | files = append(files, file{header: &f.FileHeader, content: content}) 64 | if _, err := r.Open(f.Name); err != nil { 65 | continue 66 | } 67 | } 68 | 69 | // If we were unable to read anything out of the archive don't 70 | // bother trying to roundtrip it. 71 | if len(files) == 0 { 72 | return 73 | } 74 | 75 | w := NewWriter(io.Discard) 76 | for _, f := range files { 77 | ww, err := w.CreateHeader(f.header) 78 | if err != nil { 79 | t.Fatalf("unable to write previously parsed header: %s", err) 80 | } 81 | if !strings.HasSuffix(f.header.Name, "/") { 82 | if _, err := ww.Write(f.content); err != nil { 83 | t.Fatalf("unable to write previously parsed content: %s", err) 84 | } 85 | } 86 | } 87 | 88 | if err := w.Close(); err != nil { 89 | t.Fatalf("Unable to write archive: %s", err) 90 | } 91 | 92 | // TODO: We may want to check if the archive roundtrips. 93 | }) 94 | } 95 | -------------------------------------------------------------------------------- /zip/internal/obscuretestdata/obscuretestdata.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | //go:build go1.16 6 | // +build go1.16 7 | 8 | // Package obscuretestdata contains functionality used by tests to more easily 9 | // work with testdata that must be obscured primarily due to 10 | // golang.org/issue/34986. 11 | package obscuretestdata 12 | 13 | import ( 14 | "encoding/base64" 15 | "io" 16 | "os" 17 | ) 18 | 19 | // DecodeToTempFile decodes the named file to a temporary location. 20 | // If successful, it returns the path of the decoded file. 21 | // The caller is responsible for ensuring that the temporary file is removed. 22 | func DecodeToTempFile(name string) (path string, err error) { 23 | f, err := os.Open(name) 24 | if err != nil { 25 | return "", err 26 | } 27 | defer f.Close() 28 | 29 | tmp, err := os.CreateTemp("", "obscuretestdata-decoded-") 30 | if err != nil { 31 | return "", err 32 | } 33 | if _, err := io.Copy(tmp, base64.NewDecoder(base64.StdEncoding, f)); err != nil { 34 | tmp.Close() 35 | os.Remove(tmp.Name()) 36 | return "", err 37 | } 38 | if err := tmp.Close(); err != nil { 39 | os.Remove(tmp.Name()) 40 | return "", err 41 | } 42 | return tmp.Name(), nil 43 | } 44 | 45 | // ReadFile reads the named file and returns its decoded contents. 46 | func ReadFile(name string) ([]byte, error) { 47 | f, err := os.Open(name) 48 | if err != nil { 49 | return nil, err 50 | } 51 | defer f.Close() 52 | return io.ReadAll(base64.NewDecoder(base64.StdEncoding, f)) 53 | } 54 | -------------------------------------------------------------------------------- /zip/testdata/FuzzReader-enc.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/FuzzReader-enc.zip -------------------------------------------------------------------------------- /zip/testdata/FuzzReader-raw.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/FuzzReader-raw.zip -------------------------------------------------------------------------------- /zip/testdata/comment-truncated.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/comment-truncated.zip -------------------------------------------------------------------------------- /zip/testdata/crc32-not-streamed.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/crc32-not-streamed.zip -------------------------------------------------------------------------------- /zip/testdata/dd.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/dd.zip -------------------------------------------------------------------------------- /zip/testdata/dupdir.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/dupdir.zip -------------------------------------------------------------------------------- /zip/testdata/go-no-datadesc-sig.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/go-no-datadesc-sig.zip -------------------------------------------------------------------------------- /zip/testdata/go-no-datadesc-sig.zip.base64: -------------------------------------------------------------------------------- 1 | UEsDBBQACAAAAGWHaECoZTJ+BAAAAAQAAAAHABgAZm9vLnR4dFVUBQAD3lVZT3V4CwABBPUBAAAEFAAAAGZvbwqoZTJ+BAAAAAQAAABQSwMEFAAIAAAAZodoQOmzogQEAAAABAAAAAcAGABiYXIudHh0VVQFAAPgVVlPdXgLAAEE9QEAAAQUAAAAYmFyCumzogQEAAAABAAAAFBLAQIUAxQACAAAAGWHaECoZTJ+BAAAAAQAAAAHABgAAAAAAAAAAACkgQAAAABmb28udHh0VVQFAAPeVVlPdXgLAAEE9QEAAAQUAAAAUEsBAhQDFAAIAAAAZodoQOmzogQEAAAABAAAAAcAGAAAAAAAAAAAAKSBTQAAAGJhci50eHRVVAUAA+BVWU91eAsAAQT1AQAABBQAAABQSwUGAAAAAAIAAgCaAAAAmgAAAAAA 2 | -------------------------------------------------------------------------------- /zip/testdata/go-with-datadesc-sig.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/go-with-datadesc-sig.zip -------------------------------------------------------------------------------- /zip/testdata/gophercolor16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/gophercolor16x16.png -------------------------------------------------------------------------------- /zip/testdata/readme.notzip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/readme.notzip -------------------------------------------------------------------------------- /zip/testdata/readme.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/readme.zip -------------------------------------------------------------------------------- /zip/testdata/subdir.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/subdir.zip -------------------------------------------------------------------------------- /zip/testdata/symlink.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/symlink.zip -------------------------------------------------------------------------------- /zip/testdata/test-badbase.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/test-badbase.zip -------------------------------------------------------------------------------- /zip/testdata/test-baddirsz.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/test-baddirsz.zip -------------------------------------------------------------------------------- /zip/testdata/test-prefix.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/test-prefix.zip -------------------------------------------------------------------------------- /zip/testdata/test-trailing-junk.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/test-trailing-junk.zip -------------------------------------------------------------------------------- /zip/testdata/test.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/test.zip -------------------------------------------------------------------------------- /zip/testdata/time-22738.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/time-22738.zip -------------------------------------------------------------------------------- /zip/testdata/time-7zip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/time-7zip.zip -------------------------------------------------------------------------------- /zip/testdata/time-go.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/time-go.zip -------------------------------------------------------------------------------- /zip/testdata/time-infozip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/time-infozip.zip -------------------------------------------------------------------------------- /zip/testdata/time-osx.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/time-osx.zip -------------------------------------------------------------------------------- /zip/testdata/time-win7.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/time-win7.zip -------------------------------------------------------------------------------- /zip/testdata/time-winrar.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/time-winrar.zip -------------------------------------------------------------------------------- /zip/testdata/time-winzip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/time-winzip.zip -------------------------------------------------------------------------------- /zip/testdata/unix.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/unix.zip -------------------------------------------------------------------------------- /zip/testdata/utf8-7zip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/utf8-7zip.zip -------------------------------------------------------------------------------- /zip/testdata/utf8-infozip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/utf8-infozip.zip -------------------------------------------------------------------------------- /zip/testdata/utf8-osx.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/utf8-osx.zip -------------------------------------------------------------------------------- /zip/testdata/utf8-winrar.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/utf8-winrar.zip -------------------------------------------------------------------------------- /zip/testdata/utf8-winzip.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/utf8-winzip.zip -------------------------------------------------------------------------------- /zip/testdata/winxp.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/winxp.zip -------------------------------------------------------------------------------- /zip/testdata/zip64-2.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/zip64-2.zip -------------------------------------------------------------------------------- /zip/testdata/zip64.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zip/testdata/zip64.zip -------------------------------------------------------------------------------- /zlib/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package zlib_test 6 | 7 | import ( 8 | "bytes" 9 | "compress/zlib" 10 | "fmt" 11 | "io" 12 | "os" 13 | ) 14 | 15 | func ExampleNewWriter() { 16 | var b bytes.Buffer 17 | 18 | w := zlib.NewWriter(&b) 19 | w.Write([]byte("hello, world\n")) 20 | w.Close() 21 | fmt.Println(b.Bytes()) 22 | // Output: [120 156 202 72 205 201 201 215 81 40 207 47 202 73 225 2 4 0 0 255 255 33 231 4 147] 23 | } 24 | 25 | func ExampleNewReader() { 26 | buff := []byte{120, 156, 202, 72, 205, 201, 201, 215, 81, 40, 207, 27 | 47, 202, 73, 225, 2, 4, 0, 0, 255, 255, 33, 231, 4, 147} 28 | b := bytes.NewReader(buff) 29 | 30 | r, err := zlib.NewReader(b) 31 | if err != nil { 32 | panic(err) 33 | } 34 | io.Copy(os.Stdout, r) 35 | // Output: hello, world 36 | r.Close() 37 | } 38 | -------------------------------------------------------------------------------- /zstd/_generate/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/klauspost/compress/s2/_generate 2 | 3 | go 1.22.0 4 | 5 | toolchain go1.22.4 6 | 7 | require ( 8 | github.com/klauspost/compress v1.15.15 9 | github.com/mmcloughlin/avo v0.6.0 10 | ) 11 | 12 | require ( 13 | golang.org/x/mod v0.21.0 // indirect 14 | golang.org/x/sync v0.8.0 // indirect 15 | golang.org/x/tools v0.25.0 // indirect 16 | ) 17 | 18 | replace github.com/klauspost/compress => ../.. 19 | -------------------------------------------------------------------------------- /zstd/_generate/go.sum: -------------------------------------------------------------------------------- 1 | github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY= 2 | github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8= 3 | golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= 4 | golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= 5 | golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= 6 | golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 7 | golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= 8 | golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= 9 | -------------------------------------------------------------------------------- /zstd/bitreader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019+ Klaus Post. All rights reserved. 2 | // License information can be found in the LICENSE file. 3 | // Based on work by Yann Collet, released under BSD License. 4 | 5 | package zstd 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "io" 11 | "math/bits" 12 | 13 | "github.com/klauspost/compress/internal/le" 14 | ) 15 | 16 | // bitReader reads a bitstream in reverse. 17 | // The last set bit indicates the start of the stream and is used 18 | // for aligning the input. 19 | type bitReader struct { 20 | in []byte 21 | value uint64 // Maybe use [16]byte, but shifting is awkward. 22 | cursor int // offset where next read should end 23 | bitsRead uint8 24 | } 25 | 26 | // init initializes and resets the bit reader. 27 | func (b *bitReader) init(in []byte) error { 28 | if len(in) < 1 { 29 | return errors.New("corrupt stream: too short") 30 | } 31 | b.in = in 32 | // The highest bit of the last byte indicates where to start 33 | v := in[len(in)-1] 34 | if v == 0 { 35 | return errors.New("corrupt stream, did not find end of stream") 36 | } 37 | b.cursor = len(in) 38 | b.bitsRead = 64 39 | b.value = 0 40 | if len(in) >= 8 { 41 | b.fillFastStart() 42 | } else { 43 | b.fill() 44 | b.fill() 45 | } 46 | b.bitsRead += 8 - uint8(highBits(uint32(v))) 47 | return nil 48 | } 49 | 50 | // getBits will return n bits. n can be 0. 51 | func (b *bitReader) getBits(n uint8) int { 52 | if n == 0 /*|| b.bitsRead >= 64 */ { 53 | return 0 54 | } 55 | return int(b.get32BitsFast(n)) 56 | } 57 | 58 | // get32BitsFast requires that at least one bit is requested every time. 59 | // There are no checks if the buffer is filled. 60 | func (b *bitReader) get32BitsFast(n uint8) uint32 { 61 | const regMask = 64 - 1 62 | v := uint32((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) 63 | b.bitsRead += n 64 | return v 65 | } 66 | 67 | // fillFast() will make sure at least 32 bits are available. 68 | // There must be at least 4 bytes available. 69 | func (b *bitReader) fillFast() { 70 | if b.bitsRead < 32 { 71 | return 72 | } 73 | b.cursor -= 4 74 | b.value = (b.value << 32) | uint64(le.Load32(b.in, b.cursor)) 75 | b.bitsRead -= 32 76 | } 77 | 78 | // fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read. 79 | func (b *bitReader) fillFastStart() { 80 | b.cursor -= 8 81 | b.value = le.Load64(b.in, b.cursor) 82 | b.bitsRead = 0 83 | } 84 | 85 | // fill() will make sure at least 32 bits are available. 86 | func (b *bitReader) fill() { 87 | if b.bitsRead < 32 { 88 | return 89 | } 90 | if b.cursor >= 4 { 91 | b.cursor -= 4 92 | b.value = (b.value << 32) | uint64(le.Load32(b.in, b.cursor)) 93 | b.bitsRead -= 32 94 | return 95 | } 96 | 97 | b.bitsRead -= uint8(8 * b.cursor) 98 | for b.cursor > 0 { 99 | b.cursor -= 1 100 | b.value = (b.value << 8) | uint64(b.in[b.cursor]) 101 | } 102 | } 103 | 104 | // finished returns true if all bits have been read from the bit stream. 105 | func (b *bitReader) finished() bool { 106 | return b.cursor == 0 && b.bitsRead >= 64 107 | } 108 | 109 | // overread returns true if more bits have been requested than is on the stream. 110 | func (b *bitReader) overread() bool { 111 | return b.bitsRead > 64 112 | } 113 | 114 | // remain returns the number of bits remaining. 115 | func (b *bitReader) remain() uint { 116 | return 8*uint(b.cursor) + 64 - uint(b.bitsRead) 117 | } 118 | 119 | // close the bitstream and returns an error if out-of-buffer reads occurred. 120 | func (b *bitReader) close() error { 121 | // Release reference. 122 | b.in = nil 123 | b.cursor = 0 124 | if !b.finished() { 125 | return fmt.Errorf("%d extra bits on block, should be 0", b.remain()) 126 | } 127 | if b.bitsRead > 64 { 128 | return io.ErrUnexpectedEOF 129 | } 130 | return nil 131 | } 132 | 133 | func highBits(val uint32) (n uint32) { 134 | return uint32(bits.Len32(val) - 1) 135 | } 136 | -------------------------------------------------------------------------------- /zstd/blocktype_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type=blockType,literalsBlockType,seqCompMode,tableIndex"; DO NOT EDIT. 2 | 3 | package zstd 4 | 5 | import "strconv" 6 | 7 | func _() { 8 | // An "invalid array index" compiler error signifies that the constant values have changed. 9 | // Re-run the stringer command to generate them again. 10 | var x [1]struct{} 11 | _ = x[blockTypeRaw-0] 12 | _ = x[blockTypeRLE-1] 13 | _ = x[blockTypeCompressed-2] 14 | _ = x[blockTypeReserved-3] 15 | } 16 | 17 | const _blockType_name = "blockTypeRawblockTypeRLEblockTypeCompressedblockTypeReserved" 18 | 19 | var _blockType_index = [...]uint8{0, 12, 24, 43, 60} 20 | 21 | func (i blockType) String() string { 22 | if i >= blockType(len(_blockType_index)-1) { 23 | return "blockType(" + strconv.FormatInt(int64(i), 10) + ")" 24 | } 25 | return _blockType_name[_blockType_index[i]:_blockType_index[i+1]] 26 | } 27 | func _() { 28 | // An "invalid array index" compiler error signifies that the constant values have changed. 29 | // Re-run the stringer command to generate them again. 30 | var x [1]struct{} 31 | _ = x[literalsBlockRaw-0] 32 | _ = x[literalsBlockRLE-1] 33 | _ = x[literalsBlockCompressed-2] 34 | _ = x[literalsBlockTreeless-3] 35 | } 36 | 37 | const _literalsBlockType_name = "literalsBlockRawliteralsBlockRLEliteralsBlockCompressedliteralsBlockTreeless" 38 | 39 | var _literalsBlockType_index = [...]uint8{0, 16, 32, 55, 76} 40 | 41 | func (i literalsBlockType) String() string { 42 | if i >= literalsBlockType(len(_literalsBlockType_index)-1) { 43 | return "literalsBlockType(" + strconv.FormatInt(int64(i), 10) + ")" 44 | } 45 | return _literalsBlockType_name[_literalsBlockType_index[i]:_literalsBlockType_index[i+1]] 46 | } 47 | func _() { 48 | // An "invalid array index" compiler error signifies that the constant values have changed. 49 | // Re-run the stringer command to generate them again. 50 | var x [1]struct{} 51 | _ = x[compModePredefined-0] 52 | _ = x[compModeRLE-1] 53 | _ = x[compModeFSE-2] 54 | _ = x[compModeRepeat-3] 55 | } 56 | 57 | const _seqCompMode_name = "compModePredefinedcompModeRLEcompModeFSEcompModeRepeat" 58 | 59 | var _seqCompMode_index = [...]uint8{0, 18, 29, 40, 54} 60 | 61 | func (i seqCompMode) String() string { 62 | if i >= seqCompMode(len(_seqCompMode_index)-1) { 63 | return "seqCompMode(" + strconv.FormatInt(int64(i), 10) + ")" 64 | } 65 | return _seqCompMode_name[_seqCompMode_index[i]:_seqCompMode_index[i+1]] 66 | } 67 | func _() { 68 | // An "invalid array index" compiler error signifies that the constant values have changed. 69 | // Re-run the stringer command to generate them again. 70 | var x [1]struct{} 71 | _ = x[tableLiteralLengths-0] 72 | _ = x[tableOffsets-1] 73 | _ = x[tableMatchLengths-2] 74 | } 75 | 76 | const _tableIndex_name = "tableLiteralLengthstableOffsetstableMatchLengths" 77 | 78 | var _tableIndex_index = [...]uint8{0, 19, 31, 48} 79 | 80 | func (i tableIndex) String() string { 81 | if i >= tableIndex(len(_tableIndex_index)-1) { 82 | return "tableIndex(" + strconv.FormatInt(int64(i), 10) + ")" 83 | } 84 | return _tableIndex_name[_tableIndex_index[i]:_tableIndex_index[i+1]] 85 | } 86 | -------------------------------------------------------------------------------- /zstd/bytebuf.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019+ Klaus Post. All rights reserved. 2 | // License information can be found in the LICENSE file. 3 | // Based on work by Yann Collet, released under BSD License. 4 | 5 | package zstd 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | ) 11 | 12 | type byteBuffer interface { 13 | // Read up to 8 bytes. 14 | // Returns io.ErrUnexpectedEOF if this cannot be satisfied. 15 | readSmall(n int) ([]byte, error) 16 | 17 | // Read >8 bytes. 18 | // MAY use the destination slice. 19 | readBig(n int, dst []byte) ([]byte, error) 20 | 21 | // Read a single byte. 22 | readByte() (byte, error) 23 | 24 | // Skip n bytes. 25 | skipN(n int64) error 26 | } 27 | 28 | // in-memory buffer 29 | type byteBuf []byte 30 | 31 | func (b *byteBuf) readSmall(n int) ([]byte, error) { 32 | if debugAsserts && n > 8 { 33 | panic(fmt.Errorf("small read > 8 (%d). use readBig", n)) 34 | } 35 | bb := *b 36 | if len(bb) < n { 37 | return nil, io.ErrUnexpectedEOF 38 | } 39 | r := bb[:n] 40 | *b = bb[n:] 41 | return r, nil 42 | } 43 | 44 | func (b *byteBuf) readBig(n int, dst []byte) ([]byte, error) { 45 | bb := *b 46 | if len(bb) < n { 47 | return nil, io.ErrUnexpectedEOF 48 | } 49 | r := bb[:n] 50 | *b = bb[n:] 51 | return r, nil 52 | } 53 | 54 | func (b *byteBuf) readByte() (byte, error) { 55 | bb := *b 56 | if len(bb) < 1 { 57 | return 0, io.ErrUnexpectedEOF 58 | } 59 | r := bb[0] 60 | *b = bb[1:] 61 | return r, nil 62 | } 63 | 64 | func (b *byteBuf) skipN(n int64) error { 65 | bb := *b 66 | if n < 0 { 67 | return fmt.Errorf("negative skip (%d) requested", n) 68 | } 69 | if int64(len(bb)) < n { 70 | return io.ErrUnexpectedEOF 71 | } 72 | *b = bb[n:] 73 | return nil 74 | } 75 | 76 | // wrapper around a reader. 77 | type readerWrapper struct { 78 | r io.Reader 79 | tmp [8]byte 80 | } 81 | 82 | func (r *readerWrapper) readSmall(n int) ([]byte, error) { 83 | if debugAsserts && n > 8 { 84 | panic(fmt.Errorf("small read > 8 (%d). use readBig", n)) 85 | } 86 | n2, err := io.ReadFull(r.r, r.tmp[:n]) 87 | // We only really care about the actual bytes read. 88 | if err != nil { 89 | if err == io.EOF { 90 | return nil, io.ErrUnexpectedEOF 91 | } 92 | if debugDecoder { 93 | println("readSmall: got", n2, "want", n, "err", err) 94 | } 95 | return nil, err 96 | } 97 | return r.tmp[:n], nil 98 | } 99 | 100 | func (r *readerWrapper) readBig(n int, dst []byte) ([]byte, error) { 101 | if cap(dst) < n { 102 | dst = make([]byte, n) 103 | } 104 | n2, err := io.ReadFull(r.r, dst[:n]) 105 | if err == io.EOF && n > 0 { 106 | err = io.ErrUnexpectedEOF 107 | } 108 | return dst[:n2], err 109 | } 110 | 111 | func (r *readerWrapper) readByte() (byte, error) { 112 | n2, err := io.ReadFull(r.r, r.tmp[:1]) 113 | if err != nil { 114 | if err == io.EOF { 115 | err = io.ErrUnexpectedEOF 116 | } 117 | return 0, err 118 | } 119 | if n2 != 1 { 120 | return 0, io.ErrUnexpectedEOF 121 | } 122 | return r.tmp[0], nil 123 | } 124 | 125 | func (r *readerWrapper) skipN(n int64) error { 126 | n2, err := io.CopyN(io.Discard, r.r, n) 127 | if n2 != n { 128 | err = io.ErrUnexpectedEOF 129 | } 130 | return err 131 | } 132 | -------------------------------------------------------------------------------- /zstd/bytereader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019+ Klaus Post. All rights reserved. 2 | // License information can be found in the LICENSE file. 3 | // Based on work by Yann Collet, released under BSD License. 4 | 5 | package zstd 6 | 7 | // byteReader provides a byte reader that reads 8 | // little endian values from a byte stream. 9 | // The input stream is manually advanced. 10 | // The reader performs no bounds checks. 11 | type byteReader struct { 12 | b []byte 13 | off int 14 | } 15 | 16 | // advance the stream b n bytes. 17 | func (b *byteReader) advance(n uint) { 18 | b.off += int(n) 19 | } 20 | 21 | // overread returns whether we have advanced too far. 22 | func (b *byteReader) overread() bool { 23 | return b.off > len(b.b) 24 | } 25 | 26 | // Int32 returns a little endian int32 starting at current offset. 27 | func (b byteReader) Int32() int32 { 28 | b2 := b.b[b.off:] 29 | b2 = b2[:4] 30 | v3 := int32(b2[3]) 31 | v2 := int32(b2[2]) 32 | v1 := int32(b2[1]) 33 | v0 := int32(b2[0]) 34 | return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) 35 | } 36 | 37 | // Uint8 returns the next byte 38 | func (b *byteReader) Uint8() uint8 { 39 | v := b.b[b.off] 40 | return v 41 | } 42 | 43 | // Uint32 returns a little endian uint32 starting at current offset. 44 | func (b byteReader) Uint32() uint32 { 45 | if r := b.remain(); r < 4 { 46 | // Very rare 47 | v := uint32(0) 48 | for i := 1; i <= r; i++ { 49 | v = (v << 8) | uint32(b.b[len(b.b)-i]) 50 | } 51 | return v 52 | } 53 | b2 := b.b[b.off:] 54 | b2 = b2[:4] 55 | v3 := uint32(b2[3]) 56 | v2 := uint32(b2[2]) 57 | v1 := uint32(b2[1]) 58 | v0 := uint32(b2[0]) 59 | return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) 60 | } 61 | 62 | // Uint32NC returns a little endian uint32 starting at current offset. 63 | // The caller must be sure if there are at least 4 bytes left. 64 | func (b byteReader) Uint32NC() uint32 { 65 | b2 := b.b[b.off:] 66 | b2 = b2[:4] 67 | v3 := uint32(b2[3]) 68 | v2 := uint32(b2[2]) 69 | v1 := uint32(b2[1]) 70 | v0 := uint32(b2[0]) 71 | return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) 72 | } 73 | 74 | // unread returns the unread portion of the input. 75 | func (b byteReader) unread() []byte { 76 | return b.b[b.off:] 77 | } 78 | 79 | // remain will return the number of bytes remaining. 80 | func (b byteReader) remain() int { 81 | return len(b.b) - b.off 82 | } 83 | -------------------------------------------------------------------------------- /zstd/decodeheader_test.go: -------------------------------------------------------------------------------- 1 | package zstd 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io" 7 | "os" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestHeader_Decode(t *testing.T) { 13 | zr := testCreateZipReader("testdata/headers.zip", t) 14 | 15 | // Regenerate golden data... 16 | const regen = false 17 | golden := make(map[string]Header) 18 | if !regen { 19 | b, err := os.ReadFile("testdata/headers-want.json.zst") 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | dec, err := NewReader(nil) 24 | if err != nil { 25 | t.Fatal(err) 26 | } 27 | defer dec.Close() 28 | b, err = dec.DecodeAll(b, nil) 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | err = json.Unmarshal(b, &golden) 33 | if err != nil { 34 | t.Fatal(err) 35 | } 36 | } 37 | 38 | for i, tt := range zr.File { 39 | if !strings.HasSuffix(t.Name(), "") { 40 | continue 41 | } 42 | if testing.Short() && i > 100 { 43 | break 44 | } 45 | 46 | t.Run(tt.Name, func(t *testing.T) { 47 | r, err := tt.Open() 48 | if err != nil { 49 | t.Error(err) 50 | return 51 | } 52 | defer r.Close() 53 | b, err := io.ReadAll(r) 54 | if err != nil { 55 | t.Error(err) 56 | return 57 | } 58 | want, ok := golden[tt.Name] 59 | var got Header 60 | err = got.Decode(b) 61 | if err != nil { 62 | if ok { 63 | t.Errorf("got unexpected error %v", err) 64 | } 65 | return 66 | } 67 | if regen { 68 | // errored entries are not set 69 | golden[tt.Name] = got 70 | return 71 | } 72 | if !ok { 73 | t.Errorf("want error, got result: %v", got) 74 | } 75 | if want != got { 76 | t.Errorf("header mismatch:\nwant %#v\ngot %#v", want, got) 77 | } 78 | }) 79 | } 80 | if regen { 81 | w, err := os.Create("testdata/headers-want.json.zst") 82 | if err != nil { 83 | t.Fatal(err) 84 | } 85 | defer w.Close() 86 | enc, err := NewWriter(w, WithEncoderLevel(SpeedBestCompression)) 87 | if err != nil { 88 | t.Fatal(err) 89 | } 90 | b, err := json.Marshal(golden) 91 | if err != nil { 92 | t.Fatal(err) 93 | } 94 | enc.ReadFrom(bytes.NewBuffer(b)) 95 | enc.Close() 96 | t.SkipNow() 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /zstd/encoder_options_test.go: -------------------------------------------------------------------------------- 1 | package zstd 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | ) 7 | 8 | func TestEncoderLevelFromString(t *testing.T) { 9 | type args struct { 10 | s string 11 | } 12 | tests := []struct { 13 | name string 14 | args args 15 | want bool 16 | want1 EncoderLevel 17 | }{ 18 | { 19 | name: "fastest", 20 | args: args{s: "fastest"}, 21 | want: true, 22 | want1: SpeedFastest, 23 | }, 24 | { 25 | name: "fastest-upper", 26 | args: args{s: "FASTEST"}, 27 | want: true, 28 | want1: SpeedFastest, 29 | }, 30 | { 31 | name: "default", 32 | args: args{s: "default"}, 33 | want: true, 34 | want1: SpeedDefault, 35 | }, 36 | { 37 | name: "default-UPPER", 38 | args: args{s: "Default"}, 39 | want: true, 40 | want1: SpeedDefault, 41 | }, 42 | { 43 | name: "invalid", 44 | args: args{s: "invalid"}, 45 | want: false, 46 | want1: SpeedDefault, 47 | }, 48 | { 49 | name: "unknown", 50 | args: args{s: "unknown"}, 51 | want: false, 52 | want1: SpeedDefault, 53 | }, 54 | { 55 | name: "empty", 56 | args: args{s: ""}, 57 | want: false, 58 | want1: SpeedDefault, 59 | }, 60 | { 61 | name: "fastest-string", 62 | args: args{s: SpeedFastest.String()}, 63 | want: true, 64 | want1: SpeedFastest, 65 | }, 66 | { 67 | name: "default-string", 68 | args: args{s: SpeedDefault.String()}, 69 | want: true, 70 | want1: SpeedDefault, 71 | }, 72 | } 73 | for _, tt := range tests { 74 | t.Run(tt.name, func(t *testing.T) { 75 | got, got1 := EncoderLevelFromString(tt.args.s) 76 | if got != tt.want { 77 | t.Errorf("EncoderLevelFromString() got = %v, want %v", got, tt.want) 78 | } 79 | if got1 != tt.want1 { 80 | t.Errorf("EncoderLevelFromString() got1 = %v, want %v", got1, tt.want1) 81 | } 82 | }) 83 | } 84 | } 85 | 86 | func TestEncoderLevelFromZstd(t *testing.T) { 87 | type args struct { 88 | level int 89 | } 90 | tests := []struct { 91 | name string 92 | args args 93 | want EncoderLevel 94 | }{ 95 | { 96 | name: "level-1", 97 | args: args{level: 1}, 98 | want: SpeedFastest, 99 | }, 100 | { 101 | name: "level-minus1", 102 | args: args{level: -1}, 103 | want: SpeedFastest, 104 | }, 105 | { 106 | name: "level-3", 107 | args: args{level: 3}, 108 | want: SpeedDefault, 109 | }, 110 | { 111 | name: "level-4", 112 | args: args{level: 4}, 113 | want: SpeedDefault, 114 | }, 115 | } 116 | for _, tt := range tests { 117 | t.Run(tt.name, func(t *testing.T) { 118 | if got := EncoderLevelFromZstd(tt.args.level); got != tt.want { 119 | t.Errorf("EncoderLevelFromZstd() = %v, want %v", got, tt.want) 120 | } 121 | }) 122 | } 123 | } 124 | 125 | func TestWindowSize(t *testing.T) { 126 | tests := []struct { 127 | windowSize int 128 | err bool 129 | }{ 130 | {1 << 9, true}, 131 | {1 << 10, false}, 132 | {(1 << 10) + 1, true}, 133 | {(1 << 10) * 3, true}, 134 | {MaxWindowSize, false}, 135 | } 136 | for _, tt := range tests { 137 | t.Run(strconv.Itoa(tt.windowSize), func(t *testing.T) { 138 | var options encoderOptions 139 | err := WithWindowSize(tt.windowSize)(&options) 140 | if tt.err { 141 | if err == nil { 142 | t.Error("did not get error for invalid window size") 143 | } 144 | } else { 145 | if err != nil { 146 | t.Error("got error for valid window size") 147 | } 148 | if options.windowSize != tt.windowSize { 149 | t.Error("failed to set window size") 150 | } 151 | } 152 | }) 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /zstd/example_test.go: -------------------------------------------------------------------------------- 1 | package zstd_test 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "github.com/klauspost/compress/zstd" 8 | ) 9 | 10 | func ExampleWithEncoderDictRaw() { 11 | // "Raw" dictionaries can be used for compressed delta encoding. 12 | 13 | source := []byte(` 14 | This is the source file. Compression of the target file with 15 | the source file as the dictionary will produce a compressed 16 | delta encoding of the target file.`) 17 | target := []byte(` 18 | This is the target file. Decompression of the delta encoding with 19 | the source file as the dictionary will produce this file.`) 20 | 21 | // The dictionary id is arbitrary. We use zero for compatibility 22 | // with zstd --patch-from, but applications can use any id 23 | // not in the range [32768, 1<<31). 24 | const id = 0 25 | 26 | bestLevel := zstd.WithEncoderLevel(zstd.SpeedBestCompression) 27 | 28 | w, _ := zstd.NewWriter(nil, bestLevel, 29 | zstd.WithEncoderDictRaw(id, source)) 30 | delta := w.EncodeAll(target, nil) 31 | 32 | r, _ := zstd.NewReader(nil, zstd.WithDecoderDictRaw(id, source)) 33 | out, err := r.DecodeAll(delta, nil) 34 | if err != nil || !bytes.Equal(out, target) { 35 | panic("decoding error") 36 | } 37 | 38 | // Ordinary compression, for reference. 39 | w, _ = zstd.NewWriter(nil, bestLevel) 40 | compressed := w.EncodeAll(target, nil) 41 | 42 | // Check that the delta is at most half as big as the compressed file. 43 | fmt.Println(len(delta) < len(compressed)/2) 44 | // Output: 45 | // true 46 | } 47 | -------------------------------------------------------------------------------- /zstd/fse_decoder_amd64.go: -------------------------------------------------------------------------------- 1 | //go:build amd64 && !appengine && !noasm && gc 2 | // +build amd64,!appengine,!noasm,gc 3 | 4 | package zstd 5 | 6 | import ( 7 | "fmt" 8 | ) 9 | 10 | type buildDtableAsmContext struct { 11 | // inputs 12 | stateTable *uint16 13 | norm *int16 14 | dt *uint64 15 | 16 | // outputs --- set by the procedure in the case of error; 17 | // for interpretation please see the error handling part below 18 | errParam1 uint64 19 | errParam2 uint64 20 | } 21 | 22 | // buildDtable_asm is an x86 assembly implementation of fseDecoder.buildDtable. 23 | // Function returns non-zero exit code on error. 24 | // 25 | //go:noescape 26 | func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int 27 | 28 | // please keep in sync with _generate/gen_fse.go 29 | const ( 30 | errorCorruptedNormalizedCounter = 1 31 | errorNewStateTooBig = 2 32 | errorNewStateNoBits = 3 33 | ) 34 | 35 | // buildDtable will build the decoding table. 36 | func (s *fseDecoder) buildDtable() error { 37 | ctx := buildDtableAsmContext{ 38 | stateTable: &s.stateTable[0], 39 | norm: &s.norm[0], 40 | dt: (*uint64)(&s.dt[0]), 41 | } 42 | code := buildDtable_asm(s, &ctx) 43 | 44 | if code != 0 { 45 | switch code { 46 | case errorCorruptedNormalizedCounter: 47 | position := ctx.errParam1 48 | return fmt.Errorf("corrupted input (position=%d, expected 0)", position) 49 | 50 | case errorNewStateTooBig: 51 | newState := decSymbol(ctx.errParam1) 52 | size := ctx.errParam2 53 | return fmt.Errorf("newState (%d) outside table size (%d)", newState, size) 54 | 55 | case errorNewStateNoBits: 56 | newState := decSymbol(ctx.errParam1) 57 | oldState := decSymbol(ctx.errParam2) 58 | return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, oldState) 59 | 60 | default: 61 | return fmt.Errorf("buildDtable_asm returned unhandled nonzero code = %d", code) 62 | } 63 | } 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /zstd/fse_decoder_amd64.s: -------------------------------------------------------------------------------- 1 | // Code generated by command: go run gen_fse.go -out ../fse_decoder_amd64.s -pkg=zstd. DO NOT EDIT. 2 | 3 | //go:build !appengine && !noasm && gc && !noasm 4 | 5 | // func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int 6 | TEXT ·buildDtable_asm(SB), $0-24 7 | MOVQ ctx+8(FP), CX 8 | MOVQ s+0(FP), DI 9 | 10 | // Load values 11 | MOVBQZX 4098(DI), DX 12 | XORQ AX, AX 13 | BTSQ DX, AX 14 | MOVQ (CX), BX 15 | MOVQ 16(CX), SI 16 | LEAQ -1(AX), R8 17 | MOVQ 8(CX), CX 18 | MOVWQZX 4096(DI), DI 19 | 20 | // End load values 21 | // Init, lay down lowprob symbols 22 | XORQ R9, R9 23 | JMP init_main_loop_condition 24 | 25 | init_main_loop: 26 | MOVWQSX (CX)(R9*2), R10 27 | CMPW R10, $-1 28 | JNE do_not_update_high_threshold 29 | MOVB R9, 1(SI)(R8*8) 30 | DECQ R8 31 | MOVQ $0x0000000000000001, R10 32 | 33 | do_not_update_high_threshold: 34 | MOVW R10, (BX)(R9*2) 35 | INCQ R9 36 | 37 | init_main_loop_condition: 38 | CMPQ R9, DI 39 | JL init_main_loop 40 | 41 | // Spread symbols 42 | // Calculate table step 43 | MOVQ AX, R9 44 | SHRQ $0x01, R9 45 | MOVQ AX, R10 46 | SHRQ $0x03, R10 47 | LEAQ 3(R9)(R10*1), R9 48 | 49 | // Fill add bits values 50 | LEAQ -1(AX), R10 51 | XORQ R11, R11 52 | XORQ R12, R12 53 | JMP spread_main_loop_condition 54 | 55 | spread_main_loop: 56 | XORQ R13, R13 57 | MOVWQSX (CX)(R12*2), R14 58 | JMP spread_inner_loop_condition 59 | 60 | spread_inner_loop: 61 | MOVB R12, 1(SI)(R11*8) 62 | 63 | adjust_position: 64 | ADDQ R9, R11 65 | ANDQ R10, R11 66 | CMPQ R11, R8 67 | JG adjust_position 68 | INCQ R13 69 | 70 | spread_inner_loop_condition: 71 | CMPQ R13, R14 72 | JL spread_inner_loop 73 | INCQ R12 74 | 75 | spread_main_loop_condition: 76 | CMPQ R12, DI 77 | JL spread_main_loop 78 | TESTQ R11, R11 79 | JZ spread_check_ok 80 | MOVQ ctx+8(FP), AX 81 | MOVQ R11, 24(AX) 82 | MOVQ $+1, ret+16(FP) 83 | RET 84 | 85 | spread_check_ok: 86 | // Build Decoding table 87 | XORQ DI, DI 88 | 89 | build_table_main_table: 90 | MOVBQZX 1(SI)(DI*8), CX 91 | MOVWQZX (BX)(CX*2), R8 92 | LEAQ 1(R8), R9 93 | MOVW R9, (BX)(CX*2) 94 | MOVQ R8, R9 95 | BSRQ R9, R9 96 | MOVQ DX, CX 97 | SUBQ R9, CX 98 | SHLQ CL, R8 99 | SUBQ AX, R8 100 | MOVB CL, (SI)(DI*8) 101 | MOVW R8, 2(SI)(DI*8) 102 | CMPQ R8, AX 103 | JLE build_table_check1_ok 104 | MOVQ ctx+8(FP), CX 105 | MOVQ R8, 24(CX) 106 | MOVQ AX, 32(CX) 107 | MOVQ $+2, ret+16(FP) 108 | RET 109 | 110 | build_table_check1_ok: 111 | TESTB CL, CL 112 | JNZ build_table_check2_ok 113 | CMPW R8, DI 114 | JNE build_table_check2_ok 115 | MOVQ ctx+8(FP), AX 116 | MOVQ R8, 24(AX) 117 | MOVQ DI, 32(AX) 118 | MOVQ $+3, ret+16(FP) 119 | RET 120 | 121 | build_table_check2_ok: 122 | INCQ DI 123 | CMPQ DI, AX 124 | JL build_table_main_table 125 | MOVQ $+0, ret+16(FP) 126 | RET 127 | -------------------------------------------------------------------------------- /zstd/fse_decoder_generic.go: -------------------------------------------------------------------------------- 1 | //go:build !amd64 || appengine || !gc || noasm 2 | // +build !amd64 appengine !gc noasm 3 | 4 | package zstd 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | ) 10 | 11 | // buildDtable will build the decoding table. 12 | func (s *fseDecoder) buildDtable() error { 13 | tableSize := uint32(1 << s.actualTableLog) 14 | highThreshold := tableSize - 1 15 | symbolNext := s.stateTable[:256] 16 | 17 | // Init, lay down lowprob symbols 18 | { 19 | for i, v := range s.norm[:s.symbolLen] { 20 | if v == -1 { 21 | s.dt[highThreshold].setAddBits(uint8(i)) 22 | highThreshold-- 23 | v = 1 24 | } 25 | symbolNext[i] = uint16(v) 26 | } 27 | } 28 | 29 | // Spread symbols 30 | { 31 | tableMask := tableSize - 1 32 | step := tableStep(tableSize) 33 | position := uint32(0) 34 | for ss, v := range s.norm[:s.symbolLen] { 35 | for i := 0; i < int(v); i++ { 36 | s.dt[position].setAddBits(uint8(ss)) 37 | for { 38 | // lowprob area 39 | position = (position + step) & tableMask 40 | if position <= highThreshold { 41 | break 42 | } 43 | } 44 | } 45 | } 46 | if position != 0 { 47 | // position must reach all cells once, otherwise normalizedCounter is incorrect 48 | return errors.New("corrupted input (position != 0)") 49 | } 50 | } 51 | 52 | // Build Decoding table 53 | { 54 | tableSize := uint16(1 << s.actualTableLog) 55 | for u, v := range s.dt[:tableSize] { 56 | symbol := v.addBits() 57 | nextState := symbolNext[symbol] 58 | symbolNext[symbol] = nextState + 1 59 | nBits := s.actualTableLog - byte(highBits(uint32(nextState))) 60 | s.dt[u&maxTableMask].setNBits(nBits) 61 | newState := (nextState << nBits) - tableSize 62 | if newState > tableSize { 63 | return fmt.Errorf("newState (%d) outside table size (%d)", newState, tableSize) 64 | } 65 | if newState == uint16(u) && nBits == 0 { 66 | // Seems weird that this is possible with nbits > 0. 67 | return fmt.Errorf("newState (%d) == oldState (%d) and no bits", newState, u) 68 | } 69 | s.dt[u&maxTableMask].setNewState(newState) 70 | } 71 | } 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /zstd/hash.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019+ Klaus Post. All rights reserved. 2 | // License information can be found in the LICENSE file. 3 | // Based on work by Yann Collet, released under BSD License. 4 | 5 | package zstd 6 | 7 | const ( 8 | prime3bytes = 506832829 9 | prime4bytes = 2654435761 10 | prime5bytes = 889523592379 11 | prime6bytes = 227718039650203 12 | prime7bytes = 58295818150454627 13 | prime8bytes = 0xcf1bbcdcb7a56463 14 | ) 15 | 16 | // hashLen returns a hash of the lowest mls bytes of with length output bits. 17 | // mls must be >=3 and <=8. Any other value will return hash for 4 bytes. 18 | // length should always be < 32. 19 | // Preferably length and mls should be a constant for inlining. 20 | func hashLen(u uint64, length, mls uint8) uint32 { 21 | switch mls { 22 | case 3: 23 | return (uint32(u<<8) * prime3bytes) >> (32 - length) 24 | case 5: 25 | return uint32(((u << (64 - 40)) * prime5bytes) >> (64 - length)) 26 | case 6: 27 | return uint32(((u << (64 - 48)) * prime6bytes) >> (64 - length)) 28 | case 7: 29 | return uint32(((u << (64 - 56)) * prime7bytes) >> (64 - length)) 30 | case 8: 31 | return uint32((u * prime8bytes) >> (64 - length)) 32 | default: 33 | return (uint32(u) * prime4bytes) >> (32 - length) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /zstd/history.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019+ Klaus Post. All rights reserved. 2 | // License information can be found in the LICENSE file. 3 | // Based on work by Yann Collet, released under BSD License. 4 | 5 | package zstd 6 | 7 | import ( 8 | "github.com/klauspost/compress/huff0" 9 | ) 10 | 11 | // history contains the information transferred between blocks. 12 | type history struct { 13 | // Literal decompression 14 | huffTree *huff0.Scratch 15 | 16 | // Sequence decompression 17 | decoders sequenceDecs 18 | recentOffsets [3]int 19 | 20 | // History buffer... 21 | b []byte 22 | 23 | // ignoreBuffer is meant to ignore a number of bytes 24 | // when checking for matches in history 25 | ignoreBuffer int 26 | 27 | windowSize int 28 | allocFrameBuffer int // needed? 29 | error bool 30 | dict *dict 31 | } 32 | 33 | // reset will reset the history to initial state of a frame. 34 | // The history must already have been initialized to the desired size. 35 | func (h *history) reset() { 36 | h.b = h.b[:0] 37 | h.ignoreBuffer = 0 38 | h.error = false 39 | h.recentOffsets = [3]int{1, 4, 8} 40 | h.decoders.freeDecoders() 41 | h.decoders = sequenceDecs{br: h.decoders.br} 42 | h.freeHuffDecoder() 43 | h.huffTree = nil 44 | h.dict = nil 45 | //printf("history created: %+v (l: %d, c: %d)", *h, len(h.b), cap(h.b)) 46 | } 47 | 48 | func (h *history) freeHuffDecoder() { 49 | if h.huffTree != nil { 50 | if h.dict == nil || h.dict.litEnc != h.huffTree { 51 | huffDecoderPool.Put(h.huffTree) 52 | h.huffTree = nil 53 | } 54 | } 55 | } 56 | 57 | func (h *history) setDict(dict *dict) { 58 | if dict == nil { 59 | return 60 | } 61 | h.dict = dict 62 | h.decoders.litLengths = dict.llDec 63 | h.decoders.offsets = dict.ofDec 64 | h.decoders.matchLengths = dict.mlDec 65 | h.decoders.dict = dict.content 66 | h.recentOffsets = dict.offsets 67 | h.huffTree = dict.litEnc 68 | } 69 | 70 | // append bytes to history. 71 | // This function will make sure there is space for it, 72 | // if the buffer has been allocated with enough extra space. 73 | func (h *history) append(b []byte) { 74 | if len(b) >= h.windowSize { 75 | // Discard all history by simply overwriting 76 | h.b = h.b[:h.windowSize] 77 | copy(h.b, b[len(b)-h.windowSize:]) 78 | return 79 | } 80 | 81 | // If there is space, append it. 82 | if len(b) < cap(h.b)-len(h.b) { 83 | h.b = append(h.b, b...) 84 | return 85 | } 86 | 87 | // Move data down so we only have window size left. 88 | // We know we have less than window size in b at this point. 89 | discard := len(b) + len(h.b) - h.windowSize 90 | copy(h.b, h.b[discard:]) 91 | h.b = h.b[:h.windowSize] 92 | copy(h.b[h.windowSize-len(b):], b) 93 | } 94 | 95 | // ensureBlock will ensure there is space for at least one block... 96 | func (h *history) ensureBlock() { 97 | if cap(h.b) < h.allocFrameBuffer { 98 | h.b = make([]byte, 0, h.allocFrameBuffer) 99 | return 100 | } 101 | 102 | avail := cap(h.b) - len(h.b) 103 | if avail >= h.windowSize || avail > maxCompressedBlockSize { 104 | return 105 | } 106 | // Move data down so we only have window size left. 107 | // We know we have less than window size in b at this point. 108 | discard := len(h.b) - h.windowSize 109 | copy(h.b, h.b[discard:]) 110 | h.b = h.b[:h.windowSize] 111 | } 112 | 113 | // append bytes to history without ever discarding anything. 114 | func (h *history) appendKeep(b []byte) { 115 | h.b = append(h.b, b...) 116 | } 117 | -------------------------------------------------------------------------------- /zstd/internal/xxhash/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Caleb Spare 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /zstd/internal/xxhash/README.md: -------------------------------------------------------------------------------- 1 | # xxhash 2 | 3 | VENDORED: Go to [github.com/cespare/xxhash](https://github.com/cespare/xxhash) for original package. 4 | 5 | xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a 6 | high-quality hashing algorithm that is much faster than anything in the Go 7 | standard library. 8 | 9 | This package provides a straightforward API: 10 | 11 | ``` 12 | func Sum64(b []byte) uint64 13 | func Sum64String(s string) uint64 14 | type Digest struct{ ... } 15 | func New() *Digest 16 | ``` 17 | 18 | The `Digest` type implements hash.Hash64. Its key methods are: 19 | 20 | ``` 21 | func (*Digest) Write([]byte) (int, error) 22 | func (*Digest) WriteString(string) (int, error) 23 | func (*Digest) Sum64() uint64 24 | ``` 25 | 26 | The package is written with optimized pure Go and also contains even faster 27 | assembly implementations for amd64 and arm64. If desired, the `purego` build tag 28 | opts into using the Go code even on those architectures. 29 | 30 | [xxHash]: http://cyan4973.github.io/xxHash/ 31 | 32 | ## Compatibility 33 | 34 | This package is in a module and the latest code is in version 2 of the module. 35 | You need a version of Go with at least "minimal module compatibility" to use 36 | github.com/cespare/xxhash/v2: 37 | 38 | * 1.9.7+ for Go 1.9 39 | * 1.10.3+ for Go 1.10 40 | * Go 1.11 or later 41 | 42 | I recommend using the latest release of Go. 43 | 44 | ## Benchmarks 45 | 46 | Here are some quick benchmarks comparing the pure-Go and assembly 47 | implementations of Sum64. 48 | 49 | | input size | purego | asm | 50 | | ---------- | --------- | --------- | 51 | | 4 B | 1.3 GB/s | 1.2 GB/s | 52 | | 16 B | 2.9 GB/s | 3.5 GB/s | 53 | | 100 B | 6.9 GB/s | 8.1 GB/s | 54 | | 4 KB | 11.7 GB/s | 16.7 GB/s | 55 | | 10 MB | 12.0 GB/s | 17.3 GB/s | 56 | 57 | These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C 58 | CPU using the following commands under Go 1.19.2: 59 | 60 | ``` 61 | benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$') 62 | benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$') 63 | ``` 64 | 65 | ## Projects using this package 66 | 67 | - [InfluxDB](https://github.com/influxdata/influxdb) 68 | - [Prometheus](https://github.com/prometheus/prometheus) 69 | - [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) 70 | - [FreeCache](https://github.com/coocood/freecache) 71 | - [FastCache](https://github.com/VictoriaMetrics/fastcache) 72 | -------------------------------------------------------------------------------- /zstd/internal/xxhash/xxhash_asm.go: -------------------------------------------------------------------------------- 1 | //go:build (amd64 || arm64) && !appengine && gc && !purego && !noasm 2 | // +build amd64 arm64 3 | // +build !appengine 4 | // +build gc 5 | // +build !purego 6 | // +build !noasm 7 | 8 | package xxhash 9 | 10 | // Sum64 computes the 64-bit xxHash digest of b. 11 | // 12 | //go:noescape 13 | func Sum64(b []byte) uint64 14 | 15 | //go:noescape 16 | func writeBlocks(s *Digest, b []byte) int 17 | -------------------------------------------------------------------------------- /zstd/internal/xxhash/xxhash_other.go: -------------------------------------------------------------------------------- 1 | //go:build (!amd64 && !arm64) || appengine || !gc || purego || noasm 2 | // +build !amd64,!arm64 appengine !gc purego noasm 3 | 4 | package xxhash 5 | 6 | // Sum64 computes the 64-bit xxHash digest of b. 7 | func Sum64(b []byte) uint64 { 8 | // A simpler version would be 9 | // d := New() 10 | // d.Write(b) 11 | // return d.Sum64() 12 | // but this is faster, particularly for small inputs. 13 | 14 | n := len(b) 15 | var h uint64 16 | 17 | if n >= 32 { 18 | v1 := primes[0] + prime2 19 | v2 := prime2 20 | v3 := uint64(0) 21 | v4 := -primes[0] 22 | for len(b) >= 32 { 23 | v1 = round(v1, u64(b[0:8:len(b)])) 24 | v2 = round(v2, u64(b[8:16:len(b)])) 25 | v3 = round(v3, u64(b[16:24:len(b)])) 26 | v4 = round(v4, u64(b[24:32:len(b)])) 27 | b = b[32:len(b):len(b)] 28 | } 29 | h = rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4) 30 | h = mergeRound(h, v1) 31 | h = mergeRound(h, v2) 32 | h = mergeRound(h, v3) 33 | h = mergeRound(h, v4) 34 | } else { 35 | h = prime5 36 | } 37 | 38 | h += uint64(n) 39 | 40 | for ; len(b) >= 8; b = b[8:] { 41 | k1 := round(0, u64(b[:8])) 42 | h ^= k1 43 | h = rol27(h)*prime1 + prime4 44 | } 45 | if len(b) >= 4 { 46 | h ^= uint64(u32(b[:4])) * prime1 47 | h = rol23(h)*prime2 + prime3 48 | b = b[4:] 49 | } 50 | for ; len(b) > 0; b = b[1:] { 51 | h ^= uint64(b[0]) * prime5 52 | h = rol11(h) * prime1 53 | } 54 | 55 | h ^= h >> 33 56 | h *= prime2 57 | h ^= h >> 29 58 | h *= prime3 59 | h ^= h >> 32 60 | 61 | return h 62 | } 63 | 64 | func writeBlocks(d *Digest, b []byte) int { 65 | v1, v2, v3, v4 := d.v1, d.v2, d.v3, d.v4 66 | n := len(b) 67 | for len(b) >= 32 { 68 | v1 = round(v1, u64(b[0:8:len(b)])) 69 | v2 = round(v2, u64(b[8:16:len(b)])) 70 | v3 = round(v3, u64(b[16:24:len(b)])) 71 | v4 = round(v4, u64(b[24:32:len(b)])) 72 | b = b[32:len(b):len(b)] 73 | } 74 | d.v1, d.v2, d.v3, d.v4 = v1, v2, v3, v4 75 | return n - len(b) 76 | } 77 | -------------------------------------------------------------------------------- /zstd/internal/xxhash/xxhash_safe.go: -------------------------------------------------------------------------------- 1 | package xxhash 2 | 3 | // Sum64String computes the 64-bit xxHash digest of s. 4 | func Sum64String(s string) uint64 { 5 | return Sum64([]byte(s)) 6 | } 7 | 8 | // WriteString adds more data to d. It always returns len(s), nil. 9 | func (d *Digest) WriteString(s string) (n int, err error) { 10 | return d.Write([]byte(s)) 11 | } 12 | -------------------------------------------------------------------------------- /zstd/matchlen_amd64.go: -------------------------------------------------------------------------------- 1 | //go:build amd64 && !appengine && !noasm && gc 2 | // +build amd64,!appengine,!noasm,gc 3 | 4 | // Copyright 2019+ Klaus Post. All rights reserved. 5 | // License information can be found in the LICENSE file. 6 | 7 | package zstd 8 | 9 | // matchLen returns how many bytes match in a and b 10 | // 11 | // It assumes that: 12 | // 13 | // len(a) <= len(b) and len(a) > 0 14 | // 15 | //go:noescape 16 | func matchLen(a []byte, b []byte) int 17 | -------------------------------------------------------------------------------- /zstd/matchlen_amd64.s: -------------------------------------------------------------------------------- 1 | // Copied from S2 implementation. 2 | 3 | //go:build !appengine && !noasm && gc && !noasm 4 | 5 | #include "textflag.h" 6 | 7 | // func matchLen(a []byte, b []byte) int 8 | TEXT ·matchLen(SB), NOSPLIT, $0-56 9 | MOVQ a_base+0(FP), AX 10 | MOVQ b_base+24(FP), CX 11 | MOVQ a_len+8(FP), DX 12 | 13 | // matchLen 14 | XORL SI, SI 15 | CMPL DX, $0x08 16 | JB matchlen_match4_standalone 17 | 18 | matchlen_loopback_standalone: 19 | MOVQ (AX)(SI*1), BX 20 | XORQ (CX)(SI*1), BX 21 | JZ matchlen_loop_standalone 22 | 23 | #ifdef GOAMD64_v3 24 | TZCNTQ BX, BX 25 | #else 26 | BSFQ BX, BX 27 | #endif 28 | SHRL $0x03, BX 29 | LEAL (SI)(BX*1), SI 30 | JMP gen_match_len_end 31 | 32 | matchlen_loop_standalone: 33 | LEAL -8(DX), DX 34 | LEAL 8(SI), SI 35 | CMPL DX, $0x08 36 | JAE matchlen_loopback_standalone 37 | 38 | matchlen_match4_standalone: 39 | CMPL DX, $0x04 40 | JB matchlen_match2_standalone 41 | MOVL (AX)(SI*1), BX 42 | CMPL (CX)(SI*1), BX 43 | JNE matchlen_match2_standalone 44 | LEAL -4(DX), DX 45 | LEAL 4(SI), SI 46 | 47 | matchlen_match2_standalone: 48 | CMPL DX, $0x02 49 | JB matchlen_match1_standalone 50 | MOVW (AX)(SI*1), BX 51 | CMPW (CX)(SI*1), BX 52 | JNE matchlen_match1_standalone 53 | LEAL -2(DX), DX 54 | LEAL 2(SI), SI 55 | 56 | matchlen_match1_standalone: 57 | CMPL DX, $0x01 58 | JB gen_match_len_end 59 | MOVB (AX)(SI*1), BL 60 | CMPB (CX)(SI*1), BL 61 | JNE gen_match_len_end 62 | INCL SI 63 | 64 | gen_match_len_end: 65 | MOVQ SI, ret+48(FP) 66 | RET 67 | -------------------------------------------------------------------------------- /zstd/matchlen_generic.go: -------------------------------------------------------------------------------- 1 | //go:build !amd64 || appengine || !gc || noasm 2 | // +build !amd64 appengine !gc noasm 3 | 4 | // Copyright 2019+ Klaus Post. All rights reserved. 5 | // License information can be found in the LICENSE file. 6 | 7 | package zstd 8 | 9 | import ( 10 | "math/bits" 11 | 12 | "github.com/klauspost/compress/internal/le" 13 | ) 14 | 15 | // matchLen returns the maximum common prefix length of a and b. 16 | // a must be the shortest of the two. 17 | func matchLen(a, b []byte) (n int) { 18 | left := len(a) 19 | for left >= 8 { 20 | diff := le.Load64(a, n) ^ le.Load64(b, n) 21 | if diff != 0 { 22 | return n + bits.TrailingZeros64(diff)>>3 23 | } 24 | n += 8 25 | left -= 8 26 | } 27 | a = a[n:] 28 | b = b[n:] 29 | 30 | for i := range a { 31 | if a[i] != b[i] { 32 | break 33 | } 34 | n++ 35 | } 36 | return n 37 | 38 | } 39 | -------------------------------------------------------------------------------- /zstd/race_enabled_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019+ Klaus Post. All rights reserved. 2 | // License information can be found in the LICENSE file. 3 | 4 | //go:build race 5 | // +build race 6 | 7 | package zstd 8 | 9 | func init() { 10 | isRaceTest = true 11 | } 12 | -------------------------------------------------------------------------------- /zstd/seqenc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019+ Klaus Post. All rights reserved. 2 | // License information can be found in the LICENSE file. 3 | // Based on work by Yann Collet, released under BSD License. 4 | 5 | package zstd 6 | 7 | import "math/bits" 8 | 9 | type seqCoders struct { 10 | llEnc, ofEnc, mlEnc *fseEncoder 11 | llPrev, ofPrev, mlPrev *fseEncoder 12 | } 13 | 14 | // swap coders with another (block). 15 | func (s *seqCoders) swap(other *seqCoders) { 16 | *s, *other = *other, *s 17 | } 18 | 19 | // setPrev will update the previous encoders to the actually used ones 20 | // and make sure a fresh one is in the main slot. 21 | func (s *seqCoders) setPrev(ll, ml, of *fseEncoder) { 22 | compareSwap := func(used *fseEncoder, current, prev **fseEncoder) { 23 | // We used the new one, more current to history and reuse the previous history 24 | if *current == used { 25 | *prev, *current = *current, *prev 26 | c := *current 27 | p := *prev 28 | c.reUsed = false 29 | p.reUsed = true 30 | return 31 | } 32 | if used == *prev { 33 | return 34 | } 35 | // Ensure we cannot reuse by accident 36 | prevEnc := *prev 37 | prevEnc.symbolLen = 0 38 | } 39 | compareSwap(ll, &s.llEnc, &s.llPrev) 40 | compareSwap(ml, &s.mlEnc, &s.mlPrev) 41 | compareSwap(of, &s.ofEnc, &s.ofPrev) 42 | } 43 | 44 | func highBit(val uint32) (n uint32) { 45 | return uint32(bits.Len32(val) - 1) 46 | } 47 | 48 | var llCodeTable = [64]byte{0, 1, 2, 3, 4, 5, 6, 7, 49 | 8, 9, 10, 11, 12, 13, 14, 15, 50 | 16, 16, 17, 17, 18, 18, 19, 19, 51 | 20, 20, 20, 20, 21, 21, 21, 21, 52 | 22, 22, 22, 22, 22, 22, 22, 22, 53 | 23, 23, 23, 23, 23, 23, 23, 23, 54 | 24, 24, 24, 24, 24, 24, 24, 24, 55 | 24, 24, 24, 24, 24, 24, 24, 24} 56 | 57 | // Up to 6 bits 58 | const maxLLCode = 35 59 | 60 | // llBitsTable translates from ll code to number of bits. 61 | var llBitsTable = [maxLLCode + 1]byte{ 62 | 0, 0, 0, 0, 0, 0, 0, 0, 63 | 0, 0, 0, 0, 0, 0, 0, 0, 64 | 1, 1, 1, 1, 2, 2, 3, 3, 65 | 4, 6, 7, 8, 9, 10, 11, 12, 66 | 13, 14, 15, 16} 67 | 68 | // llCode returns the code that represents the literal length requested. 69 | func llCode(litLength uint32) uint8 { 70 | const llDeltaCode = 19 71 | if litLength <= 63 { 72 | return llCodeTable[litLength&63] 73 | } 74 | return uint8(highBit(litLength)) + llDeltaCode 75 | } 76 | 77 | var mlCodeTable = [128]byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 78 | 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 79 | 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, 80 | 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, 81 | 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 82 | 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 83 | 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 84 | 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42} 85 | 86 | // Up to 6 bits 87 | const maxMLCode = 52 88 | 89 | // mlBitsTable translates from ml code to number of bits. 90 | var mlBitsTable = [maxMLCode + 1]byte{ 91 | 0, 0, 0, 0, 0, 0, 0, 0, 92 | 0, 0, 0, 0, 0, 0, 0, 0, 93 | 0, 0, 0, 0, 0, 0, 0, 0, 94 | 0, 0, 0, 0, 0, 0, 0, 0, 95 | 1, 1, 1, 1, 2, 2, 3, 3, 96 | 4, 4, 5, 7, 8, 9, 10, 11, 97 | 12, 13, 14, 15, 16} 98 | 99 | // note : mlBase = matchLength - MINMATCH; 100 | // because it's the format it's stored in seqStore->sequences 101 | func mlCode(mlBase uint32) uint8 { 102 | const mlDeltaCode = 36 103 | if mlBase <= 127 { 104 | return mlCodeTable[mlBase&127] 105 | } 106 | return uint8(highBit(mlBase)) + mlDeltaCode 107 | } 108 | 109 | func ofCode(offset uint32) uint8 { 110 | // A valid offset will always be > 0. 111 | return uint8(bits.Len32(offset) - 1) 112 | } 113 | -------------------------------------------------------------------------------- /zstd/testdata/bad.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/bad.zip -------------------------------------------------------------------------------- /zstd/testdata/benchdecoder.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/benchdecoder.zip -------------------------------------------------------------------------------- /zstd/testdata/comp-crashers.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/comp-crashers.zip -------------------------------------------------------------------------------- /zstd/testdata/decode-regression.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/decode-regression.zip -------------------------------------------------------------------------------- /zstd/testdata/decoder.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/decoder.zip -------------------------------------------------------------------------------- /zstd/testdata/delta/source.txt: -------------------------------------------------------------------------------- 1 | 0000000000000000 2 | 3 | This file is to be used as the dictionary for compressing target.txt: 4 | 5 | zstd -19 --patch-from=source.txt target.txt 6 | -------------------------------------------------------------------------------- /zstd/testdata/delta/target.txt: -------------------------------------------------------------------------------- 1 | 0000000000000000 2 | 3 | This file is to be compressed with source.txt as the dictionary: 4 | 5 | zstd -19 --patch-from=source.txt target.txt 6 | -------------------------------------------------------------------------------- /zstd/testdata/delta/target.txt.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/delta/target.txt.zst -------------------------------------------------------------------------------- /zstd/testdata/dict-tests-small.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/dict-tests-small.zip -------------------------------------------------------------------------------- /zstd/testdata/fuzz/decode-corpus-encoded.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/fuzz/decode-corpus-encoded.zip -------------------------------------------------------------------------------- /zstd/testdata/fuzz/decode-corpus-raw.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/fuzz/decode-corpus-raw.zip -------------------------------------------------------------------------------- /zstd/testdata/fuzz/decode-oss.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/fuzz/decode-oss.zip -------------------------------------------------------------------------------- /zstd/testdata/fuzz/encode-corpus-encoded.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/fuzz/encode-corpus-encoded.zip -------------------------------------------------------------------------------- /zstd/testdata/fuzz/encode-corpus-raw.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/fuzz/encode-corpus-raw.zip -------------------------------------------------------------------------------- /zstd/testdata/good.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/good.zip -------------------------------------------------------------------------------- /zstd/testdata/headers-want.json.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/headers-want.json.zst -------------------------------------------------------------------------------- /zstd/testdata/headers.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/headers.zip -------------------------------------------------------------------------------- /zstd/testdata/large.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/large.zip -------------------------------------------------------------------------------- /zstd/testdata/regression.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/regression.zip -------------------------------------------------------------------------------- /zstd/testdata/seqs-want.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/seqs-want.zip -------------------------------------------------------------------------------- /zstd/testdata/seqs.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/seqs.zip -------------------------------------------------------------------------------- /zstd/testdata/xml.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/xml.zst -------------------------------------------------------------------------------- /zstd/testdata/z000028: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/z000028 -------------------------------------------------------------------------------- /zstd/testdata/z000028.zst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klauspost/compress/33f59b4f9a998ce6b7440b1885d0aff530fef36d/zstd/testdata/z000028.zst -------------------------------------------------------------------------------- /zstd/zip_test.go: -------------------------------------------------------------------------------- 1 | package zstd_test 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/klauspost/compress/zstd" 10 | ) 11 | 12 | func ExampleZipCompressor() { 13 | // Get zstandard de/compressors for zip. 14 | // These can be used by multiple readers and writers. 15 | compr := zstd.ZipCompressor(zstd.WithWindowSize(1<<20), zstd.WithEncoderCRC(false)) 16 | decomp := zstd.ZipDecompressor() 17 | 18 | // Try it out... 19 | var buf bytes.Buffer 20 | zw := zip.NewWriter(&buf) 21 | zw.RegisterCompressor(zstd.ZipMethodWinZip, compr) 22 | zw.RegisterCompressor(zstd.ZipMethodPKWare, compr) 23 | 24 | // Create 1MB data 25 | tmp := make([]byte, 1<<20) 26 | for i := range tmp { 27 | tmp[i] = byte(i) 28 | } 29 | w, err := zw.CreateHeader(&zip.FileHeader{ 30 | Name: "file1.txt", 31 | Method: zstd.ZipMethodWinZip, 32 | }) 33 | if err != nil { 34 | panic(err) 35 | } 36 | w.Write(tmp) 37 | 38 | // Another... 39 | w, err = zw.CreateHeader(&zip.FileHeader{ 40 | Name: "file2.txt", 41 | Method: zstd.ZipMethodPKWare, 42 | }) 43 | w.Write(tmp) 44 | zw.Close() 45 | 46 | zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len())) 47 | if err != nil { 48 | panic(err) 49 | } 50 | zr.RegisterDecompressor(zstd.ZipMethodWinZip, decomp) 51 | zr.RegisterDecompressor(zstd.ZipMethodPKWare, decomp) 52 | for _, file := range zr.File { 53 | rc, err := file.Open() 54 | if err != nil { 55 | panic(err) 56 | } 57 | b, err := io.ReadAll(rc) 58 | rc.Close() 59 | if bytes.Equal(b, tmp) { 60 | fmt.Println(file.Name, "ok") 61 | } else { 62 | fmt.Println(file.Name, "mismatch") 63 | } 64 | } 65 | // Output: 66 | // file1.txt ok 67 | // file2.txt ok 68 | } 69 | --------------------------------------------------------------------------------