├── .gitattributes
├── .github
├── dependabot.yml
└── workflows
│ ├── ci.yml
│ └── scorecard.yml
├── .gitignore
├── CHANGELOG
├── Doxyfile
├── Doxyfile-internal
├── LICENSE
├── Makefile
├── README.md
├── SECURITY.md
├── appveyor.yml
├── build
├── cmake
│ ├── .gitignore
│ ├── CMakeLists.txt
│ ├── JoinPaths.cmake
│ ├── README.md
│ └── xxHashConfig.cmake.in
└── make
│ ├── README.md
│ └── multiconf.make
├── cli
├── .tipi
│ ├── deps
│ └── opts
├── COPYING
├── README.md
├── xsum_arch.c
├── xsum_arch.h
├── xsum_bench.c
├── xsum_bench.h
├── xsum_config.h
├── xsum_os_specific.c
├── xsum_os_specific.h
├── xsum_output.c
├── xsum_output.h
├── xsum_sanity_check.c
├── xsum_sanity_check.h
├── xxhsum.1
├── xxhsum.1.md
└── xxhsum.c
├── clib.json
├── doc
├── README.md
├── xxhash.cry
└── xxhash_spec.md
├── fuzz
└── fuzzer.c
├── libxxhash.pc.in
├── tests
├── Makefile
├── bench
│ ├── .clang_complete
│ ├── .gitignore
│ ├── LICENSE
│ ├── Makefile
│ ├── benchHash.c
│ ├── benchHash.h
│ ├── benchfn.c
│ ├── benchfn.h
│ ├── bhDisplay.c
│ ├── bhDisplay.h
│ ├── hashes.h
│ ├── main.c
│ ├── timefn.c
│ └── timefn.h
├── cli-comment-line.sh
├── cli-ignore-missing.sh
├── collisions
│ ├── .gitignore
│ ├── LICENSE
│ ├── Makefile
│ ├── README.md
│ ├── allcodecs
│ │ ├── README.md
│ │ ├── dummy.c
│ │ └── dummy.h
│ ├── hashes.h
│ ├── main.c
│ ├── pool.c
│ ├── pool.h
│ ├── sort.cc
│ ├── sort.hh
│ ├── threading.c
│ └── threading.h
├── filename-escape.sh
├── generate_unicode_test.c
├── multiInclude.c
├── ppc_define.c
├── sanity_test.c
├── sanity_test_vectors.h
├── sanity_test_vectors_generator.c
├── test_alias.c
└── unicode_lint.sh
├── xxh3.h
├── xxh_x86dispatch.c
├── xxh_x86dispatch.h
├── xxhash.c
└── xxhash.h
/.gitattributes:
--------------------------------------------------------------------------------
1 | # Set the default behavior
2 | * text eol=lf
3 |
4 | # Explicitly declare source files
5 | *.c text eol=lf
6 | *.h text eol=lf
7 |
8 | # Denote files that should not be modified.
9 | *.odt binary
10 |
11 |
--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | # https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
2 | version: 2
3 | updates:
4 | - package-ecosystem: "github-actions"
5 | directory: "/"
6 | schedule:
7 | interval: "weekly"
8 |
--------------------------------------------------------------------------------
/.github/workflows/scorecard.yml:
--------------------------------------------------------------------------------
1 | # This workflow uses actions that are not certified by GitHub. They are provided
2 | # by a third-party and are governed by separate terms of service, privacy
3 | # policy, and support documentation.
4 |
5 | name: Scorecard supply-chain security
6 | on:
7 | # For Branch-Protection check. Only the default branch is supported. See
8 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
9 | branch_protection_rule:
10 | # To guarantee Maintained check is occasionally updated. See
11 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
12 | schedule:
13 | - cron: '35 19 * * 2'
14 | push:
15 | branches: [ "dev" ]
16 |
17 | # Declare default permissions as read only.
18 | permissions: read-all
19 |
20 | jobs:
21 | analysis:
22 | name: Scorecard analysis
23 | runs-on: ubuntu-latest
24 | permissions:
25 | # Needed to upload the results to code-scanning dashboard.
26 | security-events: write
27 | # Needed to publish results and get a badge (see publish_results below).
28 | id-token: write
29 | # Uncomment the permissions below if installing in a private repository.
30 | # contents: read
31 | # actions: read
32 |
33 | steps:
34 | - name: "Checkout code"
35 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
36 | with:
37 | persist-credentials: false
38 |
39 | - name: "Run analysis"
40 | uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2
41 | with:
42 | results_file: results.sarif
43 | results_format: sarif
44 | # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
45 | # - you want to enable the Branch-Protection check on a *public* repository, or
46 | # - you are installing Scorecard on a *private* repository
47 | # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
48 | # repo_token: ${{ secrets.SCORECARD_TOKEN }}
49 |
50 | # Public repositories:
51 | # - Publish results to OpenSSF REST API for easy access by consumers
52 | # - Allows the repository to include the Scorecard badge.
53 | # - See https://github.com/ossf/scorecard-action#publishing-results.
54 | # For private repositories:
55 | # - `publish_results` will always be set to `false`, regardless
56 | # of the value entered here.
57 | publish_results: true
58 |
59 | # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
60 | # format to the repository Actions tab.
61 | - name: "Upload artifact"
62 | uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
63 | with:
64 | name: SARIF file
65 | path: results.sarif
66 | retention-days: 5
67 |
68 | # Upload the results to GitHub's code scanning dashboard.
69 | - name: "Upload to code-scanning"
70 | uses: github/codeql-action/upload-sarif@c6c77c8c2d62cfd5b2e8d548817fd3d1582ac744 # v2.14.5
71 | with:
72 | sarif_file: results.sarif
73 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # objects
2 | *.o
3 | *.obj
4 | *.s
5 |
6 | # libraries
7 | libxxhash.*
8 | !libxxhash.pc.in
9 |
10 | # Executables
11 | *.exe
12 | xxh32sum
13 | xxh64sum
14 | xxh128sum
15 | xxh3sum
16 | xxhsum
17 | xxhsum32
18 | xxhsum_privateXXH
19 | xxhsum_inlinedXXH
20 | dispatch
21 | tests/generate_unicode_test
22 | tests/sanity_test
23 | tests/sanity_test_vectors_generator
24 | tests/test_alias
25 | fuzzer
26 |
27 | # Mac OS-X artefacts
28 | *.dSYM
29 | .DS_Store
30 |
31 | # Wasm / emcc / emscripten artefacts
32 | *.html
33 | *.wasm
34 | *.js
35 |
36 | # build artifacts
37 | cachedObjs/
38 | cmakebuild/
39 |
40 |
41 | # project managers artifacts
42 | .projectile
43 |
44 | # analyzer artifacts
45 | infer-out
46 |
47 | # test artifacts
48 | .test*
49 | tmp*
50 | tests/*.unicode
51 | tests/unicode_test*
52 | *.txt
53 | !CMakeLists.txt
54 | *.xxhsum
55 |
56 | # editor artifacts
57 | .clang_complete
58 | .clangd
59 | *.swp
60 | .vscode/
61 | .vs/
62 |
63 | # Doxygen
64 | doxygen/
65 |
--------------------------------------------------------------------------------
/CHANGELOG:
--------------------------------------------------------------------------------
1 | v0.8.3
2 | - fix : variant `XXH3_128bits_withSecretandSeed()` could produce an invalid result in some specific set of conditions, #894 by @hltj
3 | - cli : vector extension detected at runtime on x86/x64, enabled by default
4 | - cli : new commands `--filelist` and `--files-from`, by @Ian-Clowes
5 | - cli : XXH3 64-bits GNU format can now be generated and checked (command `-H3`)
6 | - portability: LoongArch SX SIMD extension, by @lrzlin
7 | - portability: can build on AIX, suggested by @likema
8 | - portability: validated for SPARC cpus
9 |
10 | v0.8.2
11 | - fix : XXH3 S390x vector implementation (@hzhuang1)
12 | - fix : PowerPC vector compilation with IBM XL compiler (@MaxiBoether)
13 | - perf : improved WASM speed by x2/x3 using SIMD128 (@easyaspi314)
14 | - perf : improved speed (+20%) for XXH3 on ARM NEON (@easyaspi314)
15 | - cli : Fix filename contain /LF character (@t-mat)
16 | - cli : Support # comment lines in --check files (@t-mat)
17 | - cli : Support commands --binary and --ignore-missing (@t-mat)
18 | - build: fix -Og compilation (@easyaspi314, @t-mat)
19 | - build: fix pkgconfig generation with cmake (@ilya-fedin)
20 | - build: fix icc compilation
21 | - build: fix cmake install directories
22 | - build: new build options XXH_NO_XXH3, XXH_SIZE_OPT and XXH_NO_STREAM to reduce binary size (@easyaspi314)
23 | - build: dedicated install targets (@ffontaine)
24 | - build: support DISPATCH mode in cmake (@hzhuang1)
25 | - portability: fix x86dispatch when building with Visual + clang-cl (@t-mat)
26 | - portability: SVE vector implementation of XXH3 (@hzhuang1)
27 | - portability: compatibility with freestanding environments, using XXH_NO_STDLIB
28 | - portability: can build on Haiku (@Begasus)
29 | - portability: validated on m68k and risc-v
30 | - doc : XXH3 specification (@Adrien1018)
31 | - doc : improved doxygen documentation (@easyaspi314, @t-mat)
32 | - misc : dedicated sanity test binary (@t-mat)
33 |
34 | v0.8.1
35 | - perf : much improved performance for XXH3 streaming variants, notably on gcc and msvc
36 | - perf : improved XXH64 speed and latency on small inputs
37 | - perf : small XXH32 speed and latency improvement on small inputs of random size
38 | - perf : minor stack usage improvement for XXH32 and XXH64
39 | - api : new experimental variants XXH3_*_withSecretandSeed()
40 | - api : update XXH3_generateSecret(), can no generate secret of any size (>= XXH3_SECRET_SIZE_MIN)
41 | - cli : xxhsum can now generate and check XXH3 checksums, using command `-H3`
42 | - build: can build xxhash without XXH3, with new build macro XXH_NO_XXH3
43 | - build: fix xxh_x86dispatch build with MSVC, by @apankrat
44 | - build: XXH_INLINE_ALL can always be used safely, even after XXH_NAMESPACE or a previous XXH_INLINE_ALL
45 | - build: improved PPC64LE vector support, by @mpe
46 | - install: fix pkgconfig, by @ellert
47 | - install: compatibility with Haiku, by @Begasus
48 | - doc : code comments made compatible with doxygen, by @easyaspi314
49 | - misc : XXH_ACCEPT_NULL_INPUT_POINTER is no longer necessary, all functions can accept NULL input pointers, as long as size == 0
50 | - misc : complete refactor of CI tests on Github Actions, offering much larger coverage, by @t-mat
51 | - misc : xxhsum code base split into multiple specialized units, within directory cli/, by @easyaspi314
52 |
53 | v0.8.0
54 | - api : stabilize XXH3
55 | - cli : xxhsum can parse BSD-style --check lines, by @WayneD
56 | - cli : `xxhsum -` accepts console input, requested by @jaki
57 | - cli : xxhsum accepts -- separator, by @jaki
58 | - cli : fix : print correct default algo for symlinked helpers, by @martinetd
59 | - install: improved pkgconfig script, allowing custom install locations, requested by @ellert
60 |
61 | v0.7.4
62 | - perf: automatic vector detection and selection at runtime (`xxh_x86dispatch.h`), initiated by @easyaspi314
63 | - perf: added AVX512 support, by @gzm55
64 | - api : new: secret generator `XXH_generateSecret()`, suggested by @koraa
65 | - api : fix: XXH3_state_t is movable, identified by @koraa
66 | - api : fix: state is correctly aligned in AVX mode (unlike `malloc()`), by @easyaspi314
67 | - api : fix: streaming generated wrong values in some combination of random ingestion lengths, reported by @WayneD
68 | - cli : fix unicode print on Windows, by @easyaspi314
69 | - cli : can `-c` check file generated by sfv
70 | - build: `make DISPATCH=1` generates `xxhsum` and `libxxhash` with runtime vector detection (x86/x64 only)
71 | - install: cygwin installation support
72 | - doc : Cryptol specification of XXH32 and XXH64, by @weaversa
73 |
74 | v0.7.3
75 | - perf: improved speed for large inputs (~+20%)
76 | - perf: improved latency for small inputs (~10%)
77 | - perf: s390x Vectorial code, by @easyaspi314
78 | - cli: improved support for Unicode filenames on Windows, thanks to @easyaspi314 and @t-mat
79 | - api: `xxhash.h` can now be included in any order, with and without `XXH_STATIC_LINKING_ONLY` and `XXH_INLINE_ALL`
80 | - build: xxHash's implementation transferred into `xxhash.h`. No more need to have `xxhash.c` in the `/include` directory for `XXH_INLINE_ALL` to work
81 | - install: created pkg-config file, by @bket
82 | - install: VCpkg installation instructions, by @LilyWangL
83 | - doc: Highly improved code documentation, by @easyaspi314
84 | - misc: New test tool in `/tests/collisions`: brute force collision tester for 64-bit hashes
85 |
86 | v0.7.2
87 | - Fixed collision ratio of `XXH128` for some specific input lengths, reported by @svpv
88 | - Improved `VSX` and `NEON` variants, by @easyaspi314
89 | - Improved performance of scalar code path (`XXH_VECTOR=0`), by @easyaspi314
90 | - `xxhsum`: can generate 128-bit hashes with the `-H2` option (note: for experimental purposes only! `XXH128` is not yet frozen)
91 | - `xxhsum`: option `-q` removes status notifications
92 |
93 | v0.7.1
94 | - Secret first: the algorithm computation can be altered by providing a "secret", which is any blob of bytes, of size >= `XXH3_SECRET_SIZE_MIN`.
95 | - `seed` is still available, and acts as a secret generator
96 | - updated `ARM NEON` variant by @easyaspi314
97 | - Streaming implementation is available
98 | - Improve compatibility and performance with Visual Studio, with help from @aras-p
99 | - Better integration when using `XXH_INLINE_ALL`: do not pollute host namespace, use its own macros, such as `XXH_ASSERT()`, `XXH_ALIGN`, etc.
100 | - 128-bit variant provides helper functions for comparison of hashes.
101 | - Better `clang` generation of `rotl` instruction, thanks to @easyaspi314
102 | - `XXH_REROLL` build macro to reduce binary size, by @easyaspi314
103 | - Improved `cmake` script, by @Mezozoysky
104 | - Full benchmark program provided in `/tests/bench`
105 |
--------------------------------------------------------------------------------
/Doxyfile:
--------------------------------------------------------------------------------
1 | # Doxygen config for xxHash
2 | DOXYFILE_ENCODING = UTF-8
3 |
4 | PROJECT_NAME = "xxHash"
5 | PROJECT_NUMBER = "0.8.3"
6 | PROJECT_BRIEF = "Extremely fast non-cryptographic hash function"
7 | OUTPUT_DIRECTORY = doxygen
8 | OUTPUT_LANGUAGE = English
9 |
10 | # We already separate the internal docs.
11 | INTERNAL_DOCS = NO
12 | # Consistency
13 | SORT_MEMBER_DOCS = NO
14 | BRIEF_MEMBER_DESC = YES
15 | REPEAT_BRIEF = YES
16 |
17 | # Warnings
18 | QUIET = YES
19 | # Until we document everything
20 | WARN_IF_UNDOCUMENTED = NO
21 |
22 | # TODO: Add the other files. It is just xxhash.h for now.
23 | FILE_PATTERNS = xxhash.h xxh_x86dispatch.c
24 | # Note: xxHash's source files are technically ASCII only.
25 | INPUT_ENCODING = UTF-8
26 | TAB_SIZE = 4
27 | MARKDOWN_SUPPORT = YES
28 |
29 | # xxHash is a C library
30 | OPTIMIZE_OUTPUT_FOR_C = YES
31 | # We hide private part from public document
32 | EXTRACT_STATIC = NO
33 | # We hide private part from public document
34 | EXTRACT_PRIVATE = NO
35 | # Document the macros
36 | MACRO_EXPANSION = YES
37 | EXPAND_ONLY_PREDEF = YES
38 | # Predefine some macros to clean up the output.
39 | PREDEFINED = "XXH_DOXYGEN=" \
40 | "XXH_PUBLIC_API=" \
41 | "XXH_NOESCAPE=" \
42 | "XXH_FORCE_INLINE=static inline" \
43 | "XXH_NO_INLINE=static" \
44 | "XXH_RESTRICT=restrict" \
45 | "XSUM_API=" \
46 | "XXH_STATIC_LINKING_ONLY" \
47 | "XXH_IMPLEMENTATION" \
48 | "XXH_PUREF=[[gnu::pure]]" \
49 | "XXH_CONSTF=[[gnu::const]]" \
50 | "XXH_MALLOCF=[[gnu::malloc]]" \
51 | "XXH_ALIGN(N)=alignas(N)" \
52 | "XXH_ALIGN_MEMBER(align,type)=alignas(align) type"
53 |
54 | # We want HTML docs
55 | GENERATE_HTML = YES
56 | HTML_OUTPUT = html
57 | HTML_FILE_EXTENSION = .html
58 | # Tweak the colors a bit
59 | HTML_COLORSTYLE_HUE = 220
60 | HTML_COLORSTYLE_GAMMA = 100
61 | HTML_COLORSTYLE_SAT = 100
62 |
63 | # We don't want LaTeX.
64 | GENERATE_LATEX = NO
65 |
--------------------------------------------------------------------------------
/Doxyfile-internal:
--------------------------------------------------------------------------------
1 | # Doxygen config for xxHash
2 | DOXYFILE_ENCODING = UTF-8
3 |
4 | PROJECT_NAME = "xxHash"
5 | PROJECT_NUMBER = "0.8.2"
6 | PROJECT_BRIEF = "Extremely fast non-cryptographic hash function"
7 | OUTPUT_DIRECTORY = doxygen
8 | OUTPUT_LANGUAGE = English
9 |
10 | # We already separate the internal docs.
11 | INTERNAL_DOCS = YES
12 | # Consistency
13 | SORT_MEMBER_DOCS = NO
14 | BRIEF_MEMBER_DESC = YES
15 | REPEAT_BRIEF = YES
16 |
17 | # Warnings
18 | QUIET = YES
19 | # Until we document everything
20 | WARN_IF_UNDOCUMENTED = NO
21 |
22 | # TODO: Add the other files. It is just xxhash.h for now.
23 | FILE_PATTERNS = xxhash.h xxh_x86dispatch.c
24 | # Note: xxHash's source files are technically ASCII only.
25 | INPUT_ENCODING = UTF-8
26 | TAB_SIZE = 4
27 | MARKDOWN_SUPPORT = YES
28 |
29 | # xxHash is a C library
30 | OPTIMIZE_OUTPUT_FOR_C = YES
31 | # So we can document the internals
32 | EXTRACT_STATIC = YES
33 | # We show private part in the internal document
34 | EXTRACT_PRIVATE = YES
35 | # Document the macros
36 | MACRO_EXPANSION = YES
37 | EXPAND_ONLY_PREDEF = YES
38 | # Predefine some macros to clean up the output.
39 | PREDEFINED = "XXH_DOXYGEN=" \
40 | "XXH_PUBLIC_API=" \
41 | "XXH_NOESCAPE=" \
42 | "XXH_FORCE_INLINE=static inline" \
43 | "XXH_NO_INLINE=static" \
44 | "XXH_RESTRICT=restrict" \
45 | "XSUM_API=" \
46 | "XXH_STATIC_LINKING_ONLY" \
47 | "XXH_IMPLEMENTATION" \
48 | "XXH_PUREF=[[gnu::pure]]" \
49 | "XXH_CONSTF=[[gnu::const]]" \
50 | "XXH_MALLOCF=[[gnu::malloc]]" \
51 | "XXH_ALIGN(N)=alignas(N)" \
52 | "XXH_ALIGN_MEMBER(align,type)=alignas(align) type"
53 |
54 | # We want HTML docs
55 | GENERATE_HTML = YES
56 | HTML_OUTPUT = html
57 | HTML_FILE_EXTENSION = .html
58 | # Tweak the colors a bit
59 | HTML_COLORSTYLE_HUE = 220
60 | HTML_COLORSTYLE_GAMMA = 100
61 | HTML_COLORSTYLE_SAT = 100
62 |
63 | # We don't want LaTeX.
64 | GENERATE_LATEX = NO
65 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | xxHash Library
2 | Copyright (c) 2012-2021 Yann Collet
3 | All rights reserved.
4 |
5 | BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
6 |
7 | Redistribution and use in source and binary forms, with or without modification,
8 | are permitted provided that the following conditions are met:
9 |
10 | * Redistributions of source code must retain the above copyright notice, this
11 | list of conditions and the following disclaimer.
12 |
13 | * Redistributions in binary form must reproduce the above copyright notice, this
14 | list of conditions and the following disclaimer in the documentation and/or
15 | other materials provided with the distribution.
16 |
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
20 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
21 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
22 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
23 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
24 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
26 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 |
--------------------------------------------------------------------------------
/SECURITY.md:
--------------------------------------------------------------------------------
1 | # Security Policy
2 |
3 | ## Supported Versions
4 |
5 | Security updates are applied only to the latest release.
6 |
7 | ## Reporting a Vulnerability
8 |
9 | If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released.
10 |
11 | Please disclose it at [security advisory](https://github.com/Cyan4973/xxHash/security/advisories/new).
12 |
13 | This project is maintained by a team of volunteers on a reasonable-effort basis. As such, please give us at least 90 days to work on a fix before public exposure.
14 |
--------------------------------------------------------------------------------
/appveyor.yml:
--------------------------------------------------------------------------------
1 | # ==============================================================================
2 | # AppVeyor CI Configuration
3 | # ==============================================================================
4 |
5 | version: 1.0.{build}
6 | max_jobs: 2
7 | clone_depth: 2
8 |
9 | # ==============================================================================
10 | # Build Matrix
11 | # ==============================================================================
12 |
13 | environment:
14 | matrix:
15 | # Legacy Visual Studio 2013 build (not supported by GitHub Actions)
16 | - COMPILER: "visual"
17 | ARCHITECTURES: "Win32,x64"
18 | AVX2_ENABLED: "false"
19 | APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2013"
20 | TEST_XXHSUM: "true"
21 |
22 | # Legacy Visual Studio 2015 build (not supported by GitHub Actions)
23 | - COMPILER: "visual"
24 | ARCHITECTURES: "Win32,x64"
25 | AVX2_ENABLED: "true"
26 | APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2015"
27 | TEST_XXHSUM: "true"
28 |
29 | # Visual Studio 2017 build (not supported by GitHub Actions) - both Win32 and x64
30 | - COMPILER: "visual"
31 | ARCHITECTURES: "Win32,x64"
32 | AVX2_ENABLED: "true"
33 | APPVEYOR_BUILD_WORKER_IMAGE: "Visual Studio 2017"
34 | TEST_XXHSUM: "true"
35 |
36 | # ARM build (no testing - cannot run on x86/x64 CI)
37 | - COMPILER: "visual"
38 | ARCHITECTURES: "ARM"
39 | AVX2_ENABLED: "false"
40 | TEST_XXHSUM: "false"
41 |
42 | # ==============================================================================
43 | # Build Scripts
44 | # ==============================================================================
45 |
46 | build_script:
47 | - echo "========================================"
48 | - echo "Building Visual Studio configurations"
49 | - echo "========================================"
50 |
51 | # Visual Studio build process
52 | - ps: |
53 | Set-Location "build/cmake"
54 |
55 | # Split architectures and build each one
56 | $architectures = $env:ARCHITECTURES -split ","
57 |
58 | foreach ($arch in $architectures) {
59 | $arch = $arch.Trim()
60 | Write-Host "========================================"
61 | Write-Host "Building for architecture: $arch"
62 | Write-Host "========================================"
63 |
64 | # Create architecture-specific build directory
65 | $buildDir = "build_$arch"
66 | New-Item -Path $buildDir -ItemType Directory -Force | Out-Null
67 | Set-Location $buildDir
68 |
69 | # Configure CMake for this architecture with fast Debug builds
70 | cmake .. -A $arch -DXXHASH_C_FLAGS="/W4 /WX" -DCMAKE_C_FLAGS_DEBUG="/Od /Zi /MDd"
71 |
72 | # Build Debug configuration (fast compilation, no optimizations)
73 | Write-Host "Building Debug configuration for $arch..."
74 | cmake --build . --config Debug
75 |
76 | # Build Debug with SSE2 if Win32 (x64 has SSE2 by default, ARM doesn't support it)
77 | if ($arch -eq "Win32") {
78 | Write-Host "Building Debug configuration with SSE2 for $arch..."
79 | cmake .. -A $arch -DXXHASH_C_FLAGS="/W4 /WX /arch:SSE2" -DCMAKE_C_FLAGS_DEBUG="/Od /Zi /MDd"
80 | cmake --build . --config Debug
81 | }
82 |
83 | # Build Debug with AVX2 if enabled and supported
84 | if ($env:AVX2_ENABLED -eq "true" -and $arch -ne "ARM") {
85 | Write-Host "Building Debug configuration with AVX2 for $arch..."
86 | cmake .. -A $arch -DXXHASH_C_FLAGS="/W4 /WX /arch:AVX2" -DCMAKE_C_FLAGS_DEBUG="/Od /Zi /MDd"
87 | cmake --build . --config Debug
88 | }
89 |
90 | # Build Release configuration
91 | Write-Host "Building Release configuration for $arch..."
92 | cmake --build . --config Release
93 |
94 | # Go back to cmake directory for next architecture
95 | Set-Location ..
96 | }
97 |
98 | # ==============================================================================
99 | # Test Scripts
100 | # ==============================================================================
101 |
102 | test_script:
103 | # Test xxhsum Release binaries (only for x86/x64 architectures that can run on CI)
104 | - ps: |
105 | if ($env:TEST_XXHSUM -eq "true") {
106 | Set-Location "build/cmake"
107 |
108 | # Split architectures and test each one
109 | $architectures = $env:ARCHITECTURES -split ","
110 |
111 | foreach ($arch in $architectures) {
112 | $arch = $arch.Trim()
113 |
114 | # Only test architectures that can run on CI (x86/x64)
115 | if ($arch -eq "ARM") {
116 | Write-Host "Skipping testing for $arch (cannot run on x86/x64 CI)"
117 | continue
118 | }
119 |
120 | Write-Host "========================================"
121 | Write-Host "Testing Visual Studio $arch"
122 | Write-Host "========================================"
123 |
124 | Set-Location "build_$arch"
125 |
126 | # Test Release configuration only
127 | Write-Host "Testing Release binary for $arch..."
128 | Set-Location "Release"
129 | cmd /c "xxhsum.exe -b99i0"
130 | if ($LASTEXITCODE -ne 0) {
131 | Write-Host "Release xxhsum.exe failed for $arch with exit code $LASTEXITCODE"
132 | exit 1
133 | }
134 | Write-Host "Release binary test completed successfully for $arch"
135 |
136 | # Go back to cmake directory for next architecture
137 | Set-Location "../.."
138 | }
139 |
140 | Write-Host "------- All xxhsum testing completed successfully -------"
141 | }
142 |
143 | # ==============================================================================
144 | # Artifacts
145 | # ==============================================================================
146 | # Currently no artifacts are collected
147 | # Future: Consider collecting built binaries for distribution
148 |
149 | # ==============================================================================
150 | # Build Notes
151 | # ==============================================================================
152 | # 1. ARM/ARM64 builds cannot be tested on x86/x64 CI infrastructure
153 | # 2. AppVeyor focuses on legacy Visual Studio versions (2013, 2015, 2017)
154 | # 3. Modern Visual Studio versions (2019, 2022) are tested on GitHub Actions
155 |
--------------------------------------------------------------------------------
/build/cmake/.gitignore:
--------------------------------------------------------------------------------
1 | # cmake artifacts
2 |
3 | CMakeCache.txt
4 | CMakeFiles
5 | Makefile
6 | cmake_install.cmake
7 |
8 | build
9 |
10 |
11 | # make compilation results
12 |
13 | *.dylib
14 | *.a
15 |
--------------------------------------------------------------------------------
/build/cmake/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | # To the extent possible under law, the author(s) have dedicated all
2 | # copyright and related and neighboring rights to this software to
3 | # the public domain worldwide. This software is distributed without
4 | # any warranty.
5 | #
6 | # For details, see <https://creativecommons.org/publicdomain/zero/1.0/>.
7 |
8 | cmake_minimum_required (VERSION 3.10 FATAL_ERROR)
9 |
10 | set(XXHASH_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../..")
11 |
12 | file(STRINGS "${XXHASH_DIR}/xxhash.h" XXHASH_VERSION_MAJOR REGEX "^#define XXH_VERSION_MAJOR +([0-9]+) *quot;)
13 | string(REGEX REPLACE "^#define XXH_VERSION_MAJOR +([0-9]+) *quot; "\\1" XXHASH_VERSION_MAJOR "${XXHASH_VERSION_MAJOR}")
14 | file(STRINGS "${XXHASH_DIR}/xxhash.h" XXHASH_VERSION_MINOR REGEX "^#define XXH_VERSION_MINOR +([0-9]+) *quot;)
15 | string(REGEX REPLACE "^#define XXH_VERSION_MINOR +([0-9]+) *quot; "\\1" XXHASH_VERSION_MINOR "${XXHASH_VERSION_MINOR}")
16 | file(STRINGS "${XXHASH_DIR}/xxhash.h" XXHASH_VERSION_RELEASE REGEX "^#define XXH_VERSION_RELEASE +([0-9]+) *quot;)
17 | string(REGEX REPLACE "^#define XXH_VERSION_RELEASE +([0-9]+) *quot; "\\1" XXHASH_VERSION_RELEASE "${XXHASH_VERSION_RELEASE}")
18 | set(XXHASH_VERSION_STRING "${XXHASH_VERSION_MAJOR}.${XXHASH_VERSION_MINOR}.${XXHASH_VERSION_RELEASE}")
19 | set(XXHASH_LIB_VERSION ${XXHASH_VERSION_STRING})
20 | set(XXHASH_LIB_SOVERSION "${XXHASH_VERSION_MAJOR}")
21 | mark_as_advanced(XXHASH_VERSION_MAJOR XXHASH_VERSION_MINOR XXHASH_VERSION_RELEASE XXHASH_VERSION_STRING XXHASH_LIB_VERSION XXHASH_LIB_SOVERSION)
22 |
23 | if("${CMAKE_VERSION}" VERSION_LESS "3.13")
24 | #message(WARNING "CMake ${CMAKE_VERSION} has no CMP0077 policy: options will erase uncached/untyped normal vars!")
25 | else()
26 | cmake_policy (SET CMP0077 NEW)
27 | endif()
28 | cmake_policy (SET CMP0048 NEW)
29 | project(xxHash
30 | VERSION ${XXHASH_VERSION_STRING}
31 | LANGUAGES C)
32 |
33 | if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
34 | set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Project build type" FORCE)
35 | set_property(CACHE CMAKE_BUILD_TYPE
36 | PROPERTY STRINGS "Debug" "Release" "RelWithDebInfo" "MinSizeRel")
37 | endif()
38 | if(NOT CMAKE_CONFIGURATION_TYPES)
39 | message(STATUS "xxHash build type: ${CMAKE_BUILD_TYPE}")
40 | endif()
41 |
42 | # Enable assert() statements in debug builds
43 | if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
44 | if("${CMAKE_VERSION}" VERSION_LESS "3.12")
45 | # add_compile_definitions is not available for older cmake => do nothing
46 | else()
47 | add_compile_definitions(XXH_DEBUGLEVEL=1)
48 | endif()
49 | endif()
50 |
51 | option(BUILD_SHARED_LIBS "Build shared library" ON)
52 | option(XXHASH_BUILD_XXHSUM "Build the xxhsum binary" ON)
53 |
54 | # If XXHASH is being bundled in another project, we don't want to
55 | # install anything. However, we want to let people override this, so
56 | # we'll use the XXHASH_BUNDLED_MODE variable to let them do that; just
57 | # set it to OFF in your project before you add_subdirectory(xxhash/cmake_unofficial).
58 | if(NOT DEFINED XXHASH_BUNDLED_MODE)
59 | if("${PROJECT_SOURCE_DIR}" STREQUAL "${CMAKE_SOURCE_DIR}")
60 | set(XXHASH_BUNDLED_MODE OFF)
61 | else()
62 | set(XXHASH_BUNDLED_MODE ON)
63 | endif()
64 | endif()
65 | set(XXHASH_BUNDLED_MODE ${XXHASH_BUNDLED_MODE} CACHE BOOL "" FORCE)
66 | mark_as_advanced(XXHASH_BUNDLED_MODE)
67 |
68 | # Allow people to choose whether to build shared or static libraries
69 | # via the BUILD_SHARED_LIBS option unless we are in bundled mode, in
70 | # which case we always use static libraries.
71 | include(CMakeDependentOption)
72 | CMAKE_DEPENDENT_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON "NOT XXHASH_BUNDLED_MODE" OFF)
73 |
74 | # detect architecture for DISPATCH mode
75 | CMAKE_HOST_SYSTEM_INFORMATION(RESULT PLATFORM QUERY OS_PLATFORM)
76 | message(STATUS "Architecture: ${PLATFORM}")
77 |
78 | # libxxhash
79 | if((DEFINED DISPATCH) AND (DEFINED PLATFORM))
80 | # Only support DISPATCH option on x86_64.
81 | if(("${PLATFORM}" STREQUAL "x86_64") OR ("${PLATFORM}" STREQUAL "AMD64"))
82 | set(XXHSUM_DISPATCH ON)
83 | message(STATUS "Enable xxHash dispatch mode")
84 | add_library(xxhash "${XXHASH_DIR}/xxh_x86dispatch.c"
85 | "${XXHASH_DIR}/xxhash.c"
86 | )
87 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DXXHSUM_DISPATCH=1")
88 | else()
89 | add_library(xxhash "${XXHASH_DIR}/xxhash.c")
90 | endif()
91 | else()
92 | add_library(xxhash "${XXHASH_DIR}/xxhash.c")
93 | endif()
94 | add_library(${PROJECT_NAME}::xxhash ALIAS xxhash)
95 |
96 | target_include_directories(xxhash
97 | PUBLIC
98 | lt;BUILD_INTERFACE:${XXHASH_DIR}>
99 | lt;INSTALL_INTERFACE:include/>)
100 | if (BUILD_SHARED_LIBS)
101 | target_compile_definitions(xxhash PUBLIC XXH_EXPORT)
102 | endif ()
103 | set_target_properties(xxhash PROPERTIES
104 | SOVERSION "${XXHASH_LIB_SOVERSION}"
105 | VERSION "${XXHASH_VERSION_STRING}")
106 |
107 | if(XXHASH_BUILD_XXHSUM)
108 | set(XXHSUM_DIR "${XXHASH_DIR}/cli")
109 | # xxhsum
110 | set(XXHSUM_SOURCES)
111 | if (XXHSUM_DISPATCH)
112 | list(APPEND XXHSUM_SOURCES "${XXHASH_DIR}/xxh_x86dispatch.c")
113 | endif()
114 | list(APPEND XXHSUM_SOURCES "${XXHSUM_DIR}/xxhsum.c"
115 | "${XXHSUM_DIR}/xsum_arch.c"
116 | "${XXHSUM_DIR}/xsum_os_specific.c"
117 | "${XXHSUM_DIR}/xsum_output.c"
118 | "${XXHSUM_DIR}/xsum_sanity_check.c"
119 | "${XXHSUM_DIR}/xsum_bench.c"
120 | )
121 | add_executable(xxhsum ${XXHSUM_SOURCES})
122 | add_executable(${PROJECT_NAME}::xxhsum ALIAS xxhsum)
123 |
124 | target_link_libraries(xxhsum PRIVATE xxhash)
125 | target_include_directories(xxhsum PRIVATE "${XXHASH_DIR}")
126 | endif(XXHASH_BUILD_XXHSUM)
127 |
128 | # Extra warning flags
129 | include (CheckCCompilerFlag)
130 | if (XXHASH_C_FLAGS)
131 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${XXHASH_C_FLAGS}")
132 | endif()
133 |
134 | if(NOT XXHASH_BUNDLED_MODE)
135 | include(GNUInstallDirs)
136 |
137 | install(TARGETS xxhash
138 | EXPORT xxHashTargets
139 | RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
140 | LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
141 | ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}")
142 | install(FILES "${XXHASH_DIR}/xxhash.h"
143 | DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
144 | install(FILES "${XXHASH_DIR}/xxh3.h"
145 | DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
146 | if(DISPATCH)
147 | install(FILES "${XXHASH_DIR}/xxh_x86dispatch.h"
148 | DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
149 | endif()
150 | if(XXHASH_BUILD_XXHSUM)
151 | install(TARGETS xxhsum
152 | EXPORT xxHashTargets
153 | RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
154 | install(FILES "${XXHSUM_DIR}/xxhsum.1"
155 | DESTINATION "${CMAKE_INSTALL_MANDIR}/man1")
156 | endif(XXHASH_BUILD_XXHSUM)
157 |
158 | include(CMakePackageConfigHelpers)
159 |
160 | set(xxHash_VERSION_CONFIG "${PROJECT_BINARY_DIR}/xxHashConfigVersion.cmake")
161 | set(xxHash_PROJECT_CONFIG "${PROJECT_BINARY_DIR}/xxHashConfig.cmake")
162 | set(xxHash_TARGETS_CONFIG "${PROJECT_BINARY_DIR}/xxHashTargets.cmake")
163 | set(xxHash_CONFIG_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/xxHash/")
164 | write_basic_package_version_file(${xxHash_VERSION_CONFIG}
165 | VERSION ${XXHASH_VERSION_STRING}
166 | COMPATIBILITY AnyNewerVersion)
167 | configure_package_config_file(
168 | ${PROJECT_SOURCE_DIR}/xxHashConfig.cmake.in
169 | ${xxHash_PROJECT_CONFIG}
170 | INSTALL_DESTINATION ${xxHash_CONFIG_INSTALL_DIR})
171 | export(EXPORT xxHashTargets
172 | FILE ${xxHash_TARGETS_CONFIG}
173 | NAMESPACE ${PROJECT_NAME}::)
174 |
175 | install(FILES ${xxHash_PROJECT_CONFIG} ${xxHash_VERSION_CONFIG}
176 | DESTINATION ${xxHash_CONFIG_INSTALL_DIR})
177 | install(EXPORT xxHashTargets
178 | DESTINATION ${xxHash_CONFIG_INSTALL_DIR}
179 | NAMESPACE ${PROJECT_NAME}::)
180 |
181 | # configure and install pkg-config
182 | include(JoinPaths.cmake)
183 | set(PREFIX ${CMAKE_INSTALL_PREFIX})
184 | set(EXECPREFIX "\${prefix}")
185 | join_paths(INCLUDEDIR "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
186 | join_paths(LIBDIR "\${prefix}" "${CMAKE_INSTALL_LIBDIR}")
187 | set(VERSION "${XXHASH_VERSION_STRING}")
188 | configure_file(${XXHASH_DIR}/libxxhash.pc.in ${CMAKE_BINARY_DIR}/libxxhash.pc @ONLY)
189 |
190 | install(FILES ${CMAKE_BINARY_DIR}/libxxhash.pc
191 | DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
192 |
193 | endif(NOT XXHASH_BUNDLED_MODE)
194 |
195 | include(CPack)
196 |
--------------------------------------------------------------------------------
/build/cmake/JoinPaths.cmake:
--------------------------------------------------------------------------------
1 | # This module provides function for joining paths
2 | # known from most languages
3 | #
4 | # SPDX-License-Identifier: (MIT OR CC0-1.0)
5 | # Copyright 2020 Jan Tojnar
6 | # https://github.com/jtojnar/cmake-snips
7 | #
8 | # Modelled after Python’s os.path.join
9 | # https://docs.python.org/3.7/library/os.path.html#os.path.join
10 | # Windows not supported
11 | function(join_paths joined_path first_path_segment)
12 | set(temp_path "${first_path_segment}")
13 | foreach(current_segment IN LISTS ARGN)
14 | if(NOT ("${current_segment}" STREQUAL ""))
15 | if(IS_ABSOLUTE "${current_segment}")
16 | set(temp_path "${current_segment}")
17 | else()
18 | set(temp_path "${temp_path}/${current_segment}")
19 | endif()
20 | endif()
21 | endforeach()
22 | set(${joined_path} "${temp_path}" PARENT_SCOPE)
23 | endfunction()
24 |
--------------------------------------------------------------------------------
/build/cmake/README.md:
--------------------------------------------------------------------------------
1 |
2 | # xxHash CMake Integration
3 |
4 | This document explains how to integrate xxHash into your CMake project. Choose the method that best fits your needs.
5 |
6 | ## Method 1: Install and Import (Recommended)
7 |
8 | **Best for:** Projects that want to use xxHash as a system-wide library.
9 |
10 | ### Step 1: Build and Install xxHash
11 |
12 | ```bash
13 | cd /path/to/xxHash
14 | cmake -S build/cmake -B cmake_build
15 | cmake --build cmake_build --parallel
16 | cmake --install cmake_build
17 | ```
18 |
19 | ### Step 2: Use in Your Project
20 |
21 | Add to your `CMakeLists.txt`:
22 |
23 | ```cmake
24 | find_package(xxHash 0.8 CONFIG REQUIRED)
25 | target_link_libraries(YourTarget PRIVATE xxHash::xxhash)
26 | ```
27 |
28 | ### Build Options
29 |
30 | Configure the build with these options:
31 |
32 | - `-DXXHASH_BUILD_XXHSUM=OFF` - Skip building the command line tool (default: ON)
33 | - `-DBUILD_SHARED_LIBS=OFF` - Build static library instead of shared (default: ON)
34 | - `-DCMAKE_INSTALL_PREFIX=/custom/path` - Install to custom location
35 | - `-DDISPATCH=OFF` - Disable CPU dispatch optimization (default: ON for x64)
36 |
37 | ## Method 2: Add as Subdirectory
38 |
39 | **Best for:** Projects that want to bundle xxHash directly without system installation.
40 |
41 | Add to your `CMakeLists.txt`:
42 |
43 | ```cmake
44 | # Optional: Configure xxHash before adding
45 | set(XXHASH_BUILD_XXHSUM OFF) # Don't build command line tool
46 | option(BUILD_SHARED_LIBS OFF) # Build static library
47 |
48 | # Add xxHash to your project
49 | add_subdirectory(path/to/xxHash/build/cmake xxhash_build EXCLUDE_FROM_ALL)
50 |
51 | # Link to your target
52 | target_link_libraries(YourTarget PRIVATE xxHash::xxhash)
53 | ```
54 |
55 |
--------------------------------------------------------------------------------
/build/cmake/xxHashConfig.cmake.in:
--------------------------------------------------------------------------------
1 | @PACKAGE_INIT@
2 |
3 | include(${CMAKE_CURRENT_LIST_DIR}/xxHashTargets.cmake)
4 |
5 |
--------------------------------------------------------------------------------
/build/make/README.md:
--------------------------------------------------------------------------------
1 | # multiconf.make
2 |
3 | **multiconf.make** is a self-contained Makefile include that lets you build the **same targets under many different flag sets**. For example debug vs release, ASan vs UBSan, GCC vs Clang.
4 | Each different set of flags generates object files into its own **dedicated cache directory**, so objects compiled with one configuration are never reused by another.
5 | Object files from previous configurations are preserved, so swapping back to a previous configuration only requires compiling objects which have actually changed.
6 |
7 | ---
8 |
9 | ## Benefits at a glance
10 |
11 | | Benefit | What `multiconf.make` does |
12 | | --- | --- |
13 | | **Isolated configs** | Stores objects into `cachedObjs/<hash>/`, one directory per unique flag set. |
14 | | **Fast switching** | Reusing an old config is instant—link only, no recompilation. |
15 | | **Header deps** | Edits to headers trigger only needed rebuilds. |
16 | | **One-liner targets** | Macros (`c_program`, `cxx_program`, …) hide all rule boilerplate. |
17 | | **Parallel-ready** | Safe with `make -j`, no duplicate compiles of shared sources. |
18 | | **Controlled verbosity** | Default only lists objects, while `V=1` display full commands. |
19 | | **`clean` included** | `make clean` deletes all objects, binaries and links. |
20 |
21 | ---
22 |
23 | ## Quick Start
24 |
25 | ### 1 · List your sources
26 |
27 | ```make
28 | C_SRCDIRS := src src/cdeps # all .c are in these directories
29 | CXX_SRCDIRS := src src/cxxdeps # all .cpp are in these directories
30 | ```
31 |
32 | ### 2 · Add and include
33 |
34 | ```make
35 | # root/Makefile
36 | include multiconf.make
37 | ```
38 |
39 | ### 3 · Declare targets
40 |
41 | ```make
42 | app:
43 | $(eval $(call c_program,app,app.o cdeps/obj.o))
44 |
45 | test:
46 | $(eval $(call cxx_program,test, test.o cxxdeps/objcxx.o))
47 |
48 | lib.a:
49 | $(eval $(call static_library,lib.a, lib.o cdeps/obj.o))
50 |
51 | lib.so:
52 | $(eval $(call c_dynamic_library,lib.so, lib.o cdeps/obj.o))
53 | ```
54 |
55 | ### 4 · Build any config you like
56 |
57 | ```sh
58 | # Release with GCC
59 | make CFLAGS="-O3"
60 |
61 | # Debug with Clang + AddressSanitizer (new cache dir)
62 | make CC=clang CFLAGS="-g -O0 -fsanitize=address"
63 |
64 | # Switch back to GCC release (objects still valid, relink only)
65 | make CFLAGS="-O3"
66 | ```
67 |
68 | Objects for each command live in different sub-folders; nothing overlaps.
69 |
70 | ---
71 |
72 |
--------------------------------------------------------------------------------
/cli/.tipi/deps:
--------------------------------------------------------------------------------
1 | {
2 | "Cyan4973/xxHash": { }
3 | }
--------------------------------------------------------------------------------
/cli/.tipi/opts:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Cyan4973/xxHash/38d555879fddae7300ece2c0820b3332c1d5748f/cli/.tipi/opts
--------------------------------------------------------------------------------
/cli/README.md:
--------------------------------------------------------------------------------
1 | This directory contains source code dedicated to the `xxhsum` command line utility,
2 | which is a user program of `libxxhash`.
3 |
4 | Note that, in contrast with the library `libxxhash`, the command line utility `xxhsum` ships with GPLv2 license.
5 |
--------------------------------------------------------------------------------
/cli/xsum_arch.c:
--------------------------------------------------------------------------------
1 | /*
2 | * xxhsum - Command line interface for xxhash algorithms
3 | * Copyright (C) 2013-2024 Yann Collet
4 | *
5 | * GPL v2 License
6 | *
7 | * This program is free software; you can redistribute it and/or modify
8 | * it under the terms of the GNU General Public License as published by
9 | * the Free Software Foundation; either version 2 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * This program is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU General Public License along
18 | * with this program; if not, write to the Free Software Foundation, Inc.,
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | *
21 | * You can contact the author at:
22 | * - xxHash homepage: https://www.xxhash.com
23 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | */
25 |
26 | int g_xsumarch_avoid_empty_unit = 0;
27 |
28 | #if ((defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC)) || defined(__i386__) || defined(_M_IX86) || defined(_M_IX86_FP)
29 | #if defined(XXHSUM_DISPATCH)
30 |
31 | #include "../xxh_x86dispatch.h"
32 |
33 | const char* XSUM_autox86(void)
34 | {
35 | int vecVersion = XXH_featureTest();
36 | switch(vecVersion) {
37 | case XXH_SCALAR:
38 | return "x86 autoVec (scalar: no vector extension detected)";
39 | case XXH_SSE2:
40 | return "x86 autoVec (SSE2 detected)";
41 | case XXH_AVX2:
42 | return "x86 autoVec (AVX2 detected)";
43 | case XXH_AVX512:
44 | return "x86 autoVec (AVX512 detected)";
45 | default:;
46 | }
47 | return " autoVec (error detecting vector extension)";
48 | }
49 |
50 | #endif /* XXHSUM_DISPATCH */
51 | #endif /* x86 */
52 |
--------------------------------------------------------------------------------
/cli/xsum_arch.h:
--------------------------------------------------------------------------------
1 | /*
2 | * xxhsum - Command line interface for xxhash algorithms
3 | * Copyright (C) 2013-2021 Yann Collet
4 | *
5 | * GPL v2 License
6 | *
7 | * This program is free software; you can redistribute it and/or modify
8 | * it under the terms of the GNU General Public License as published by
9 | * the Free Software Foundation; either version 2 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * This program is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU General Public License along
18 | * with this program; if not, write to the Free Software Foundation, Inc.,
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | *
21 | * You can contact the author at:
22 | * - xxHash homepage: https://www.xxhash.com
23 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | */
25 |
26 | /*
27 | * Checks for predefined macros by the compiler to try and get both the arch
28 | * and the compiler version.
29 | */
30 | #ifndef XSUM_ARCH_H
31 | #define XSUM_ARCH_H
32 |
33 | #include "xsum_config.h"
34 |
35 | #define XSUM_LIB_VERSION XXH_VERSION_MAJOR.XXH_VERSION_MINOR.XXH_VERSION_RELEASE
36 | #define XSUM_QUOTE(str) #str
37 | #define XSUM_EXPAND_AND_QUOTE(str) XSUM_QUOTE(str)
38 | #define XSUM_PROGRAM_VERSION XSUM_EXPAND_AND_QUOTE(XSUM_LIB_VERSION)
39 |
40 |
41 | /* Show compiler versions in WELCOME_MESSAGE. XSUM_CC_VERSION_FMT will return the printf specifiers,
42 | * and VERSION will contain the comma separated list of arguments to the XSUM_CC_VERSION_FMT string. */
43 | #if defined(__clang_version__)
44 | /* Clang does its own thing. */
45 | # ifdef __apple_build_version__
46 | # define XSUM_CC_VERSION_FMT "Apple Clang %s"
47 | # else
48 | # define XSUM_CC_VERSION_FMT "Clang %s"
49 | # endif
50 | # define XSUM_CC_VERSION __clang_version__
51 | #elif defined(__VERSION__)
52 | /* GCC and ICC */
53 | # define XSUM_CC_VERSION_FMT "%s"
54 | # ifdef __INTEL_COMPILER /* icc adds its prefix */
55 | # define XSUM_CC_VERSION __VERSION__
56 | # else /* assume GCC */
57 | # define XSUM_CC_VERSION "GCC " __VERSION__
58 | # endif
59 | #elif defined(_MSC_FULL_VER) && defined(_MSC_BUILD)
60 | /*
61 | * MSVC
62 | * "For example, if the version number of the Visual C++ compiler is
63 | * 15.00.20706.01, the _MSC_FULL_VER macro evaluates to 150020706."
64 | *
65 | * https://docs.microsoft.com/en-us/cpp/preprocessor/predefined-macros?view=vs-2017
66 | */
67 | # define XSUM_CC_VERSION_FMT "MSVC %02i.%02i.%05i.%02i"
68 | # define XSUM_CC_VERSION _MSC_FULL_VER / 10000000 % 100, _MSC_FULL_VER / 100000 % 100, _MSC_FULL_VER % 100000, _MSC_BUILD
69 | #elif defined(_MSC_VER) /* old MSVC */
70 | # define XSUM_CC_VERSION_FMT "MSVC %02i.%02i"
71 | # define XSUM_CC_VERSION _MSC_VER / 100, _MSC_VER % 100
72 | #elif defined(__TINYC__)
73 | /* tcc stores its version in the __TINYC__ macro. */
74 | # define XSUM_CC_VERSION_FMT "tcc %i.%i.%i"
75 | # define XSUM_CC_VERSION __TINYC__ / 10000 % 100, __TINYC__ / 100 % 100, __TINYC__ % 100
76 | #else
77 | # define XSUM_CC_VERSION_FMT "%s"
78 | # define XSUM_CC_VERSION "unknown compiler"
79 | #endif
80 |
81 | /* makes the next part easier */
82 | #if (defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC)
83 | # define XSUM_ARCH_X64 1
84 | # define XSUM_ARCH_X86 "x86_64"
85 | #elif defined(__i386__) || defined(_M_IX86) || defined(_M_IX86_FP)
86 | # define XSUM_ARCH_X86 "i386"
87 | #endif
88 |
89 | /* Try to detect the architecture. */
90 | #if defined(XSUM_ARCH_X86)
91 | # if defined(XXHSUM_DISPATCH)
92 | const char* XSUM_autox86(void);
93 | # define XSUM_ARCH XSUM_autox86()
94 | # elif defined(__AVX512F__)
95 | # define XSUM_ARCH XSUM_ARCH_X86 " + AVX512"
96 | # elif defined(__AVX2__)
97 | # define XSUM_ARCH XSUM_ARCH_X86 " + AVX2"
98 | # elif defined(__AVX__)
99 | # define XSUM_ARCH XSUM_ARCH_X86 " + AVX"
100 | # elif defined(_M_X64) || defined(__x86_64__) \
101 | || defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP == 2)
102 | # define XSUM_ARCH XSUM_ARCH_X86 " + SSE2"
103 | # else
104 | # define XSUM_ARCH XSUM_ARCH_X86
105 | # endif
106 | #elif defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)
107 | # if defined(__ARM_FEATURE_SVE)
108 | # define XSUM_ARCH "aarch64 + SVE"
109 | # else
110 | # define XSUM_ARCH "aarch64 + NEON"
111 | # endif
112 | #elif defined(__arm__) || defined(__thumb__) || defined(__thumb2__) || defined(_M_ARM)
113 | /* ARM has a lot of different features that can change xxHash significantly. */
114 | # ifdef __ARM_ARCH
115 | # define XSUM_ARCH_ARM_VER XSUM_EXPAND_AND_QUOTE(__ARM_ARCH)
116 | # else
117 | # define XSUM_ARCH_ARM_VER XSUM_EXPAND_AND_QUOTE(_M_ARM)
118 | # endif
119 | # if defined(_M_ARM) /* windows arm is always thumb-2 */ \
120 | || defined(__thumb2__) || (defined(__thumb__) && (__thumb__ == 2 || __ARM_ARCH >= 7))
121 | # define XSUM_ARCH_THUMB " Thumb-2"
122 | # elif defined(__thumb__)
123 | # define XSUM_ARCH_THUMB " Thumb-1"
124 | # else
125 | # define XSUM_ARCH_THUMB ""
126 | # endif
127 | /* ARMv7 has unaligned by default */
128 | # if defined(__ARM_FEATURE_UNALIGNED) || __ARM_ARCH >= 7 || defined(_M_ARM)
129 | # define XSUM_ARCH_UNALIGNED " + unaligned"
130 | # else
131 | # define XSUM_ARCH_UNALIGNED ""
132 | # endif
133 | # if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM)
134 | # define XSUM_ARCH_NEON " + NEON"
135 | # else
136 | # define XSUM_ARCH_NEON ""
137 | # endif
138 | # define XSUM_ARCH "ARMv" XSUM_ARCH_ARM_VER XSUM_ARCH_THUMB XSUM_ARCH_NEON XSUM_ARCH_UNALIGNED
139 | #elif defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__)
140 | # if defined(__GNUC__) && defined(__POWER9_VECTOR__)
141 | # define XSUM_ARCH "ppc64 + POWER9 vector"
142 | # elif defined(__GNUC__) && defined(__POWER8_VECTOR__)
143 | # define XSUM_ARCH "ppc64 + POWER8 vector"
144 | # else
145 | # define XSUM_ARCH "ppc64"
146 | # endif
147 | #elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__)
148 | # define XSUM_ARCH "ppc"
149 | #elif defined(__AVR)
150 | # define XSUM_ARCH "AVR"
151 | #elif defined(__mips64)
152 | # define XSUM_ARCH "mips64"
153 | #elif defined(__mips)
154 | # define XSUM_ARCH "mips"
155 | #elif defined(__s390x__)
156 | # define XSUM_ARCH "s390x"
157 | #elif defined(__s390__)
158 | # define XSUM_ARCH "s390"
159 | #elif defined(__wasm__) || defined(__asmjs__) || defined(__EMSCRIPTEN__)
160 | # if defined(__wasm_simd128__)
161 | # define XSUM_ARCH "wasm/asmjs + simd128"
162 | # else
163 | # define XSUM_ARCH "wasm/asmjs"
164 | # endif
165 | #elif defined(__riscv)
166 | # define XSUM_ARCH "riscv"
167 | #elif defined(__loongarch_lp64)
168 | # if defined(__loongarch_asx)
169 | # define XSUM_ARCH "loongarch64 + lasx"
170 | # elif defined(__loongarch_sx)
171 | # define XSUM_ARCH "loongarch64 + lsx"
172 | # else
173 | # define XSUM_ARCH "loongarch64"
174 | # endif
175 | #else
176 | # define XSUM_ARCH "unknown"
177 | #endif
178 |
179 |
180 | #endif /* XSUM_ARCH_H */
181 |
--------------------------------------------------------------------------------
/cli/xsum_bench.h:
--------------------------------------------------------------------------------
1 | /*
2 | * xsum_bench - Benchmark functions for xxhsum
3 | * Copyright (C) 2013-2021 Yann Collet
4 | *
5 | * GPL v2 License
6 | *
7 | * This program is free software; you can redistribute it and/or modify
8 | * it under the terms of the GNU General Public License as published by
9 | * the Free Software Foundation; either version 2 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * This program is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU General Public License along
18 | * with this program; if not, write to the Free Software Foundation, Inc.,
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | *
21 | * You can contact the author at:
22 | * - xxHash homepage: https://www.xxhash.com
23 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | */
25 |
26 | #ifndef XSUM_BENCH_H
27 | #define XSUM_BENCH_H
28 |
29 | #include <stddef.h> /* size_t */
30 |
31 | #define NBLOOPS_DEFAULT 3 /* Default number of benchmark iterations */
32 |
33 | extern int const g_nbTestFunctions;
34 | extern char g_testIDs[]; /* size : g_nbTestFunctions */
35 | extern const char k_testIDs_default[];
36 | extern int g_nbIterations;
37 |
38 | int XSUM_benchInternal(size_t keySize);
39 | int XSUM_benchFiles(const char* fileNamesTable[], int nbFiles);
40 |
41 |
42 | #ifdef __cplusplus
43 | extern "C" {
44 | #endif
45 |
46 |
47 | #ifdef __cplusplus
48 | }
49 | #endif
50 |
51 | #endif /* XSUM_BENCH_H */
52 |
--------------------------------------------------------------------------------
/cli/xsum_config.h:
--------------------------------------------------------------------------------
1 | /*
2 | * xxhsum - Command line interface for xxhash algorithms
3 | * Copyright (C) 2013-2021 Yann Collet
4 | *
5 | * GPL v2 License
6 | *
7 | * This program is free software; you can redistribute it and/or modify
8 | * it under the terms of the GNU General Public License as published by
9 | * the Free Software Foundation; either version 2 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * This program is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU General Public License along
18 | * with this program; if not, write to the Free Software Foundation, Inc.,
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | *
21 | * You can contact the author at:
22 | * - xxHash homepage: https://www.xxhash.com
23 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | */
25 |
26 | /*
27 | * This contains various configuration parameters and feature detection for
28 | * xxhsum.
29 | *
30 | * Similar to config.h in Autotools, this should be the first header included.
31 | */
32 |
33 | #ifndef XSUM_CONFIG_H
34 | #define XSUM_CONFIG_H
35 |
36 |
37 | /* ************************************
38 | * Compiler Options
39 | **************************************/
40 | /*
41 | * Disable Visual C's warnings when using the "insecure" CRT functions instead
42 | * of the "secure" _s functions.
43 | *
44 | * These functions are not portable, and aren't necessary if you are using the
45 | * original functions properly.
46 | */
47 | #if defined(_MSC_VER) || defined(_WIN32)
48 | # ifndef _CRT_SECURE_NO_WARNINGS
49 | # define _CRT_SECURE_NO_WARNINGS
50 | # endif
51 | #endif
52 |
53 | #if defined(_MSC_VER)
54 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
55 | #endif
56 |
57 | /* Under Linux at least, pull in the *64 commands */
58 | #ifndef _LARGEFILE64_SOURCE
59 | # define _LARGEFILE64_SOURCE
60 | #endif
61 | #ifndef _FILE_OFFSET_BITS
62 | # define _FILE_OFFSET_BITS 64
63 | #endif
64 |
65 | /*
66 | * So we can use __attribute__((__format__))
67 | */
68 | #ifdef __GNUC__
69 | # define XSUM_ATTRIBUTE(x) __attribute__(x)
70 | #else
71 | # define XSUM_ATTRIBUTE(x)
72 | #endif
73 |
74 | #if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
75 | || defined(__midipix__) || defined(__VMS))
76 | # if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1-2001 (SUSv3) conformant */ \
77 | || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */
78 | # define XSUM_PLATFORM_POSIX_VERSION 200112L
79 | # else
80 | # if defined(__linux__) || defined(__linux)
81 | # ifndef _POSIX_C_SOURCE
82 | # define _POSIX_C_SOURCE 200112L /* use feature test macro */
83 | # endif
84 | # endif
85 | # include <unistd.h> /* declares _POSIX_VERSION */
86 | # if defined(_POSIX_VERSION) /* POSIX compliant */
87 | # define XSUM_PLATFORM_POSIX_VERSION _POSIX_VERSION
88 | # else
89 | # define XSUM_PLATFORM_POSIX_VERSION 0
90 | # endif
91 | # endif
92 | #endif
93 | #if !defined(XSUM_PLATFORM_POSIX_VERSION)
94 | # define XSUM_PLATFORM_POSIX_VERSION -1
95 | #endif
96 |
97 | #if !defined(S_ISREG)
98 | # define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
99 | #endif
100 |
101 |
102 | /* ************************************
103 | * Windows helpers
104 | **************************************/
105 |
106 | /*
107 | * Whether to use the Windows UTF-16 APIs instead of the portable libc 8-bit
108 | * ("ANSI") APIs.
109 | *
110 | * Windows is not UTF-8 clean by default, and the only way to access every file
111 | * on the OS is to use UTF-16.
112 | *
113 | * Do note that xxhsum uses UTF-8 internally and only uses UTF-16 for command
114 | * line arguments, console I/O, and opening files.
115 | *
116 | * Additionally, this guarantees all piped output is UTF-8.
117 | */
118 | #if defined(XSUM_WIN32_USE_WCHAR) && !defined(_WIN32)
119 | /* We use Windows APIs, only use this on Windows. */
120 | # undef XSUM_WIN32_USE_WCHAR
121 | #endif
122 |
123 | #ifndef XSUM_WIN32_USE_WCHAR
124 | # if defined(_WIN32)
125 | # include <wchar.h>
126 | # if WCHAR_MAX == 0xFFFFU /* UTF-16 wchar_t */
127 | # define XSUM_WIN32_USE_WCHAR 1
128 | # else
129 | # define XSUM_WIN32_USE_WCHAR 0
130 | # endif
131 | # else
132 | # define XSUM_WIN32_USE_WCHAR 0
133 | # endif
134 | #endif
135 |
136 | #if !XSUM_WIN32_USE_WCHAR
137 | /*
138 | * It doesn't make sense to have one without the other.
139 | * Due to XSUM_WIN32_USE_WCHAR being undef'd, this also handles
140 | * non-WIN32 platforms.
141 | */
142 | # undef XSUM_WIN32_USE_WMAIN
143 | # define XSUM_WIN32_USE_WMAIN 0
144 | #else
145 | /*
146 | * Whether to use wmain() or main().
147 | *
148 | * wmain() is preferred because we don't have to mess with internal hidden
149 | * APIs.
150 | *
151 | * It always works on MSVC, but in MinGW, it only works on MinGW-w64 with the
152 | * -municode flag.
153 | *
154 | * Therefore we have to use main() -- there is no better option.
155 | */
156 | # ifndef XSUM_WIN32_USE_WMAIN
157 | # if defined(_UNICODE) || defined(UNICODE) /* MinGW -municode */ \
158 | || defined(_MSC_VER) /* MSVC */
159 | # define XSUM_WIN32_USE_WMAIN 1
160 | # else
161 | # define XSUM_WIN32_USE_WMAIN 0
162 | # endif
163 | # endif
164 | /*
165 | * It is always good practice to define these to prevent accidental use of the
166 | * ANSI APIs, even if the program primarily uses UTF-8.
167 | */
168 | # ifndef _UNICODE
169 | # define _UNICODE
170 | # endif
171 | # ifndef UNICODE
172 | # define UNICODE
173 | # endif
174 | #endif /* XSUM_WIN32_USE_WCHAR */
175 |
176 | #ifndef XSUM_API
177 | # define XSUM_API
178 | #endif
179 |
180 | #ifndef XSUM_NO_TESTS
181 | # define XSUM_NO_TESTS 0
182 | #endif
183 |
184 | /* ***************************
185 | * Basic types
186 | * ***************************/
187 |
188 | #if defined(__cplusplus) /* C++ */ \
189 | || (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) /* C99 */
190 | # include <stdint.h>
191 | typedef uint8_t XSUM_U8;
192 | typedef uint32_t XSUM_U32;
193 | typedef uint64_t XSUM_U64;
194 | # else
195 | # include <limits.h>
196 | typedef unsigned char XSUM_U8;
197 | # if UINT_MAX == 0xFFFFFFFFUL
198 | typedef unsigned int XSUM_U32;
199 | # else
200 | typedef unsigned long XSUM_U32;
201 | # endif
202 | typedef unsigned long long XSUM_U64;
203 | #endif /* not C++/C99 */
204 |
205 | /* ***************************
206 | * Common constants
207 | * ***************************/
208 |
209 | #define KB *( 1<<10)
210 | #define MB *( 1<<20)
211 | #define GB *(1U<<30)
212 |
213 |
214 | #endif /* XSUM_CONFIG_H */
215 |
--------------------------------------------------------------------------------
/cli/xsum_os_specific.h:
--------------------------------------------------------------------------------
1 | /*
2 | * xxhsum - Command line interface for xxhash algorithms
3 | * Copyright (C) 2013-2021 Yann Collet
4 | *
5 | * GPL v2 License
6 | *
7 | * This program is free software; you can redistribute it and/or modify
8 | * it under the terms of the GNU General Public License as published by
9 | * the Free Software Foundation; either version 2 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * This program is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU General Public License along
18 | * with this program; if not, write to the Free Software Foundation, Inc.,
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | *
21 | * You can contact the author at:
22 | * - xxHash homepage: https://www.xxhash.com
23 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | */
25 |
26 | #ifndef XSUM_OS_SPECIFIC_H
27 | #define XSUM_OS_SPECIFIC_H
28 |
29 | #include "xsum_config.h"
30 | #include <stdio.h>
31 | #include <stdarg.h>
32 |
33 | #ifdef __cplusplus
34 | extern "C" {
35 | #endif
36 |
37 | /*
38 | * Declared here to be implemented in user code.
39 | *
40 | * Functions like main(), but is passed UTF-8 arguments even on Windows.
41 | */
42 | XSUM_API int XSUM_main(int argc, const char* argv[]);
43 |
44 | /*
45 | * Returns whether stream is a console.
46 | *
47 | * Functionally equivalent to isatty(fileno(stream)).
48 | */
49 | XSUM_API int XSUM_isConsole(FILE* stream);
50 |
51 | /*
52 | * Sets stream to pure binary mode (a.k.a. no CRLF conversions).
53 | */
54 | XSUM_API void XSUM_setBinaryMode(FILE* stream);
55 |
56 | /*
57 | * Returns whether the file at filename is a directory.
58 | */
59 | XSUM_API int XSUM_isDirectory(const char* filename);
60 |
61 | /*
62 | * Returns the file size of the file at filename.
63 | */
64 | XSUM_API XSUM_U64 XSUM_getFileSize(const char* filename);
65 |
66 | /*
67 | * UTF-8 stdio wrappers primarily for Windows
68 | */
69 |
70 | /*
71 | * fopen() wrapper. Accepts UTF-8 filenames on Windows.
72 | *
73 | * Specifically, on Windows, the arguments will be converted to UTF-16
74 | * and passed to _wfopen().
75 | */
76 | XSUM_API FILE* XSUM_fopen(const char* filename, const char* mode);
77 |
78 | /*
79 | * vfprintf() wrapper which prints UTF-8 strings to Windows consoles
80 | * if applicable.
81 | */
82 | XSUM_ATTRIBUTE((__format__(__printf__, 2, 0)))
83 | XSUM_API int XSUM_vfprintf(FILE* stream, const char* format, va_list ap);
84 |
85 | #ifdef __cplusplus
86 | }
87 | #endif
88 |
89 | #endif /* XSUM_OS_SPECIFIC_H */
90 |
--------------------------------------------------------------------------------
/cli/xsum_output.c:
--------------------------------------------------------------------------------
1 | /*
2 | * xxhsum - Command line interface for xxhash algorithms
3 | * Copyright (C) 2013-2021 Yann Collet
4 | *
5 | * GPL v2 License
6 | *
7 | * This program is free software; you can redistribute it and/or modify
8 | * it under the terms of the GNU General Public License as published by
9 | * the Free Software Foundation; either version 2 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * This program is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU General Public License along
18 | * with this program; if not, write to the Free Software Foundation, Inc.,
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | *
21 | * You can contact the author at:
22 | * - xxHash homepage: https://www.xxhash.com
23 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | */
25 |
26 | #include "xsum_output.h"
27 | #include "xsum_os_specific.h" /* XSUM_API */
28 |
29 | int XSUM_logLevel = 2;
30 |
31 | XSUM_ATTRIBUTE((__format__(__printf__, 1, 2)))
32 | XSUM_API int XSUM_log(const char* format, ...)
33 | {
34 | int ret;
35 | va_list ap;
36 | va_start(ap, format);
37 | ret = XSUM_vfprintf(stderr, format, ap);
38 | va_end(ap);
39 | return ret;
40 | }
41 |
42 |
43 | XSUM_ATTRIBUTE((__format__(__printf__, 1, 2)))
44 | XSUM_API int XSUM_output(const char* format, ...)
45 | {
46 | int ret;
47 | va_list ap;
48 | va_start(ap, format);
49 | ret = XSUM_vfprintf(stdout, format, ap);
50 | va_end(ap);
51 | return ret;
52 | }
53 |
54 | XSUM_ATTRIBUTE((__format__(__printf__, 2, 3)))
55 | XSUM_API int XSUM_logVerbose(int minLevel, const char* format, ...)
56 | {
57 | if (XSUM_logLevel >= minLevel) {
58 | int ret;
59 | va_list ap;
60 | va_start(ap, format);
61 | ret = XSUM_vfprintf(stderr, format, ap);
62 | va_end(ap);
63 | return ret;
64 | }
65 | return 0;
66 | }
67 |
--------------------------------------------------------------------------------
/cli/xsum_output.h:
--------------------------------------------------------------------------------
1 | /*
2 | * xxhsum - Command line interface for xxhash algorithms
3 | * Copyright (C) 2013-2021 Yann Collet
4 | *
5 | * GPL v2 License
6 | *
7 | * This program is free software; you can redistribute it and/or modify
8 | * it under the terms of the GNU General Public License as published by
9 | * the Free Software Foundation; either version 2 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * This program is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU General Public License along
18 | * with this program; if not, write to the Free Software Foundation, Inc.,
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | *
21 | * You can contact the author at:
22 | * - xxHash homepage: https://www.xxhash.com
23 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | */
25 |
26 | #ifndef XSUM_OUTPUT_H
27 | #define XSUM_OUTPUT_H
28 |
29 | #include "xsum_config.h"
30 |
31 | #ifdef __cplusplus
32 | extern "C" {
33 | #endif
34 |
35 | /*
36 | * How verbose the output is.
37 | */
38 | extern int XSUM_logLevel;
39 |
40 | /*
41 | * Same as fprintf(stderr, format, ...)
42 | */
43 | XSUM_ATTRIBUTE((__format__(__printf__, 1, 2)))
44 | XSUM_API int XSUM_log(const char *format, ...);
45 |
46 | /*
47 | * Like XSUM_log, but only outputs if XSUM_logLevel >= minLevel.
48 | */
49 | XSUM_ATTRIBUTE((__format__(__printf__, 2, 3)))
50 | XSUM_API int XSUM_logVerbose(int minLevel, const char *format, ...);
51 |
52 | /*
53 | * Same as printf(format, ...)
54 | */
55 | XSUM_ATTRIBUTE((__format__(__printf__, 1, 2)))
56 | XSUM_API int XSUM_output(const char *format, ...);
57 |
58 | #ifdef __cplusplus
59 | }
60 | #endif
61 |
62 | #endif /* XSUM_OUTPUT_H */
63 |
--------------------------------------------------------------------------------
/cli/xsum_sanity_check.h:
--------------------------------------------------------------------------------
1 | /*
2 | * xxhsum - Command line interface for xxhash algorithms
3 | * Copyright (C) 2013-2021 Yann Collet
4 | *
5 | * GPL v2 License
6 | *
7 | * This program is free software; you can redistribute it and/or modify
8 | * it under the terms of the GNU General Public License as published by
9 | * the Free Software Foundation; either version 2 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * This program is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU General Public License along
18 | * with this program; if not, write to the Free Software Foundation, Inc.,
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | *
21 | * You can contact the author at:
22 | * - xxHash homepage: https://www.xxhash.com
23 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | */
25 |
26 | #ifndef XSUM_SANITY_CHECK_H
27 | #define XSUM_SANITY_CHECK_H
28 |
29 | #include "xsum_config.h" /* XSUM_API, XSUM_U8 */
30 |
31 | #include <stddef.h> /* size_t */
32 |
33 |
34 | #ifdef __cplusplus
35 | extern "C" {
36 | #endif
37 |
38 | /*
39 | * Runs a series of self-tests.
40 | *
41 | * Exits if any of these tests fail, printing a message to stderr.
42 | *
43 | * If XSUM_NO_TESTS is defined to non-zero,
44 | * this will instead print a warning if this is called (e.g. via xxhsum -b).
45 | */
46 | XSUM_API void XSUM_sanityCheck(void);
47 |
48 | /*
49 | * Fills a test buffer with pseudorandom data.
50 | *
51 | * This is used in the sanity check and the benchmarks.
52 | * Its values must not be changed.
53 | */
54 | XSUM_API void XSUM_fillTestBuffer(XSUM_U8* buffer, size_t len);
55 |
56 | #ifdef __cplusplus
57 | }
58 | #endif
59 |
60 | #endif /* XSUM_SANITY_CHECK_H */
61 |
--------------------------------------------------------------------------------
/cli/xxhsum.1:
--------------------------------------------------------------------------------
1 | .
2 | .TH "XXHSUM" "1" "May 2024" "xxhsum 0.8.3" "User Commands"
3 | .
4 | .SH "NAME"
5 | \fBxxhsum\fR \- print or check xxHash non\-cryptographic checksums
6 | .
7 | .SH "SYNOPSIS"
8 | \fBxxhsum\fR [\fIOPTION\fR]\.\.\. [\fIFILE\fR]\.\.\. \fBxxhsum \-b\fR [\fIOPTION\fR]\.\.\.
9 | .
10 | .P
11 | \fBxxh32sum\fR is equivalent to \fBxxhsum \-H0\fR, \fBxxh64sum\fR is equivalent to \fBxxhsum \-H1\fR, \fBxxh128sum\fR is equivalent to \fBxxhsum \-H2\fR, \fBxxh3sum\fR is equivalent to \fBxxhsum \-H3\fR\.
12 | .
13 | .SH "DESCRIPTION"
14 | Print or check xxHash (32, 64 or 128 bits) checksums\. When no \fIFILE\fR, read standard input, except if it\'s the console\. When \fIFILE\fR is \fB\-\fR, read standard input even if it\'s the console\.
15 | .
16 | .P
17 | \fBxxhsum\fR supports a command line syntax similar but not identical to md5sum(1)\. Differences are:
18 | .
19 | .IP "\(bu" 4
20 | \fBxxhsum\fR doesn\'t have text mode switch (\fB\-t\fR)
21 | .
22 | .IP "\(bu" 4
23 | \fBxxhsum\fR doesn\'t have short binary mode switch (\fB\-b\fR)
24 | .
25 | .IP "\(bu" 4
26 | \fBxxhsum\fR always treats files as binary file
27 | .
28 | .IP "\(bu" 4
29 | \fBxxhsum\fR has a hash selection switch (\fB\-H\fR)
30 | .
31 | .IP "" 0
32 | .
33 | .P
34 | As xxHash is a fast non\-cryptographic checksum algorithm, \fBxxhsum\fR should not be used for security related purposes\.
35 | .
36 | .P
37 | \fBxxhsum \-b\fR invokes benchmark mode\. See OPTIONS and EXAMPLES for details\.
38 | .
39 | .SH "OPTIONS"
40 | .
41 | .TP
42 | \fB\-V\fR, \fB\-\-version\fR
43 | Displays xxhsum version and exits
44 | .
45 | .TP
46 | \fB\-H\fR\fIHASHTYPE\fR
47 | Hash selection\. \fIHASHTYPE\fR means \fB0\fR=XXH32, \fB1\fR=XXH64, \fB2\fR=XXH128, \fB3\fR=XXH3\. Alternatively, \fIHASHTYPE\fR \fB32\fR=XXH32, \fB64\fR=XXH64, \fB128\fR=XXH128\. Default value is \fB1\fR (XXH64)
48 | .
49 | .TP
50 | \fB\-\-binary\fR
51 | Read in binary mode\.
52 | .
53 | .TP
54 | \fB\-\-tag\fR
55 | Output in the BSD style\.
56 | .
57 | .TP
58 | \fB\-\-little\-endian\fR
59 | Set output hexadecimal checksum value as little endian convention\. By default, value is displayed as big endian\.
60 | .
61 | .TP
62 | \fB\-h\fR, \fB\-\-help\fR
63 | Displays help and exits
64 | .
65 | .SS "The following options are useful only when verifying checksums (\-c):"
66 | .
67 | .TP
68 | \fB\-c\fR, \fB\-\-check\fR \fIFILE\fR
69 | Read xxHash sums from \fIFILE\fR and check them
70 | .
71 | .TP
72 | \fB\-q\fR, \fB\-\-quiet\fR
73 | Don\'t print OK for each successfully verified file
74 | .
75 | .TP
76 | \fB\-\-strict\fR
77 | Return an error code if any line in the file is invalid, not just if some checksums are wrong\. This policy is disabled by default, though UI will prompt an informational message if any line in the file is detected invalid\.
78 | .
79 | .TP
80 | \fB\-\-status\fR
81 | Don\'t output anything\. Status code shows success\.
82 | .
83 | .TP
84 | \fB\-w\fR, \fB\-\-warn\fR
85 | Emit a warning message about each improperly formatted checksum line\.
86 | .
87 | .SS "The following options are useful only benchmark purpose:"
88 | .
89 | .TP
90 | \fB\-b\fR
91 | Benchmark mode\. See EXAMPLES for details\.
92 | .
93 | .TP
94 | \fB\-b#\fR
95 | Specify ID of variant to be tested\. Multiple variants can be selected, separated by a \',\' comma\.
96 | .
97 | .TP
98 | \fB\-B\fR\fIBLOCKSIZE\fR
99 | Only useful for benchmark mode (\fB\-b\fR)\. See \fIEXAMPLES\fR for details\. \fIBLOCKSIZE\fR specifies benchmark mode\'s test data block size in bytes\. Default value is 102400
100 | .
101 | .TP
102 | \fB\-i\fR\fIITERATIONS\fR
103 | Only useful for benchmark mode (\fB\-b\fR)\. See \fIEXAMPLES\fR for details\. \fIITERATIONS\fR specifies number of iterations in benchmark\. Single iteration lasts approximately 1000 milliseconds\. Default value is 3
104 | .
105 | .SH "EXIT STATUS"
106 | \fBxxhsum\fR exit \fB0\fR on success, \fB1\fR if at least one file couldn\'t be read or doesn\'t have the same checksum as the \fB\-c\fR option\.
107 | .
108 | .SH "EXAMPLES"
109 | Output xxHash (64bit) checksum values of specific files to standard output
110 | .
111 | .IP "" 4
112 | .
113 | .nf
114 |
115 | $ xxhsum \-H1 foo bar baz
116 | .
117 | .fi
118 | .
119 | .IP "" 0
120 | .
121 | .P
122 | Output xxHash (32bit and 64bit) checksum values of specific files to standard output, and redirect it to \fBxyz\.xxh32\fR and \fBqux\.xxh64\fR
123 | .
124 | .IP "" 4
125 | .
126 | .nf
127 |
128 | $ xxhsum \-H0 foo bar baz > xyz\.xxh32
129 | $ xxhsum \-H1 foo bar baz > qux\.xxh64
130 | .
131 | .fi
132 | .
133 | .IP "" 0
134 | .
135 | .P
136 | Read xxHash sums from specific files and check them
137 | .
138 | .IP "" 4
139 | .
140 | .nf
141 |
142 | $ xxhsum \-c xyz\.xxh32 qux\.xxh64
143 | .
144 | .fi
145 | .
146 | .IP "" 0
147 | .
148 | .P
149 | Benchmark xxHash algorithm\. By default, \fBxxhsum\fR benchmarks xxHash main variants on a synthetic sample of 100 KB, and print results into standard output\. The first column is the algorithm, the second column is the source data size in bytes, the third column is the number of hashes generated per second (throughput), and finally the last column translates speed in megabytes per second\.
150 | .
151 | .IP "" 4
152 | .
153 | .nf
154 |
155 | $ xxhsum \-b
156 | .
157 | .fi
158 | .
159 | .IP "" 0
160 | .
161 | .P
162 | In the following example, the sample to hash is set to 16384 bytes, the variants to be benched are selected by their IDs, and each benchmark test is repeated 10 times, for increased accuracy\.
163 | .
164 | .IP "" 4
165 | .
166 | .nf
167 |
168 | $ xxhsum \-b1,2,3 \-i10 \-B16384
169 | .
170 | .fi
171 | .
172 | .IP "" 0
173 | .
174 | .SH "BUGS"
175 | Report bugs at: https://github\.com/Cyan4973/xxHash/issues/
176 | .
177 | .SH "AUTHOR"
178 | Yann Collet
179 | .
180 | .SH "SEE ALSO"
181 | md5sum(1)
182 |
--------------------------------------------------------------------------------
/cli/xxhsum.1.md:
--------------------------------------------------------------------------------
1 | xxhsum(1) -- print or check xxHash non-cryptographic checksums
2 | ==============================================================
3 |
4 | SYNOPSIS
5 | --------
6 |
7 | `xxhsum` [*OPTION*]... [*FILE*]...
8 |
9 | `xxhsum -b` [*OPTION*]...
10 |
11 | `xxh32sum` is equivalent to `xxhsum -H0`,
12 | `xxh64sum` is equivalent to `xxhsum -H1`,
13 | `xxh128sum` is equivalent to `xxhsum -H2`,
14 | `xxh3sum` is equivalent to `xxhsum -H3`.
15 |
16 |
17 | DESCRIPTION
18 | -----------
19 |
20 | Print or check xxHash (32, 64 or 128 bits) checksums.
21 | When no *FILE*, read standard input, except if it's the console.
22 | When *FILE* is `-`, read standard input even if it's the console.
23 |
24 | `xxhsum` supports a command line syntax similar but not identical to md5sum(1). Differences are:
25 |
26 | * `xxhsum` doesn't have text mode switch (`-t`)
27 | * `xxhsum` doesn't have short binary mode switch (`-b`)
28 | * `xxhsum` always treats files as binary file
29 | * `xxhsum` has a hash selection switch (`-H`)
30 |
31 | As xxHash is a fast non-cryptographic checksum algorithm,
32 | `xxhsum` should not be used for security related purposes.
33 |
34 | `xxhsum -b` invokes benchmark mode. See OPTIONS and EXAMPLES for details.
35 |
36 | OPTIONS
37 | -------
38 |
39 | * `-H`*HASHTYPE*:
40 | Hash selection. *HASHTYPE* means `0`=XXH32, `1`=XXH64, `2`=XXH128, `3`=XXH3.
41 | Alternatively, *HASHTYPE* `32`=XXH32, `64`=XXH64, `128`=XXH128.
42 | Default value is `1` (XXH64)
43 |
44 | * `--binary`:
45 | Read in binary mode.
46 |
47 | * `--tag`:
48 | Output in the BSD style.
49 |
50 | * `--little-endian`:
51 | Set output hexadecimal checksum value as little endian convention.
52 | By default, value is displayed as big endian.
53 |
54 | * `-V`, `--version`:
55 | Displays xxhsum version and exits
56 |
57 | * `-h`, `--help`:
58 | Displays help and exits
59 |
60 | ### Advanced file input options
61 |
62 | * `--files-from`, `--filelist` *FILE*:
63 | Read filenames from *FILE* and generate hashes for them.
64 | `stdin` is also a valid way to provide filenames (when no *FILE* or `-` provided).
65 | Valid format is one filename per line, which can include embedded spaces, etc with no need for quotes, escapes, etc.
66 | A line commencing with '\\' will enable the convention used in the encoding of filenames against output hashes,
67 | whereby subsequent \\\\, \n and \r sequences are converted to the single
68 | character 0x5C, 0x0A and 0x0D respectively.
69 |
70 | ### The following options are useful only for checksum verification:
71 |
72 | * `-c`, `--check` *FILE*:
73 | Read xxHash sums from *FILE* and check them
74 |
75 | * `--strict`:
76 | Return an error code if any line in *FILE* is invalid,
77 | not just if some checksums are wrong.
78 | This policy is disabled by default,
79 | though UI will prompt an informational message
80 | if any line in the file is detected invalid.
81 |
82 | * `-w`, `--warn`:
83 | Emit a warning message about each improperly formatted line in *FILE*.
84 |
85 | * `-q`, `--quiet`:
86 | Don't print OK for each successfully verified hash
87 |
88 | * `--status`:
89 | Don't output anything. Only generate a Status code to show success.
90 |
91 | ### The following options are useful only benchmark purpose:
92 |
93 | * `-b`:
94 | Benchmark mode. See EXAMPLES for details.
95 |
96 | * `-b#`:
97 | Specify ID of variant to be tested.
98 | Multiple variants can be selected, separated by a ',' comma.
99 |
100 | * `-B`*BLOCKSIZE*:
101 | Only useful for benchmark mode (`-b`). See *EXAMPLES* for details.
102 | <BLOCKSIZE> specifies benchmark mode's test data block size in bytes.
103 | Default value is 102400
104 |
105 | * `-i`*ITERATIONS*:
106 | Only useful for benchmark mode (`-b`). See *EXAMPLES* for details.
107 | <ITERATIONS> specifies number of iterations in benchmark. Single iteration
108 | lasts approximately 1000 milliseconds. Default value is 3
109 |
110 | EXIT STATUS
111 | -----------
112 |
113 | `xxhsum` exit `0` on success, `1` if at least one file couldn't be read or
114 | doesn't have the same checksum as the `-c` option.
115 |
116 | EXAMPLES
117 | --------
118 |
119 | Output xxHash (64bit) checksum values of specific files to standard output
120 |
121 | $ xxhsum -H1 foo bar baz
122 |
123 | Output xxHash (32bit and 64bit) checksum values of specific files to standard
124 | output, and redirect it to `xyz.xxh32` and `qux.xxh64`
125 |
126 | $ xxhsum -H0 foo bar baz > xyz.xxh32
127 | $ xxhsum -H1 foo bar baz > qux.xxh64
128 |
129 | Read xxHash sums from specific files and check them
130 |
131 | $ xxhsum -c xyz.xxh32 qux.xxh64
132 |
133 | Produce a list of files, then generate hashes for that list
134 |
135 | $ find . -type f -name '*.[ch]' > c-files.txt
136 | $ xxhsum --files-from c-files.txt
137 |
138 | Read the list of files from standard input to avoid the need for an intermediate file
139 |
140 | $ find . -type f -name '*.[ch]' | xxhsum --files-from -
141 |
142 | Note that if shell expansion, length of argument list, clarity of use of spaces in filenames, etc allow it then the same output as the previous example can be generated like this
143 |
144 | $ xxhsum `find . -name '*.[ch]'`
145 |
146 | Benchmark xxHash algorithm.
147 | By default, `xxhsum` benchmarks xxHash main variants
148 | on a synthetic sample of 100 KB,
149 | and print results into standard output.
150 | The first column is the algorithm,
151 | the second column is the source data size in bytes,
152 | the third column is the number of hashes generated per second (throughput),
153 | and finally the last column translates speed in megabytes per second.
154 |
155 | $ xxhsum -b
156 |
157 | In the following example,
158 | the sample to hash is set to 16384 bytes,
159 | the variants to be benched are selected by their IDs,
160 | and each benchmark test is repeated 10 times, for increased accuracy.
161 |
162 | $ xxhsum -b1,2,3 -i10 -B16384
163 |
164 | BUGS
165 | ----
166 |
167 | Report bugs at: https://github.com/Cyan4973/xxHash/issues/
168 |
169 | AUTHOR
170 | ------
171 |
172 | Yann Collet
173 |
174 | SEE ALSO
175 | --------
176 |
177 | md5sum(1)
178 |
--------------------------------------------------------------------------------
/clib.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "xxhash",
3 | "version": "0.8.2",
4 | "repo": "Cyan4973/xxhash",
5 | "description": "Extremely fast non-cryptographic hash algorithm",
6 | "keywords": ["xxhash", "hashing"],
7 | "license": "BSD-2-Clause",
8 | "src": [
9 | "xxhash.c",
10 | "xxhash.h"
11 | ]
12 | }
13 |
--------------------------------------------------------------------------------
/doc/README.md:
--------------------------------------------------------------------------------
1 | xxHash Specification
2 | =======================
3 |
4 | This directory contains material defining the xxHash algorithm.
5 | It's described in [this specification document](xxhash_spec.md).
6 |
7 | The algorithm is also be illustrated by a [simple educational library](https://github.com/easyaspi314/xxhash-clean),
8 | written by @easyaspi314 and designed for readability
9 | (as opposed to the reference library which is designed for speed).
10 |
--------------------------------------------------------------------------------
/doc/xxhash.cry:
--------------------------------------------------------------------------------
1 | module xxhash where
2 |
3 | /**
4 | * The 32-bit variant of xxHash. The first argument is the sequence
5 | * of L bytes to hash. The second argument is a seed value.
6 | */
7 | XXH32 : {L} (fin L) => [L][8] -> [32] -> [32]
8 | XXH32 input seed = XXH32_avalanche acc1
9 | where (stripes16 # stripes4 # stripes1) = input
10 | accR = foldl XXH32_rounds (XXH32_init seed) (split stripes16 : [L/16][16][8])
11 | accL = `(L % 2^^32) + if (`L:Integer) < 16
12 | then seed + PRIME32_5
13 | else XXH32_converge accR
14 | acc4 = foldl XXH32_digest4 accL (split stripes4 : [(L%16)/4][4][8])
15 | acc1 = foldl XXH32_digest1 acc4 (stripes1 : [L%4][8])
16 |
17 | /**
18 | * The 64-bit variant of xxHash. The first argument is the sequence
19 | * of L bytes to hash. The second argument is a seed value.
20 | */
21 | XXH64 : {L} (fin L) => [L][8] -> [64] -> [64]
22 | XXH64 input seed = XXH64_avalanche acc1
23 | where (stripes32 # stripes8 # stripes4 # stripes1) = input
24 | accR = foldl XXH64_rounds (XXH64_init seed) (split stripes32 : [L/32][32][8])
25 | accL = `(L % 2^^64) + if (`L:Integer) < 32
26 | then seed + PRIME64_5
27 | else XXH64_converge accR
28 | acc8 = foldl XXH64_digest8 accL (split stripes8 : [(L%32)/8][8][8])
29 | acc4 = foldl XXH64_digest4 acc8 (split stripes4 : [(L%8)/4][4][8])
30 | acc1 = foldl XXH64_digest1 acc4 (stripes1 : [L%4][8])
31 |
32 | private
33 |
34 | //Utility functions
35 |
36 | /**
37 | * Combines a sequence of bytes into a word using the little-endian
38 | * convention.
39 | */
40 | toLE bytes = join (reverse bytes)
41 |
42 | //32-bit xxHash helper functions
43 |
44 | //32-bit prime number constants
45 | PRIME32_1 = 0x9E3779B1 : [32]
46 | PRIME32_2 = 0x85EBCA77 : [32]
47 | PRIME32_3 = 0xC2B2AE3D : [32]
48 | PRIME32_4 = 0x27D4EB2F : [32]
49 | PRIME32_5 = 0x165667B1 : [32]
50 |
51 | /**
52 | * The property shows that the hexadecimal representation of the
53 | * PRIME32 constants is the same as the binary representation.
54 | */
55 | property PRIME32s_as_bits_correct =
56 | (PRIME32_1 == 0b10011110001101110111100110110001) /\
57 | (PRIME32_2 == 0b10000101111010111100101001110111) /\
58 | (PRIME32_3 == 0b11000010101100101010111000111101) /\
59 | (PRIME32_4 == 0b00100111110101001110101100101111) /\
60 | (PRIME32_5 == 0b00010110010101100110011110110001)
61 |
62 | /**
63 | * This function initializes the four internal accumulators of XXH32.
64 | */
65 | XXH32_init : [32] -> [4][32]
66 | XXH32_init seed = [acc1, acc2, acc3, acc4]
67 | where acc1 = seed + PRIME32_1 + PRIME32_2
68 | acc2 = seed + PRIME32_2
69 | acc3 = seed + 0
70 | acc4 = seed - PRIME32_1
71 |
72 | /**
73 | * This processes a single lane of the main round function of XXH32.
74 | */
75 | XXH32_round : [32] -> [32] -> [32]
76 | XXH32_round accN laneN = ((accN + laneN * PRIME32_2) <<< 13) * PRIME32_1
77 |
78 | /**
79 | * This is the main round function of XXH32 and processes a stripe,
80 | * i.e. 4 lanes with 4 bytes each.
81 | */
82 | XXH32_rounds : [4][32] -> [16][8] -> [4][32]
83 | XXH32_rounds accs stripe =
84 | [ XXH32_round accN (toLE laneN) | accN <- accs | laneN <- split stripe ]
85 |
86 | /**
87 | * This function combines the four lane accumulators into a single
88 | * 32-bit value.
89 | */
90 | XXH32_converge : [4][32] -> [32]
91 | XXH32_converge [acc1, acc2, acc3, acc4] =
92 | (acc1 <<< 1) + (acc2 <<< 7) + (acc3 <<< 12) + (acc4 <<< 18)
93 |
94 | /**
95 | * This function digests a four byte lane
96 | */
97 | XXH32_digest4 : [32] -> [4][8] -> [32]
98 | XXH32_digest4 acc lane = ((acc + toLE lane * PRIME32_3) <<< 17) * PRIME32_4
99 |
100 | /**
101 | * This function digests a single byte lane
102 | */
103 | XXH32_digest1 : [32] -> [8] -> [32]
104 | XXH32_digest1 acc lane = ((acc + (0 # lane) * PRIME32_5) <<< 11) * PRIME32_1
105 |
106 | /**
107 | * This function ensures that all input bits have a chance to impact
108 | * any bit in the output digest, resulting in an unbiased
109 | * distribution.
110 | */
111 | XXH32_avalanche : [32] -> [32]
112 | XXH32_avalanche acc0 = acc5
113 | where acc1 = acc0 ^ (acc0 >> 15)
114 | acc2 = acc1 * PRIME32_2
115 | acc3 = acc2 ^ (acc2 >> 13)
116 | acc4 = acc3 * PRIME32_3
117 | acc5 = acc4 ^ (acc4 >> 16)
118 |
119 | //64-bit xxHash helper functions
120 |
121 | //64-bit prime number constants
122 | PRIME64_1 = 0x9E3779B185EBCA87 : [64]
123 | PRIME64_2 = 0xC2B2AE3D27D4EB4F : [64]
124 | PRIME64_3 = 0x165667B19E3779F9 : [64]
125 | PRIME64_4 = 0x85EBCA77C2B2AE63 : [64]
126 | PRIME64_5 = 0x27D4EB2F165667C5 : [64]
127 |
128 | /**
129 | * The property shows that the hexadecimal representation of the
130 | * PRIME64 constants is the same as the binary representation.
131 | */
132 | property PRIME64s_as_bits_correct =
133 | (PRIME64_1 == 0b1001111000110111011110011011000110000101111010111100101010000111) /\
134 | (PRIME64_2 == 0b1100001010110010101011100011110100100111110101001110101101001111) /\
135 | (PRIME64_3 == 0b0001011001010110011001111011000110011110001101110111100111111001) /\
136 | (PRIME64_4 == 0b1000010111101011110010100111011111000010101100101010111001100011) /\
137 | (PRIME64_5 == 0b0010011111010100111010110010111100010110010101100110011111000101)
138 |
139 | /**
140 | * This function initializes the four internal accumulators of XXH64.
141 | */
142 | XXH64_init : [64] -> [4][64]
143 | XXH64_init seed = [acc1, acc2, acc3, acc4]
144 | where acc1 = seed + PRIME64_1 + PRIME64_2
145 | acc2 = seed + PRIME64_2
146 | acc3 = seed + 0
147 | acc4 = seed - PRIME64_1
148 |
149 | /**
150 | * This processes a single lane of the main round function of XXH64.
151 | */
152 | XXH64_round : [64] -> [64] -> [64]
153 | XXH64_round accN laneN = ((accN + laneN * PRIME64_2) <<< 31) * PRIME64_1
154 |
155 | /**
156 | * This is the main round function of XXH64 and processes a stripe,
157 | * i.e. 4 lanes with 8 bytes each.
158 | */
159 | XXH64_rounds : [4][64] -> [32][8] -> [4][64]
160 | XXH64_rounds accs stripe =
161 | [ XXH64_round accN (toLE laneN) | accN <- accs | laneN <- split stripe ]
162 |
163 | /**
164 | * This is a helper function, used to merge the four lane accumulators.
165 | */
166 | mergeAccumulator : [64] -> [64] -> [64]
167 | mergeAccumulator acc accN = (acc ^ XXH64_round 0 accN) * PRIME64_1 + PRIME64_4
168 |
169 | /**
170 | * This function combines the four lane accumulators into a single
171 | * 64-bit value.
172 | */
173 | XXH64_converge : [4][64] -> [64]
174 | XXH64_converge [acc1, acc2, acc3, acc4] =
175 | foldl mergeAccumulator ((acc1 <<< 1) + (acc2 <<< 7) + (acc3 <<< 12) + (acc4 <<< 18)) [acc1, acc2, acc3, acc4]
176 |
177 | /**
178 | * This function digests an eight byte lane
179 | */
180 | XXH64_digest8 : [64] -> [8][8] -> [64]
181 | XXH64_digest8 acc lane = ((acc ^ XXH64_round 0 (toLE lane)) <<< 27) * PRIME64_1 + PRIME64_4
182 |
183 | /**
184 | * This function digests a four byte lane
185 | */
186 | XXH64_digest4 : [64] -> [4][8] -> [64]
187 | XXH64_digest4 acc lane = ((acc ^ (0 # toLE lane) * PRIME64_1) <<< 23) * PRIME64_2 + PRIME64_3
188 |
189 | /**
190 | * This function digests a single byte lane
191 | */
192 | XXH64_digest1 : [64] -> [8] -> [64]
193 | XXH64_digest1 acc lane = ((acc ^ (0 # lane) * PRIME64_5) <<< 11) * PRIME64_1
194 |
195 | /**
196 | * This function ensures that all input bits have a chance to impact
197 | * any bit in the output digest, resulting in an unbiased
198 | * distribution.
199 | */
200 | XXH64_avalanche : [64] -> [64]
201 | XXH64_avalanche acc0 = acc5
202 | where acc1 = acc0 ^ (acc0 >> 33)
203 | acc2 = acc1 * PRIME64_2
204 | acc3 = acc2 ^ (acc2 >> 29)
205 | acc4 = acc3 * PRIME64_3
206 | acc5 = acc4 ^ (acc4 >> 32)
207 |
--------------------------------------------------------------------------------
/fuzz/fuzzer.c:
--------------------------------------------------------------------------------
1 | #include <stdint.h>
2 | #include "xxhash.h"
3 |
4 |
5 | int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
6 | volatile XXH64_hash_t hash64 = XXH64(data, size, 0);
7 | volatile XXH32_hash_t hash32 = XXH32(data, size, 0);
8 | (void)hash64;
9 | (void)hash32;
10 | return 0;
11 | }
12 |
--------------------------------------------------------------------------------
/libxxhash.pc.in:
--------------------------------------------------------------------------------
1 | # xxHash - Extremely fast hash algorithm
2 | # Copyright (C) 2012-2021, Yann Collet, Facebook
3 | # BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
4 |
5 | prefix=@PREFIX@
6 | exec_prefix=@EXECPREFIX@
7 | includedir=@INCLUDEDIR@
8 | libdir=@LIBDIR@
9 |
10 | Name: xxhash
11 | Description: extremely fast hash algorithm
12 | URL: http://www.xxhash.com/
13 | Version: @VERSION@
14 | License: BSD-2-clause
15 | Libs: -L${libdir} -lxxhash
16 | Cflags: -I${includedir}
17 |
--------------------------------------------------------------------------------
/tests/Makefile:
--------------------------------------------------------------------------------
1 | # ################################################################
2 | # xxHash Makefile
3 | # Copyright (C) 2012-2021 Yann Collet
4 | #
5 | # GPL v2 License
6 | #
7 | # This program is free software; you can redistribute it and/or modify
8 | # it under the terms of the GNU General Public License as published by
9 | # the Free Software Foundation; either version 2 of the License, or
10 | # (at your option) any later version.
11 | #
12 | # This program is distributed in the hope that it will be useful,
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | # GNU General Public License for more details.
16 | #
17 | # You should have received a copy of the GNU General Public License along
18 | # with this program; if not, write to the Free Software Foundation, Inc.,
19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | #
21 | # You can contact the author at:
22 | # - xxHash homepage: https://www.xxhash.com
23 | # - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | # ################################################################
25 |
26 | MAKEFLAGS += --no-print-directory
27 | CFLAGS += -Wall -Wextra -Wundef -g
28 |
29 | CP = cp
30 | NM = nm
31 | GREP = grep
32 |
33 | # Define *.exe as extension for Windows systems
34 | ifneq (,$(filter Windows%,$(OS)))
35 | EXT =.exe
36 | else
37 | EXT =
38 | endif
39 |
40 | ifneq (,$(filter %UTF-8,$(LANG)))
41 | ENABLE_UNICODE ?= 1
42 | else
43 | ENABLE_UNICODE ?= 0
44 | endif
45 |
46 | .PHONY: default
47 | default: all
48 |
49 | C_SRCDIRS = . .. ../cli
50 | include ../build/make/multiconf.make
51 |
52 | .PHONY: all
53 | all: test test_alias
54 |
55 | .PHONY: test
56 | test: test_multiInclude test_unicode test_sanity
57 |
58 | .PHONY: test_multiInclude
59 | test_multiInclude:
60 | # compile without xxhash.o, ensure symbols exist within target
61 | # Note: built using only default rules
62 | $(MAKE) multiInclude
63 | $(RM) multiInclude
64 | # compile with xxhash.o, to detect duplicated symbols
65 | $(MAKE) multiInclude_withxxhash
66 | $(RM) multiInclude_withxxhash
67 | # compile with XXH_NAMESPACE before XXH_INLINE_ALL
68 | CPPFLAGS=-DXXH_NAMESPACE=TESTN_ $(MAKE) multiInclude
69 | # no symbol prefixed TESTN_ should exist
70 | ! $(NM) multiInclude | $(GREP) TESTN_
71 | $(RM) multiInclude
72 | # compile xxhash.o with XXH_NAMESPACE
73 | CPPFLAGS=-DXXH_NAMESPACE=TESTN_ $(MAKE) multiInclude_withxxhash
74 | # symbols prefixed TESTN_ should exist in xxhash.o (though not be invoked)
75 | $(NM) multiInclude_withxxhash | $(GREP) TESTN_
76 | $(RM) multiInclude_withxxhash
77 |
78 | .PHONY: test_ppc_redefine
79 | test_ppc_redefine: ppc_define.o
80 |
81 | XXHSUM_OBJS = xxhash.o $(notdir $(patsubst %.c,%.o,$(wildcard ../cli/*.c)))
82 | xxhsum:
83 | $(eval $(call c_program,xxhsum,$(XXHSUM_OBJS)))
84 |
85 | generate_unicode_test:
86 | $(eval $(call c_program,generate_unicode_test,generate_unicode_test.o))
87 |
88 | # Make sure that Unicode filenames work.
89 | # https://github.com/Cyan4973/xxHash/issues/293
90 | .PHONY: test_unicode
91 | ifeq (0,$(ENABLE_UNICODE))
92 | test_unicode:
93 | @echo "Skipping Unicode test, your terminal doesn't appear to support UTF-8."
94 | @echo "Try with ENABLE_UNICODE=1"
95 | else
96 | test_unicode: generate_unicode_test xxhsum
97 | # Generate a Unicode filename test dynamically
98 | # to keep UTF-8 out of the source tree.
99 | ./generate_unicode_test$(EXT)
100 | $(SHELL) ./unicode_test.sh
101 | endif
102 |
103 | .PHONY: test_filename_escape
104 | test_filename_escape: xxhsum
105 | ./filename-escape.sh
106 |
107 | .PHONY: test_cli_comment_line
108 | test_cli_comment_line: xxhsum
109 | $(SHELL) ./cli-comment-line.sh
110 |
111 | .PHONY: test_cli_ignore_missing
112 | test_cli_ignore_missing: xxhsum
113 | $(SHELL) ./cli-ignore-missing.sh
114 |
115 | .PHONY: test_sanity
116 | test_sanity: sanity_test.c
117 | $(CC) $(CFLAGS) $(LDFLAGS) sanity_test.c -o sanity_test$(EXT)
118 | $(RUN_ENV) ./sanity_test$(EXT)
119 |
120 | sanity_test_vectors_generator:
121 | $(eval $(call c_program,sanity_test_vectors_generator,sanity_test_vectors_generator.o))
122 |
123 | .PHONY: sanity_test_vectors.h
124 | sanity_test_vectors.h: sanity_test_vectors_generator
125 | ./sanity_test_vectors_generator$(EXT)
126 |
127 | multiInclude_withxxhash:
128 | $(eval $(call c_program,multiInclude_withxxhash,multiInclude.o xxhash.o))
129 |
130 | test_alias: CPPFLAGS += -I..
131 | test_alias: CFLAGS += -O3 -Werror
132 | $(eval $(call c_program,test_alias,test_alias.o))
133 |
134 | .PHONY: clean
135 | clean:
136 | @$(RM) *.o
137 | @$(RM) multiInclude
138 | @$(RM) *.unicode unicode_test.*
139 | @$(RM) sanity_test$(EXT) sanity_test_vectors_generator$(EXT)
140 |
--------------------------------------------------------------------------------
/tests/bench/.clang_complete:
--------------------------------------------------------------------------------
1 | -I../..
2 |
--------------------------------------------------------------------------------
/tests/bench/.gitignore:
--------------------------------------------------------------------------------
1 | # build artifacts
2 |
3 | *.o
4 | benchHash
5 | benchHash32
6 | benchHash_avx2
7 | benchHash_avx512
8 | benchHash_hw
9 |
10 | # test files
11 |
12 | test*
13 |
--------------------------------------------------------------------------------
/tests/bench/Makefile:
--------------------------------------------------------------------------------
1 | # ################################################################
2 | # xxHash benchHash Makefile
3 | # Copyright (C) 2019-2021 Yann Collet
4 | #
5 | # GPL v2 License
6 | #
7 | # This program is free software; you can redistribute it and/or modify
8 | # it under the terms of the GNU General Public License as published by
9 | # the Free Software Foundation; either version 2 of the License, or
10 | # (at your option) any later version.
11 | #
12 | # This program is distributed in the hope that it will be useful,
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | # GNU General Public License for more details.
16 | #
17 | # You should have received a copy of the GNU General Public License along
18 | # with this program; if not, write to the Free Software Foundation, Inc.,
19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | #
21 | # You can contact the author at:
22 | # - xxHash homepage: https://www.xxhash.com
23 | # - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | # ################################################################
25 | # benchHash: A generic benchmark for hash algorithms
26 | # measuring throughput, latency and bandwidth
27 | # ################################################################
28 |
29 |
30 | CPPFLAGS += -I../.. # directory of xxHash source files
31 | CFLAGS ?= -O3
32 | CFLAGS += -Wall -Wextra -Wstrict-aliasing=1 \
33 | -std=c99
34 | CFLAGS += $(MOREFLAGS) # custom way to add flags
35 | CXXFLAGS ?= -O3
36 | LDFLAGS += $(MOREFLAGS)
37 |
38 |
39 | OBJ_LIST = main.o bhDisplay.o benchHash.o benchfn.o timefn.o
40 |
41 |
42 | default: benchHash
43 |
44 | all: benchHash
45 |
46 | benchHash32: CFLAGS += -m32
47 | benchHash32: CXXFLAGS += -m32
48 |
49 | benchHash_avx2: CFLAGS += -mavx2
50 | benchHash_avx2: CXXFLAGS += -mavx2
51 |
52 | benchHash_avx512: CFLAGS += -mavx512f
53 | benchHash_avx512: CXXFLAGS += -mavx512f
54 |
55 | benchHash_rvv: CFLAGS += -march=rv64gcv -O2
56 | benchHash_rvv: CXXFLAGS += -march=rv64gcv -O2
57 |
58 | benchHash_hw: CPPFLAGS += -DHARDWARE_SUPPORT
59 | benchHash_hw: CFLAGS += -mavx2 -maes
60 | benchHash_hw: CXXFLAGS += -mavx2 -mpclmul -std=c++14
61 |
62 | benchHash benchHash32 benchHash_avx2 benchHash_avx512 benchHash_nosimd benchHash_hw benchHash_rvv: $(OBJ_LIST)
63 | $(CXX) $(CPPFLAGS) $(CXXFLAGS) $^ $(LDFLAGS) -o $@
64 |
65 |
66 | main.o: bhDisplay.h hashes.h
67 |
68 | bhDisplay.o: bhDisplay.h benchHash.h
69 |
70 | benchHash.o: benchHash.h
71 |
72 |
73 | clean:
74 | $(RM) *.o benchHash benchHash32 benchHash_avx2 benchHash_avx512 benchHash_hw benchHash_rvv
75 |
--------------------------------------------------------------------------------
/tests/bench/benchHash.c:
--------------------------------------------------------------------------------
1 | /*
2 | * Hash benchmark module
3 | * Part of the xxHash project
4 | * Copyright (C) 2019-2021 Yann Collet
5 | *
6 | * GPL v2 License
7 | *
8 | * This program is free software; you can redistribute it and/or modify
9 | * it under the terms of the GNU General Public License as published by
10 | * the Free Software Foundation; either version 2 of the License, or
11 | * (at your option) any later version.
12 | *
13 | * This program is distributed in the hope that it will be useful,
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 | * GNU General Public License for more details.
17 | *
18 | * You should have received a copy of the GNU General Public License along
19 | * with this program; if not, write to the Free Software Foundation, Inc.,
20 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 | *
22 | * You can contact the author at:
23 | * - xxHash homepage: https://www.xxhash.com
24 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
25 | */
26 |
27 | /* benchmark hash functions */
28 |
29 | #include <stdlib.h> // malloc
30 | #include <assert.h>
31 | #include <string.h>
32 |
33 | #include "benchHash.h"
34 |
35 |
36 | static void initBuffer(void* buffer, size_t size)
37 | {
38 | const unsigned long long k1 = 11400714785074694791ULL; /* 0b1001111000110111011110011011000110000101111010111100101010000111 */
39 | const unsigned long long k2 = 14029467366897019727ULL; /* 0b1100001010110010101011100011110100100111110101001110101101001111 */
40 | unsigned long long acc = k2;
41 | unsigned char* const p = (unsigned char*)buffer;
42 | for (size_t s = 0; s < size; s++) {
43 | acc *= k1;
44 | p[s] = (unsigned char)(acc >> 56);
45 | }
46 | }
47 |
48 |
49 | #define MARGIN_FOR_LATENCY 1024
50 | #define START_MASK (MARGIN_FOR_LATENCY-1)
51 |
52 | typedef size_t (*sizeFunction_f)(size_t targetSize);
53 |
54 | /*
55 | * bench_hash_internal():
56 | * Benchmarks hashfn repeateadly over single input of size `size`
57 | * return: nb of hashes per second
58 | */
59 | static double
60 | bench_hash_internal(BMK_benchFn_t hashfn, void* payload,
61 | size_t nbBlocks, sizeFunction_f selectSize, size_t size,
62 | unsigned total_time_ms, unsigned iter_time_ms)
63 | {
64 | BMK_timedFnState_shell shell;
65 | BMK_timedFnState_t* const txf = BMK_initStatic_timedFnState(&shell, sizeof(shell), total_time_ms, iter_time_ms);
66 | assert(txf != NULL);
67 |
68 | size_t const srcSize = (size_t)size;
69 | size_t const srcBufferSize = srcSize + MARGIN_FOR_LATENCY;
70 | void* const srcBuffer = malloc(srcBufferSize);
71 | assert(srcBuffer != NULL);
72 | initBuffer(srcBuffer, srcBufferSize);
73 | #define FAKE_DSTSIZE 32
74 | size_t const dstSize = FAKE_DSTSIZE;
75 | char dstBuffer_static[FAKE_DSTSIZE] = {0};
76 |
77 | #define NB_BLOCKS_MAX 1024
78 | const void* srcBuffers[NB_BLOCKS_MAX];
79 | size_t srcSizes[NB_BLOCKS_MAX];
80 | void* dstBuffers[NB_BLOCKS_MAX];
81 | size_t dstCapacities[NB_BLOCKS_MAX];
82 | assert(nbBlocks < NB_BLOCKS_MAX);
83 |
84 | assert(size > 0);
85 | for (size_t n=0; n < nbBlocks; n++) {
86 | srcBuffers[n] = srcBuffer;
87 | srcSizes[n] = selectSize(size);
88 | dstBuffers[n] = dstBuffer_static;
89 | dstCapacities[n] = dstSize;
90 | }
91 |
92 |
93 | BMK_benchParams_t params = {
94 | .benchFn = hashfn,
95 | .benchPayload = payload,
96 | .initFn = NULL,
97 | .initPayload = NULL,
98 | .errorFn = NULL,
99 | .blockCount = nbBlocks,
100 | .srcBuffers = srcBuffers,
101 | .srcSizes = srcSizes,
102 | .dstBuffers = dstBuffers,
103 | .dstCapacities = dstCapacities,
104 | .blockResults = NULL
105 | };
106 | BMK_runOutcome_t result;
107 |
108 | memset(&result, 0, sizeof(result));
109 | while (!BMK_isCompleted_TimedFn(txf)) {
110 | result = BMK_benchTimedFn(txf, params);
111 | assert(BMK_isSuccessful_runOutcome(result));
112 | }
113 |
114 | BMK_runTime_t const runTime = BMK_extract_runTime(result);
115 |
116 | free(srcBuffer);
117 | assert(runTime.nanoSecPerRun != 0);
118 | return (1000000000U / runTime.nanoSecPerRun) * nbBlocks;
119 |
120 | }
121 |
122 |
123 | static size_t rand_1_N(size_t N) { return ((size_t)rand() % N) + 1; }
124 |
125 | static size_t identity(size_t s) { return s; }
126 |
127 | static size_t
128 | benchLatency(const void* src, size_t srcSize,
129 | void* dst, size_t dstCapacity,
130 | void* customPayload)
131 | {
132 | (void)dst; (void)dstCapacity;
133 | BMK_benchFn_t benchfn = (BMK_benchFn_t)customPayload;
134 | static size_t hash = 0;
135 |
136 | const void* const start = (const char*)src + (hash & START_MASK);
137 |
138 | return hash = benchfn(start, srcSize, dst, dstCapacity, NULL);
139 | }
140 |
141 |
142 |
143 | #ifndef SIZE_TO_HASH_PER_ROUND
144 | # define SIZE_TO_HASH_PER_ROUND 200000
145 | #endif
146 |
147 | #ifndef NB_HASH_ROUNDS_MAX
148 | # define NB_HASH_ROUNDS_MAX 1000
149 | #endif
150 |
151 | double bench_hash(BMK_benchFn_t hashfn,
152 | BMK_benchMode benchMode,
153 | size_t size, BMK_sizeMode sizeMode,
154 | unsigned total_time_ms, unsigned iter_time_ms)
155 | {
156 | sizeFunction_f const sizef = (sizeMode == BMK_fixedSize) ? identity : rand_1_N;
157 | BMK_benchFn_t const benchfn = (benchMode == BMK_throughput) ? hashfn : benchLatency;
158 | BMK_benchFn_t const payload = (benchMode == BMK_throughput) ? NULL : hashfn;
159 |
160 | size_t nbBlocks = (SIZE_TO_HASH_PER_ROUND / size) + 1;
161 | if (nbBlocks > NB_HASH_ROUNDS_MAX) nbBlocks = NB_HASH_ROUNDS_MAX;
162 |
163 | return bench_hash_internal(benchfn, payload,
164 | nbBlocks, sizef, size,
165 | total_time_ms, iter_time_ms);
166 | }
167 |
--------------------------------------------------------------------------------
/tests/bench/benchHash.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Hash benchmark module
3 | * Part of the xxHash project
4 | * Copyright (C) 2019-2021 Yann Collet
5 | *
6 | * GPL v2 License
7 | *
8 | * This program is free software; you can redistribute it and/or modify
9 | * it under the terms of the GNU General Public License as published by
10 | * the Free Software Foundation; either version 2 of the License, or
11 | * (at your option) any later version.
12 | *
13 | * This program is distributed in the hope that it will be useful,
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 | * GNU General Public License for more details.
17 | *
18 | * You should have received a copy of the GNU General Public License along
19 | * with this program; if not, write to the Free Software Foundation, Inc.,
20 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 | *
22 | * You can contact the author at:
23 | * - xxHash homepage: https://www.xxhash.com
24 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
25 | */
26 |
27 |
28 | #ifndef BENCH_HASH_H_983426678
29 | #define BENCH_HASH_H_983426678
30 |
31 | #if defined (__cplusplus)
32 | extern "C" {
33 | #endif
34 |
35 |
36 | /* === Dependencies === */
37 |
38 | #include "benchfn.h" /* BMK_benchFn_t */
39 |
40 |
41 | /* === Declarations === */
42 |
43 | typedef enum { BMK_throughput, BMK_latency } BMK_benchMode;
44 |
45 | typedef enum { BMK_fixedSize, /* hash always `size` bytes */
46 | BMK_randomSize, /* hash a random nb of bytes, between 1 and `size` (inclusive) */
47 | } BMK_sizeMode;
48 |
49 | /*
50 | * bench_hash():
51 | * Returns speed expressed as nb hashes per second.
52 | * total_time_ms: time spent benchmarking the hash function with given parameters
53 | * iter_time_ms: time spent for one round. If multiple rounds are run,
54 | * bench_hash() will report the speed of best round.
55 | */
56 | double bench_hash(BMK_benchFn_t hashfn,
57 | BMK_benchMode benchMode,
58 | size_t size, BMK_sizeMode sizeMode,
59 | unsigned total_time_ms, unsigned iter_time_ms);
60 |
61 |
62 |
63 | #if defined (__cplusplus)
64 | }
65 | #endif
66 |
67 | #endif /* BENCH_HASH_H_983426678 */
68 |
--------------------------------------------------------------------------------
/tests/bench/benchfn.c:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2016-2021 Yann Collet, Facebook, Inc.
3 | * All rights reserved.
4 | *
5 | * This source code is licensed under both the BSD-style license (found in the
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 | * in the COPYING file in the root directory of this source tree).
8 | * You may select, at your option, one of the above-listed licenses.
9 | */
10 |
11 |
12 |
13 | /* *************************************
14 | * Includes
15 | ***************************************/
16 | #include <stdlib.h> /* malloc, free */
17 | #include <string.h> /* memset */
18 | #undef NDEBUG /* assert must not be disabled */
19 | #include <assert.h> /* assert */
20 |
21 | #include "timefn.h" /* UTIL_time_t, UTIL_getTime */
22 | #include "benchfn.h"
23 |
24 |
25 | /* *************************************
26 | * Constants
27 | ***************************************/
28 | #define TIMELOOP_MICROSEC SEC_TO_MICRO /* 1 second */
29 | #define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */
30 |
31 | #define KB *(1 <<10)
32 | #define MB *(1 <<20)
33 | #define GB *(1U<<30)
34 |
35 |
36 | /* *************************************
37 | * Debug errors
38 | ***************************************/
39 | #if defined(DEBUG) && (DEBUG >= 1)
40 | # include <stdio.h> /* fprintf */
41 | # define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
42 | # define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); }
43 | #else
44 | # define DEBUGOUTPUT(...)
45 | #endif
46 |
47 |
48 | /* error without displaying */
49 | #define RETURN_QUIET_ERROR(retValue, ...) { \
50 | DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \
51 | DEBUGOUTPUT("Error : "); \
52 | DEBUGOUTPUT(__VA_ARGS__); \
53 | DEBUGOUTPUT(" \n"); \
54 | return retValue; \
55 | }
56 |
57 |
58 | /* *************************************
59 | * Benchmarking an arbitrary function
60 | ***************************************/
61 |
62 | int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome)
63 | {
64 | return outcome.error_tag_never_ever_use_directly == 0;
65 | }
66 |
67 | /* warning : this function will stop program execution if outcome is invalid !
68 | * check outcome validity first, using BMK_isValid_runResult() */
69 | BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome)
70 | {
71 | assert(outcome.error_tag_never_ever_use_directly == 0);
72 | return outcome.internal_never_ever_use_directly;
73 | }
74 |
75 | size_t BMK_extract_errorResult(BMK_runOutcome_t outcome)
76 | {
77 | assert(outcome.error_tag_never_ever_use_directly != 0);
78 | return outcome.error_result_never_ever_use_directly;
79 | }
80 |
81 | static BMK_runOutcome_t BMK_runOutcome_error(size_t errorResult)
82 | {
83 | BMK_runOutcome_t b;
84 | memset(&b, 0, sizeof(b));
85 | b.error_tag_never_ever_use_directly = 1;
86 | b.error_result_never_ever_use_directly = errorResult;
87 | return b;
88 | }
89 |
90 | static BMK_runOutcome_t BMK_setValid_runTime(BMK_runTime_t runTime)
91 | {
92 | BMK_runOutcome_t outcome;
93 | memset(&outcome, 0, sizeof(outcome));
94 | outcome.error_tag_never_ever_use_directly = 0;
95 | outcome.internal_never_ever_use_directly = runTime;
96 | return outcome;
97 | }
98 |
99 |
100 | /* initFn will be measured once, benchFn will be measured `nbLoops` times */
101 | /* initFn is optional, provide NULL if none */
102 | /* benchFn must return a size_t value that errorFn can interpret */
103 | /* takes # of blocks and list of size & stuff for each. */
104 | /* can report result of benchFn for each block into blockResult. */
105 | /* blockResult is optional, provide NULL if this information is not required */
106 | /* note : time per loop can be reported as zero if run time < timer resolution */
107 | BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t p,
108 | unsigned nbLoops)
109 | {
110 | /* init */
111 | { size_t i;
112 | for (i = 0; i < p.blockCount; i++) {
113 | memset(p.dstBuffers[i], 0xE5, p.dstCapacities[i]); /* warm up and erase result buffer */
114 | } }
115 |
116 | /* benchmark */
117 | { UTIL_time_t const clockStart = UTIL_getTime();
118 | size_t dstSize = 0;
119 | unsigned loopNb, blockNb;
120 | nbLoops += !nbLoops; /* minimum nbLoops is 1 */
121 | if (p.initFn != NULL) p.initFn(p.initPayload);
122 | for (loopNb = 0; loopNb < nbLoops; loopNb++) {
123 | for (blockNb = 0; blockNb < p.blockCount; blockNb++) {
124 | size_t const res = p.benchFn(p.srcBuffers[blockNb], p.srcSizes[blockNb],
125 | p.dstBuffers[blockNb], p.dstCapacities[blockNb],
126 | p.benchPayload);
127 | if (loopNb == 0) {
128 | if (p.blockResults != NULL) p.blockResults[blockNb] = res;
129 | if ((p.errorFn != NULL) && (p.errorFn(res))) {
130 | RETURN_QUIET_ERROR(BMK_runOutcome_error(res),
131 | "Function benchmark failed on block %u (of size %u) with error %i",
132 | blockNb, (unsigned)p.srcSizes[blockNb], (int)res);
133 | }
134 | dstSize += res;
135 | } }
136 | } /* for (loopNb = 0; loopNb < nbLoops; loopNb++) */
137 |
138 | { PTime const totalTime = UTIL_clockSpanNano(clockStart);
139 | BMK_runTime_t rt;
140 | rt.nanoSecPerRun = (double)totalTime / nbLoops;
141 | rt.sumOfReturn = dstSize;
142 | return BMK_setValid_runTime(rt);
143 | } }
144 | }
145 |
146 |
147 | /* ==== Benchmarking any function, providing intermediate results ==== */
148 |
149 | struct BMK_timedFnState_s {
150 | PTime timeSpent_ns;
151 | PTime timeBudget_ns;
152 | PTime runBudget_ns;
153 | BMK_runTime_t fastestRun;
154 | unsigned nbLoops;
155 | UTIL_time_t coolTime;
156 | }; /* typedef'd to BMK_timedFnState_t within bench.h */
157 |
158 | BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms)
159 | {
160 | BMK_timedFnState_t* const r = (BMK_timedFnState_t*)malloc(sizeof(*r));
161 | if (r == NULL) return NULL; /* malloc() error */
162 | BMK_resetTimedFnState(r, total_ms, run_ms);
163 | return r;
164 | }
165 |
166 | void BMK_freeTimedFnState(BMK_timedFnState_t* state) { free(state); }
167 |
168 | BMK_timedFnState_t*
169 | BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms)
170 | {
171 | typedef char check_size[ 2 * (sizeof(BMK_timedFnState_shell) >= sizeof(struct BMK_timedFnState_s)) - 1]; /* static assert : a compilation failure indicates that BMK_timedFnState_shell is not large enough */
172 | typedef struct { check_size c; BMK_timedFnState_t tfs; } tfs_align; /* force tfs to be aligned at its next best position */
173 | size_t const tfs_alignment = offsetof(tfs_align, tfs); /* provides the minimal alignment restriction for BMK_timedFnState_t */
174 | BMK_timedFnState_t* const r = (BMK_timedFnState_t*)buffer;
175 | if (buffer == NULL) return NULL;
176 | if (size < sizeof(struct BMK_timedFnState_s)) return NULL;
177 | if ((size_t)buffer % tfs_alignment) return NULL; /* buffer must be properly aligned */
178 | BMK_resetTimedFnState(r, total_ms, run_ms);
179 | return r;
180 | }
181 |
182 | void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms)
183 | {
184 | if (!total_ms) total_ms = 1 ;
185 | if (!run_ms) run_ms = 1;
186 | if (run_ms > total_ms) run_ms = total_ms;
187 | timedFnState->timeSpent_ns = 0;
188 | timedFnState->timeBudget_ns = (PTime)total_ms * TIMELOOP_NANOSEC / 1000;
189 | timedFnState->runBudget_ns = (PTime)run_ms * TIMELOOP_NANOSEC / 1000;
190 | timedFnState->fastestRun.nanoSecPerRun = (double)TIMELOOP_NANOSEC * 2000000000; /* hopefully large enough : must be larger than any potential measurement */
191 | timedFnState->fastestRun.sumOfReturn = (size_t)(-1LL);
192 | timedFnState->nbLoops = 1;
193 | timedFnState->coolTime = UTIL_getTime();
194 | }
195 |
196 | /* Tells if nb of seconds set in timedFnState for all runs is spent.
197 | * note : this function will return 1 if BMK_benchFunctionTimed() has actually errored. */
198 | int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState)
199 | {
200 | return (timedFnState->timeSpent_ns >= timedFnState->timeBudget_ns);
201 | }
202 |
203 |
204 | #undef MIN
205 | #define MIN(a,b) ( (a) < (b) ? (a) : (b) )
206 |
207 | #define MINUSABLETIME (TIMELOOP_NANOSEC / 2) /* 0.5 seconds */
208 |
209 | BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* cont,
210 | BMK_benchParams_t p)
211 | {
212 | PTime const runBudget_ns = cont->runBudget_ns;
213 | PTime const runTimeMin_ns = runBudget_ns / 2;
214 | BMK_runTime_t bestRunTime = cont->fastestRun;
215 |
216 | for (;;) {
217 | BMK_runOutcome_t const runResult = BMK_benchFunction(p, cont->nbLoops);
218 |
219 | if (!BMK_isSuccessful_runOutcome(runResult)) { /* error : move out */
220 | return runResult;
221 | }
222 |
223 | { BMK_runTime_t const newRunTime = BMK_extract_runTime(runResult);
224 | double const loopDuration_ns = newRunTime.nanoSecPerRun * cont->nbLoops;
225 |
226 | cont->timeSpent_ns += (unsigned long long)loopDuration_ns;
227 |
228 | /* estimate nbLoops for next run to last approximately 1 second */
229 | if (loopDuration_ns > (runBudget_ns / 50)) {
230 | double const fastestRun_ns = MIN(bestRunTime.nanoSecPerRun, newRunTime.nanoSecPerRun);
231 | cont->nbLoops = (unsigned)(runBudget_ns / fastestRun_ns) + 1;
232 | } else {
233 | /* previous run was too short : blindly increase workload by x multiplier */
234 | const unsigned multiplier = 10;
235 | assert(cont->nbLoops < ((unsigned)-1) / multiplier); /* avoid overflow */
236 | cont->nbLoops *= multiplier;
237 | }
238 |
239 | if (loopDuration_ns < runTimeMin_ns) {
240 | /* When benchmark run time is too small : don't report results.
241 | * increased risks of rounding errors */
242 | continue;
243 | }
244 |
245 | if (newRunTime.nanoSecPerRun < bestRunTime.nanoSecPerRun) {
246 | bestRunTime = newRunTime;
247 | }
248 | }
249 | break;
250 | } /* while (!completed) */
251 |
252 | return BMK_setValid_runTime(bestRunTime);
253 | }
254 |
--------------------------------------------------------------------------------
/tests/bench/benchfn.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2016-2021 Yann Collet, Facebook, Inc.
3 | * All rights reserved.
4 | *
5 | * This source code is licensed under both the BSD-style license (found in the
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 | * in the COPYING file in the root directory of this source tree).
8 | * You may select, at your option, one of the above-listed licenses.
9 | */
10 |
11 |
12 | /* benchfn :
13 | * benchmark any function on a set of input
14 | * providing result in nanoSecPerRun
15 | * or detecting and returning an error
16 | */
17 |
18 | #if defined (__cplusplus)
19 | extern "C" {
20 | #endif
21 |
22 | #ifndef BENCH_FN_H_23876
23 | #define BENCH_FN_H_23876
24 |
25 | /* === Dependencies === */
26 | #include <stddef.h> /* size_t */
27 |
28 |
29 | /* ==== Benchmark any function, iterated on a set of blocks ==== */
30 |
31 | /* BMK_runTime_t: valid result return type */
32 |
33 | typedef struct {
34 | double nanoSecPerRun; /* time per iteration (over all blocks) */
35 | size_t sumOfReturn; /* sum of return values */
36 | } BMK_runTime_t;
37 |
38 |
39 | /* BMK_runOutcome_t:
40 | * type expressing the outcome of a benchmark run by BMK_benchFunction(),
41 | * which can be either valid or invalid.
42 | * benchmark outcome can be invalid if errorFn is provided.
43 | * BMK_runOutcome_t must be considered "opaque" : never access its members directly.
44 | * Instead, use its assigned methods :
45 | * BMK_isSuccessful_runOutcome, BMK_extract_runTime, BMK_extract_errorResult.
46 | * The structure is only described here to allow its allocation on stack. */
47 |
48 | typedef struct {
49 | BMK_runTime_t internal_never_ever_use_directly;
50 | size_t error_result_never_ever_use_directly;
51 | int error_tag_never_ever_use_directly;
52 | } BMK_runOutcome_t;
53 |
54 |
55 | /* prototypes for benchmarked functions */
56 | typedef size_t (*BMK_benchFn_t)(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload);
57 | typedef size_t (*BMK_initFn_t)(void* initPayload);
58 | typedef unsigned (*BMK_errorFn_t)(size_t);
59 |
60 |
61 | /* BMK_benchFunction() parameters are provided via the following structure.
62 | * A structure is preferable for readability,
63 | * as the number of parameters required is fairly large.
64 | * No initializer is provided, because it doesn't make sense to provide some "default" :
65 | * all parameters must be specified by the caller.
66 | * optional parameters are labelled explicitly, and accept value NULL when not used */
67 | typedef struct {
68 | BMK_benchFn_t benchFn; /* the function to benchmark, over the set of blocks */
69 | void* benchPayload; /* pass custom parameters to benchFn :
70 | * (*benchFn)(srcBuffers[i], srcSizes[i], dstBuffers[i], dstCapacities[i], benchPayload) */
71 | BMK_initFn_t initFn; /* (*initFn)(initPayload) is run once per run, at the beginning. */
72 | void* initPayload; /* Both arguments can be NULL, in which case nothing is run. */
73 | BMK_errorFn_t errorFn; /* errorFn will check each return value of benchFn over each block, to determine if it failed or not.
74 | * errorFn can be NULL, in which case no check is performed.
75 | * errorFn must return 0 when benchFn was successful, and >= 1 if it detects an error.
76 | * Execution is stopped as soon as an error is detected.
77 | * the triggering return value can be retrieved using BMK_extract_errorResult(). */
78 | size_t blockCount; /* number of blocks to operate benchFn on.
79 | * It's also the size of all array parameters :
80 | * srcBuffers, srcSizes, dstBuffers, dstCapacities, blockResults */
81 | const void *const * srcBuffers; /* read-only array of buffers to be operated on by benchFn */
82 | const size_t* srcSizes; /* read-only array containing sizes of srcBuffers */
83 | void *const * dstBuffers; /* array of buffers to be written into by benchFn. This array is not optional, it must be provided even if unused by benchfn. */
84 | const size_t* dstCapacities; /* read-only array containing capacities of dstBuffers. This array must be present. */
85 | size_t* blockResults; /* Optional: store the return value of benchFn for each block. Use NULL if this result is not requested. */
86 | } BMK_benchParams_t;
87 |
88 |
89 | /* BMK_benchFunction() :
90 | * This function benchmarks benchFn and initFn, providing a result.
91 | *
92 | * params : see description of BMK_benchParams_t above.
93 | * nbLoops: defines number of times benchFn is run over the full set of blocks.
94 | * Minimum value is 1. A 0 is interpreted as a 1.
95 | *
96 | * @return: can express either an error or a successful result.
97 | * Use BMK_isSuccessful_runOutcome() to check if benchmark was successful.
98 | * If yes, extract the result with BMK_extract_runTime(),
99 | * it will contain :
100 | * .sumOfReturn : the sum of all return values of benchFn through all of blocks
101 | * .nanoSecPerRun : time per run of benchFn + (time for initFn / nbLoops)
102 | * .sumOfReturn is generally intended for functions which return a # of bytes written into dstBuffer,
103 | * in which case, this value will be the total amount of bytes written into dstBuffer.
104 | *
105 | * blockResults : when provided (!= NULL), and when benchmark is successful,
106 | * params.blockResults contains all return values of `benchFn` over all blocks.
107 | * when provided (!= NULL), and when benchmark failed,
108 | * params.blockResults contains return values of `benchFn` over all blocks preceding and including the failed block.
109 | */
110 | BMK_runOutcome_t BMK_benchFunction(BMK_benchParams_t params, unsigned nbLoops);
111 |
112 |
113 |
114 | /* check first if the benchmark was successful or not */
115 | int BMK_isSuccessful_runOutcome(BMK_runOutcome_t outcome);
116 |
117 | /* If the benchmark was successful, extract the result.
118 | * note : this function will abort() program execution if benchmark failed !
119 | * always check if benchmark was successful first !
120 | */
121 | BMK_runTime_t BMK_extract_runTime(BMK_runOutcome_t outcome);
122 |
123 | /* when benchmark failed, it means one invocation of `benchFn` failed.
124 | * The failure was detected by `errorFn`, operating on return values of `benchFn`.
125 | * Returns the faulty return value.
126 | * note : this function will abort() program execution if benchmark did not failed.
127 | * always check if benchmark failed first !
128 | */
129 | size_t BMK_extract_errorResult(BMK_runOutcome_t outcome);
130 |
131 |
132 |
133 | /* ==== Benchmark any function, returning intermediate results ==== */
134 |
135 | /* state information tracking benchmark session */
136 | typedef struct BMK_timedFnState_s BMK_timedFnState_t;
137 |
138 | /* BMK_benchTimedFn() :
139 | * Similar to BMK_benchFunction(), most arguments being identical.
140 | * Automatically determines `nbLoops` so that each result is regularly produced at interval of about run_ms.
141 | * Note : minimum `nbLoops` is 1, therefore a run may last more than run_ms, and possibly even more than total_ms.
142 | * Usage - initialize timedFnState, select benchmark duration (total_ms) and each measurement duration (run_ms)
143 | * call BMK_benchTimedFn() repetitively, each measurement is supposed to last about run_ms
144 | * Check if total time budget is spent or exceeded, using BMK_isCompleted_TimedFn()
145 | */
146 | BMK_runOutcome_t BMK_benchTimedFn(BMK_timedFnState_t* timedFnState,
147 | BMK_benchParams_t params);
148 |
149 | /* Tells if duration of all benchmark runs has exceeded total_ms
150 | */
151 | int BMK_isCompleted_TimedFn(const BMK_timedFnState_t* timedFnState);
152 |
153 | /* BMK_createTimedFnState() and BMK_resetTimedFnState() :
154 | * Create/Set BMK_timedFnState_t for next benchmark session,
155 | * which shall last a minimum of total_ms milliseconds,
156 | * producing intermediate results, paced at interval of (approximately) run_ms.
157 | */
158 | BMK_timedFnState_t* BMK_createTimedFnState(unsigned total_ms, unsigned run_ms);
159 | void BMK_resetTimedFnState(BMK_timedFnState_t* timedFnState, unsigned total_ms, unsigned run_ms);
160 | void BMK_freeTimedFnState(BMK_timedFnState_t* state);
161 |
162 |
163 | /* BMK_timedFnState_shell and BMK_initStatic_timedFnState() :
164 | * Makes it possible to statically allocate a BMK_timedFnState_t on stack.
165 | * BMK_timedFnState_shell is only there to allocate space,
166 | * never ever access its members.
167 | * BMK_timedFnState_t() actually accepts any buffer.
168 | * It will check if provided buffer is large enough and is correctly aligned,
169 | * and will return NULL if conditions are not respected.
170 | */
171 | #define BMK_TIMEDFNSTATE_SIZE 64
172 | typedef union {
173 | char never_access_space[BMK_TIMEDFNSTATE_SIZE];
174 | long long alignment_enforcer; /* must be aligned on 8-bytes boundaries */
175 | } BMK_timedFnState_shell;
176 | BMK_timedFnState_t* BMK_initStatic_timedFnState(void* buffer, size_t size, unsigned total_ms, unsigned run_ms);
177 |
178 |
179 | #endif /* BENCH_FN_H_23876 */
180 |
181 | #if defined (__cplusplus)
182 | }
183 | #endif
184 |
--------------------------------------------------------------------------------
/tests/bench/bhDisplay.c:
--------------------------------------------------------------------------------
1 | /*
2 | * CSV Display module for the hash benchmark program
3 | * Part of the xxHash project
4 | * Copyright (C) 2019-2021 Yann Collet
5 | *
6 | * GPL v2 License
7 | *
8 | * This program is free software; you can redistribute it and/or modify
9 | * it under the terms of the GNU General Public License as published by
10 | * the Free Software Foundation; either version 2 of the License, or
11 | * (at your option) any later version.
12 | *
13 | * This program is distributed in the hope that it will be useful,
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 | * GNU General Public License for more details.
17 | *
18 | * You should have received a copy of the GNU General Public License along
19 | * with this program; if not, write to the Free Software Foundation, Inc.,
20 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 | *
22 | * You can contact the author at :
23 | * - xxHash homepage : https://www.xxhash.com
24 | * - xxHash source repository : https://github.com/Cyan4973/xxHash
25 | */
26 |
27 |
28 | /* === Dependencies === */
29 |
30 | #include <stdlib.h> /* rand */
31 | #include <stdio.h> /* printf */
32 | #include <assert.h>
33 |
34 | #include "benchHash.h"
35 | #include "bhDisplay.h"
36 |
37 |
38 | /* === benchmark large input === */
39 |
40 | #define MB_UNIT 1000000
41 | #define BENCH_LARGE_ITER_MS 490
42 | #define BENCH_LARGE_TOTAL_MS 1010
43 | static void bench_oneHash_largeInput(Bench_Entry hashDesc, int minlog, int maxlog)
44 | {
45 | printf("%-7s", hashDesc.name);
46 | for (int sizelog=minlog; sizelog<=maxlog; sizelog++) {
47 | size_t const inputSize = (size_t)1 << sizelog;
48 | double const nbhps = bench_hash(hashDesc.hash, BMK_throughput,
49 | inputSize, BMK_fixedSize,
50 | BENCH_LARGE_TOTAL_MS, BENCH_LARGE_ITER_MS);
51 | printf(",%6.0f", nbhps * inputSize / MB_UNIT); fflush(NULL);
52 | }
53 | printf("\n");
54 | }
55 |
56 | void bench_largeInput(Bench_Entry const* hashDescTable, int nbHashes, int minlog, int maxlog)
57 | {
58 | assert(maxlog < 31);
59 | assert(minlog >= 0);
60 | printf("benchmarking large inputs : from %u bytes (log%i) to %u MB (log%i) \n",
61 | 1U << minlog, minlog,
62 | (1U << maxlog) >> 20, maxlog);
63 | for (int i=0; i<nbHashes; i++)
64 | bench_oneHash_largeInput(hashDescTable[i], minlog, maxlog);
65 | }
66 |
67 |
68 |
69 | /* === Benchmark small inputs === */
70 |
71 | #define BENCH_SMALL_ITER_MS 170
72 | #define BENCH_SMALL_TOTAL_MS 490
73 | static void bench_throughput_oneHash_smallInputs(Bench_Entry hashDesc, size_t sizeMin, size_t sizeMax)
74 | {
75 | printf("%-7s", hashDesc.name);
76 | for (size_t s=sizeMin; s<sizeMax+1; s++) {
77 | double const nbhps = bench_hash(hashDesc.hash, BMK_throughput,
78 | s, BMK_fixedSize,
79 | BENCH_SMALL_TOTAL_MS, BENCH_SMALL_ITER_MS);
80 | printf(",%10.0f", nbhps); fflush(NULL);
81 | }
82 | printf("\n");
83 | }
84 |
85 | void bench_throughput_smallInputs(Bench_Entry const* hashDescTable, int nbHashes, size_t sizeMin, size_t sizeMax)
86 | {
87 | printf("Throughput small inputs of fixed size (from %zu to %zu bytes): \n",
88 | sizeMin, sizeMax);
89 | for (int i=0; i<nbHashes; i++)
90 | bench_throughput_oneHash_smallInputs(hashDescTable[i], sizeMin, sizeMax);
91 | }
92 |
93 |
94 |
95 | /* === Latency measurements (small keys) === */
96 |
97 | static void bench_latency_oneHash_smallInputs(Bench_Entry hashDesc, size_t size_min, size_t size_max)
98 | {
99 | printf("%-7s", hashDesc.name);
100 | for (size_t s=size_min; s<size_max+1; s++) {
101 | double const nbhps = bench_hash(hashDesc.hash, BMK_latency,
102 | s, BMK_fixedSize,
103 | BENCH_SMALL_TOTAL_MS, BENCH_SMALL_ITER_MS);
104 | printf(",%10.0f", nbhps); fflush(NULL);
105 | }
106 | printf("\n");
107 | }
108 |
109 | void bench_latency_smallInputs(Bench_Entry const* hashDescTable, int nbHashes, size_t size_min, size_t size_max)
110 | {
111 | printf("Latency for small inputs of fixed size : \n");
112 | for (int i=0; i<nbHashes; i++)
113 | bench_latency_oneHash_smallInputs(hashDescTable[i], size_min, size_max);
114 | }
115 |
116 |
117 | /* === Random input Length === */
118 |
119 | static void bench_randomInputLength_withOneHash(Bench_Entry hashDesc, size_t size_min, size_t size_max)
120 | {
121 | printf("%-7s", hashDesc.name);
122 | for (size_t s=size_min; s<size_max+1; s++) {
123 | srand((unsigned)s); /* ensure random sequence of length will be the same for a given s */
124 | double const nbhps = bench_hash(hashDesc.hash, BMK_throughput,
125 | s, BMK_randomSize,
126 | BENCH_SMALL_TOTAL_MS, BENCH_SMALL_ITER_MS);
127 | printf(",%10.0f", nbhps); fflush(NULL);
128 | }
129 | printf("\n");
130 | }
131 |
132 | void bench_throughput_randomInputLength(Bench_Entry const* hashDescTable, int nbHashes, size_t size_min, size_t size_max)
133 | {
134 | printf("benchmarking random size inputs [1-N] : \n");
135 | for (int i=0; i<nbHashes; i++)
136 | bench_randomInputLength_withOneHash(hashDescTable[i], size_min, size_max);
137 | }
138 |
139 |
140 | /* === Latency with Random input Length === */
141 |
142 | static void bench_latency_oneHash_randomInputLength(Bench_Entry hashDesc, size_t size_min, size_t size_max)
143 | {
144 | printf("%-7s", hashDesc.name);
145 | for (size_t s=size_min; s<size_max+1; s++) {
146 | srand((unsigned)s); /* ensure random sequence of length will be the same for a given s */
147 | double const nbhps = bench_hash(hashDesc.hash, BMK_latency,
148 | s, BMK_randomSize,
149 | BENCH_SMALL_TOTAL_MS, BENCH_SMALL_ITER_MS);
150 | printf(",%10.0f", nbhps); fflush(NULL);
151 | }
152 | printf("\n");
153 | }
154 |
155 | void bench_latency_randomInputLength(Bench_Entry const* hashDescTable, int nbHashes, size_t size_min, size_t size_max)
156 | {
157 | printf("Latency for small inputs of random size [1-N] : \n");
158 | for (int i=0; i<nbHashes; i++)
159 | bench_latency_oneHash_randomInputLength(hashDescTable[i], size_min, size_max);
160 | }
161 |
--------------------------------------------------------------------------------
/tests/bench/bhDisplay.h:
--------------------------------------------------------------------------------
1 | /*
2 | * CSV Display module for the hash benchmark program
3 | * Part of the xxHash project
4 | * Copyright (C) 2019-2021 Yann Collet
5 | *
6 | * GPL v2 License
7 | *
8 | * This program is free software; you can redistribute it and/or modify
9 | * it under the terms of the GNU General Public License as published by
10 | * the Free Software Foundation; either version 2 of the License, or
11 | * (at your option) any later version.
12 | *
13 | * This program is distributed in the hope that it will be useful,
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 | * GNU General Public License for more details.
17 | *
18 | * You should have received a copy of the GNU General Public License along
19 | * with this program; if not, write to the Free Software Foundation, Inc.,
20 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 | *
22 | * You can contact the author at:
23 | * - xxHash homepage: https://www.xxhash.com
24 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
25 | */
26 |
27 | #ifndef BH_DISPLAY_H_192088098
28 | #define BH_DISPLAY_H_192088098
29 |
30 | #if defined (__cplusplus)
31 | extern "C" {
32 | #endif
33 |
34 |
35 | /* === Dependencies === */
36 |
37 | #include "benchfn.h" /* BMK_benchFn_t */
38 |
39 |
40 | /* === Declarations === */
41 |
42 | typedef struct {
43 | const char* name;
44 | BMK_benchFn_t hash;
45 | } Bench_Entry;
46 |
47 | void bench_largeInput(Bench_Entry const* hashDescTable, int nbHashes, int sizeLogMin, int sizeLogMax);
48 |
49 | void bench_throughput_smallInputs(Bench_Entry const* hashDescTable, int nbHashes, size_t sizeMin, size_t sizeMax);
50 | void bench_throughput_randomInputLength(Bench_Entry const* hashDescTable, int nbHashes, size_t sizeMin, size_t sizeMax);
51 |
52 | void bench_latency_smallInputs(Bench_Entry const* hashDescTable, int nbHashes, size_t sizeMin, size_t sizeMax);
53 | void bench_latency_randomInputLength(Bench_Entry const* hashDescTable, int nbHashes, size_t sizeMin, size_t sizeMax);
54 |
55 |
56 |
57 | #if defined (__cplusplus)
58 | }
59 | #endif
60 |
61 | #endif /* BH_DISPLAY_H_192088098 */
62 |
--------------------------------------------------------------------------------
/tests/bench/hashes.h:
--------------------------------------------------------------------------------
1 | /*
2 | * List hash algorithms to benchmark
3 | * Part of xxHash project
4 | * Copyright (C) 2019-2021 Yann Collet
5 | *
6 | * GPL v2 License
7 | *
8 | * This program is free software; you can redistribute it and/or modify
9 | * it under the terms of the GNU General Public License as published by
10 | * the Free Software Foundation; either version 2 of the License, or
11 | * (at your option) any later version.
12 | *
13 | * This program is distributed in the hope that it will be useful,
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 | * GNU General Public License for more details.
17 | *
18 | * You should have received a copy of the GNU General Public License along
19 | * with this program; if not, write to the Free Software Foundation, Inc.,
20 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 | *
22 | * You can contact the author at:
23 | * - xxHash homepage: https://www.xxhash.com
24 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
25 | */
26 |
27 |
28 | /* === Dependencies === */
29 |
30 | #include <stddef.h> /* size_t */
31 |
32 |
33 | /* ==================================================
34 | * Non-portable hash algorithms
35 | * =============================================== */
36 |
37 |
38 | #ifdef HARDWARE_SUPPORT
39 |
40 | /*
41 | * List any hash algorithms that depend on specific hardware support,
42 | * including for example:
43 | * - Hardware crc32c
44 | * - Hardware AES support
45 | * - Carryless Multipliers (clmul)
46 | * - AVX2
47 | */
48 |
49 | #endif
50 |
51 |
52 |
53 | /* ==================================================
54 | * List of hashes
55 | * ==================================================
56 | * Each hash must be wrapped in a thin redirector conformant with the BMK_benchfn_t.
57 | * BMK_benchfn_t is generic, not specifically designed for hashes.
58 | * For hashes, the following parameters are expected to be useless:
59 | * dst, dstCapacity, customPayload.
60 | *
61 | * The result of each hash is assumed to be provided as function return value.
62 | * This condition is important for latency measurements.
63 | */
64 |
65 | /* === xxHash === */
66 | #define XXH_INLINE_ALL
67 | #include "xxhash.h"
68 |
69 | size_t XXH32_wrapper(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload)
70 | {
71 | (void)dst; (void)dstCapacity; (void)customPayload;
72 | return (size_t) XXH32(src, srcSize, 0);
73 | }
74 |
75 |
76 | size_t XXH64_wrapper(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload)
77 | {
78 | (void)dst; (void)dstCapacity; (void)customPayload;
79 | return (size_t) XXH64(src, srcSize, 0);
80 | }
81 |
82 |
83 | size_t xxh3_wrapper(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload)
84 | {
85 | (void)dst; (void)dstCapacity; (void)customPayload;
86 | return (size_t) XXH3_64bits(src, srcSize);
87 | }
88 |
89 |
90 | size_t XXH128_wrapper(const void* src, size_t srcSize, void* dst, size_t dstCapacity, void* customPayload)
91 | {
92 | (void)dst; (void)dstCapacity; (void)customPayload;
93 | return (size_t) XXH3_128bits(src, srcSize).low64;
94 | }
95 |
96 |
97 |
98 | /* ==================================================
99 | * Table of hashes
100 | * =============================================== */
101 |
102 | #include "bhDisplay.h" /* Bench_Entry */
103 |
104 | #ifndef HARDWARE_SUPPORT
105 | # define NB_HASHES 4
106 | #else
107 | # define NB_HASHES 4
108 | #endif
109 |
110 | Bench_Entry const hashCandidates[NB_HASHES] = {
111 | { "xxh3" , xxh3_wrapper },
112 | { "XXH32" , XXH32_wrapper },
113 | { "XXH64" , XXH64_wrapper },
114 | { "XXH128", XXH128_wrapper },
115 | #ifdef HARDWARE_SUPPORT
116 | /* list here codecs which require specific hardware support, such SSE4.1, PCLMUL, AVX2, etc. */
117 | #endif
118 | };
119 |
--------------------------------------------------------------------------------
/tests/bench/main.c:
--------------------------------------------------------------------------------
1 | /*
2 | * Main program to benchmark hash functions
3 | * Part of the xxHash project
4 | * Copyright (C) 2019-2021 Yann Collet
5 | * GPL v2 License
6 | *
7 | * This program is free software; you can redistribute it and/or modify
8 | * it under the terms of the GNU General Public License as published by
9 | * the Free Software Foundation; either version 2 of the License, or
10 | * (at your option) any later version.
11 | *
12 | * This program is distributed in the hope that it will be useful,
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | * GNU General Public License for more details.
16 | *
17 | * You should have received a copy of the GNU General Public License along
18 | * with this program; if not, write to the Free Software Foundation, Inc.,
19 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | *
21 | * You can contact the author at:
22 | * - xxHash homepage: https://www.xxhash.com
23 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | */
25 |
26 |
27 | /* === dependencies === */
28 |
29 | #include <stdio.h> /* printf */
30 | #include <limits.h> /* INT_MAX */
31 | #include "bhDisplay.h" /* bench_x */
32 |
33 |
34 | /* === defines list of hashes `hashCandidates` and NB_HASHES *** */
35 |
36 | #include "hashes.h"
37 |
38 |
39 | /* === parse command line === */
40 |
41 | #undef NDEBUG
42 | #include <assert.h>
43 |
44 |
45 | /*!
46 | * readIntFromChar():
47 | * Allows and interprets K, KB, KiB, M, MB and MiB suffix.
48 | * Will also modify `*stringPtr`, advancing it to position where it stopped reading.
49 | */
50 | static int readIntFromChar(const char** stringPtr)
51 | {
52 | static int const max = (INT_MAX / 10) - 1;
53 | int result = 0;
54 | while ((**stringPtr >='0') && (**stringPtr <='9')) {
55 | assert(result < max);
56 | result *= 10;
57 | result += (unsigned)(**stringPtr - '0');
58 | (*stringPtr)++ ;
59 | }
60 | if ((**stringPtr=='K') || (**stringPtr=='M')) {
61 | int const maxK = INT_MAX >> 10;
62 | assert(result < maxK);
63 | result <<= 10;
64 | if (**stringPtr=='M') {
65 | assert(result < maxK);
66 | result <<= 10;
67 | }
68 | (*stringPtr)++; /* skip `K` or `M` */
69 | if (**stringPtr=='i') (*stringPtr)++;
70 | if (**stringPtr=='B') (*stringPtr)++;
71 | }
72 | return result;
73 | }
74 |
75 |
76 | /**
77 | * isCommand():
78 | * Checks if string is the same as longCommand.
79 | * If yes, @return 1, otherwise @return 0
80 | */
81 | static int isCommand(const char* string, const char* longCommand)
82 | {
83 | assert(string);
84 | assert(longCommand);
85 | size_t const comSize = strlen(longCommand);
86 | return !strncmp(string, longCommand, comSize);
87 | }
88 |
89 | /*
90 | * longCommandWArg():
91 | * Checks if *stringPtr is the same as longCommand.
92 | * If yes, @return 1 and advances *stringPtr to the position which immediately
93 | * follows longCommand.
94 | * @return 0 and doesn't modify *stringPtr otherwise.
95 | */
96 | static int longCommandWArg(const char** stringPtr, const char* longCommand)
97 | {
98 | assert(stringPtr);
99 | assert(longCommand);
100 | size_t const comSize = strlen(longCommand);
101 | int const result = isCommand(*stringPtr, longCommand);
102 | if (result) *stringPtr += comSize;
103 | return result;
104 | }
105 |
106 |
107 | /* === default values - can be redefined at compilation time === */
108 |
109 | #ifndef SMALL_SIZE_MIN_DEFAULT
110 | # define SMALL_SIZE_MIN_DEFAULT 1
111 | #endif
112 | #ifndef SMALL_SIZE_MAX_DEFAULT
113 | # define SMALL_SIZE_MAX_DEFAULT 30
114 | #endif
115 | #ifndef LARGE_SIZELOG_MIN_DEFAULT
116 | # define LARGE_SIZELOG_MIN_DEFAULT 9
117 | #endif
118 | #ifndef LARGE_SIZELOG_MAX_DEFAULT
119 | # define LARGE_SIZELOG_MAX_DEFAULT 27
120 | #endif
121 |
122 |
123 | static int display_hash_names(void)
124 | {
125 | int i;
126 | printf("available hashes : \n");
127 | for (i=0; i<NB_HASHES; i++) {
128 | printf("%s, ", hashCandidates[i].name);
129 | }
130 | printf("\b\b \n");
131 | return 0;
132 | }
133 |
134 | /*
135 | * @return: hashID (necessarily between 0 and NB_HASHES) if present
136 | * -1 on error (hname not present)
137 | */
138 | static int hashID(const char* hname)
139 | {
140 | int id;
141 | assert(hname);
142 | for (id=0; id < NB_HASHES; id++) {
143 | assert(hashCandidates[id].name);
144 | if (strlen(hname) != strlen(hashCandidates[id].name)) continue;
145 | if (isCommand(hname, hashCandidates[id].name)) return id;
146 | }
147 | return -1;
148 | }
149 |
150 | static int help(const char* exename)
151 | {
152 | printf("Usage: %s [options]... [hash]\n", exename);
153 | printf("Runs various benchmarks at various lengths for the listed hash functions\n");
154 | printf("and outputs them in a CSV format.\n\n");
155 | printf("Options: \n");
156 | printf(" --list Name available hash algorithms and exit \n");
157 | printf(" --mins=LEN Starting length for small size bench (default: %i) \n", SMALL_SIZE_MIN_DEFAULT);
158 | printf(" --maxs=LEN End length for small size bench (default: %i) \n", SMALL_SIZE_MAX_DEFAULT);
159 | printf(" --minl=LEN Starting log2(length) for large size bench (default: %i) \n", LARGE_SIZELOG_MIN_DEFAULT);
160 | printf(" --maxl=LEN End log2(length) for large size bench (default: %i) \n", LARGE_SIZELOG_MAX_DEFAULT);
161 | printf(" [hash] Optional, bench all available hashes if not provided \n");
162 | return 0;
163 | }
164 |
165 | static int badusage(const char* exename)
166 | {
167 | printf("Bad command ... \n");
168 | help(exename);
169 | return 1;
170 | }
171 |
172 | int main(int argc, const char* argv[])
173 | {
174 | const char* const exename = argv[0];
175 | int hashNb = 0;
176 | int nb_h_test = NB_HASHES;
177 | int largeTest_log_min = LARGE_SIZELOG_MIN_DEFAULT;
178 | int largeTest_log_max = LARGE_SIZELOG_MAX_DEFAULT;
179 | size_t smallTest_size_min = SMALL_SIZE_MIN_DEFAULT;
180 | size_t smallTest_size_max = SMALL_SIZE_MAX_DEFAULT;
181 |
182 | int arg_nb;
183 | for (arg_nb = 1; arg_nb < argc; arg_nb++) {
184 | const char** arg = argv + arg_nb;
185 | if (isCommand(*arg, "-h")) { assert(argc >= 1); return help(exename); }
186 | if (isCommand(*arg, "--list")) { return display_hash_names(); }
187 | if (longCommandWArg(arg, "--n=")) { nb_h_test = readIntFromChar(arg); continue; } /* hidden command */
188 | if (longCommandWArg(arg, "--minl=")) { largeTest_log_min = readIntFromChar(arg); continue; }
189 | if (longCommandWArg(arg, "--maxl=")) { largeTest_log_max = readIntFromChar(arg); continue; }
190 | if (longCommandWArg(arg, "--mins=")) { smallTest_size_min = (size_t)readIntFromChar(arg); continue; }
191 | if (longCommandWArg(arg, "--maxs=")) { smallTest_size_max = (size_t)readIntFromChar(arg); continue; }
192 | /* not a command: must be a hash name */
193 | hashNb = hashID(*arg);
194 | if (hashNb >= 0) {
195 | nb_h_test = 1;
196 | } else {
197 | /* not a hash name: error */
198 | return badusage(exename);
199 | }
200 | }
201 |
202 | /* border case (requires (mis)using hidden command `--n=#`) */
203 | if (hashNb + nb_h_test > NB_HASHES) {
204 | printf("wrong hash selection \n");
205 | return 1;
206 | }
207 |
208 | printf(" === benchmarking %i hash functions === \n", nb_h_test);
209 | if (largeTest_log_max >= largeTest_log_min) {
210 | bench_largeInput(hashCandidates+hashNb, nb_h_test, largeTest_log_min, largeTest_log_max);
211 | }
212 | if (smallTest_size_max >= smallTest_size_min) {
213 | bench_throughput_smallInputs(hashCandidates+hashNb, nb_h_test, smallTest_size_min, smallTest_size_max);
214 | bench_throughput_randomInputLength(hashCandidates+hashNb, nb_h_test, smallTest_size_min, smallTest_size_max);
215 | bench_latency_smallInputs(hashCandidates+hashNb, nb_h_test, smallTest_size_min, smallTest_size_max);
216 | bench_latency_randomInputLength(hashCandidates+hashNb, nb_h_test, smallTest_size_min, smallTest_size_max);
217 | }
218 |
219 | return 0;
220 | }
221 |
--------------------------------------------------------------------------------
/tests/bench/timefn.c:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2019-2021 Yann Collet, Facebook, Inc.
3 | * All rights reserved.
4 | *
5 | * This source code is licensed under both the BSD-style license (found in the
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 | * in the COPYING file in the root directory of this source tree).
8 | * You may select, at your option, one of the above-listed licenses.
9 | */
10 |
11 |
12 | /* === Dependencies === */
13 |
14 | #include "timefn.h"
15 |
16 |
17 | /*-****************************************
18 | * Time functions
19 | ******************************************/
20 |
21 | #if defined(_WIN32) /* Windows */
22 |
23 | #include <stdlib.h> /* abort */
24 | #include <stdio.h> /* perror */
25 |
26 | UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
27 |
28 | PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
29 | {
30 | static LARGE_INTEGER ticksPerSecond;
31 | static int init = 0;
32 | if (!init) {
33 | if (!QueryPerformanceFrequency(&ticksPerSecond)) {
34 | perror("timefn::QueryPerformanceFrequency");
35 | abort();
36 | }
37 | init = 1;
38 | }
39 | return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
40 | }
41 |
42 | PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
43 | {
44 | static LARGE_INTEGER ticksPerSecond;
45 | static int init = 0;
46 | if (!init) {
47 | if (!QueryPerformanceFrequency(&ticksPerSecond)) {
48 | perror("timefn::QueryPerformanceFrequency");
49 | abort();
50 | }
51 | init = 1;
52 | }
53 | return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
54 | }
55 |
56 |
57 |
58 | #elif defined(__APPLE__) && defined(__MACH__)
59 |
60 | UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
61 |
62 | PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
63 | {
64 | static mach_timebase_info_data_t rate;
65 | static int init = 0;
66 | if (!init) {
67 | mach_timebase_info(&rate);
68 | init = 1;
69 | }
70 | return (((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom))/1000ULL;
71 | }
72 |
73 | PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
74 | {
75 | static mach_timebase_info_data_t rate;
76 | static int init = 0;
77 | if (!init) {
78 | mach_timebase_info(&rate);
79 | init = 1;
80 | }
81 | return ((clockEnd - clockStart) * (PTime)rate.numer) / ((PTime)rate.denom);
82 | }
83 |
84 |
85 |
86 | #elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
87 | && defined(TIME_UTC) /* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance */
88 |
89 | #include <stdlib.h> /* abort */
90 | #include <stdio.h> /* perror */
91 |
92 | UTIL_time_t UTIL_getTime(void)
93 | {
94 | /* time must be initialized, othersize it may fail msan test.
95 | * No good reason, likely a limitation of timespec_get() for some target */
96 | UTIL_time_t time = UTIL_TIME_INITIALIZER;
97 | if (timespec_get(&time, TIME_UTC) != TIME_UTC) {
98 | perror("timefn::timespec_get");
99 | abort();
100 | }
101 | return time;
102 | }
103 |
104 | static UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
105 | {
106 | UTIL_time_t diff;
107 | if (end.tv_nsec < begin.tv_nsec) {
108 | diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
109 | diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
110 | } else {
111 | diff.tv_sec = end.tv_sec - begin.tv_sec;
112 | diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
113 | }
114 | return diff;
115 | }
116 |
117 | PTime UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
118 | {
119 | UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
120 | PTime micro = 0;
121 | micro += 1000000ULL * diff.tv_sec;
122 | micro += diff.tv_nsec / 1000ULL;
123 | return micro;
124 | }
125 |
126 | PTime UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
127 | {
128 | UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
129 | PTime nano = 0;
130 | nano += 1000000000ULL * diff.tv_sec;
131 | nano += diff.tv_nsec;
132 | return nano;
133 | }
134 |
135 |
136 |
137 | #else /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
138 |
139 | UTIL_time_t UTIL_getTime(void) { return clock(); }
140 | PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
141 | PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
142 |
143 | #endif
144 |
145 |
146 |
147 | /* returns time span in microseconds */
148 | PTime UTIL_clockSpanMicro(UTIL_time_t clockStart )
149 | {
150 | UTIL_time_t const clockEnd = UTIL_getTime();
151 | return UTIL_getSpanTimeMicro(clockStart, clockEnd);
152 | }
153 |
154 | /* returns time span in microseconds */
155 | PTime UTIL_clockSpanNano(UTIL_time_t clockStart )
156 | {
157 | UTIL_time_t const clockEnd = UTIL_getTime();
158 | return UTIL_getSpanTimeNano(clockStart, clockEnd);
159 | }
160 |
161 | void UTIL_waitForNextTick(void)
162 | {
163 | UTIL_time_t const clockStart = UTIL_getTime();
164 | UTIL_time_t clockEnd;
165 | do {
166 | clockEnd = UTIL_getTime();
167 | } while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
168 | }
169 |
--------------------------------------------------------------------------------
/tests/bench/timefn.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
3 | * All rights reserved.
4 | *
5 | * This source code is licensed under both the BSD-style license (found in the
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 | * in the COPYING file in the root directory of this source tree).
8 | * You may select, at your option, one of the above-listed licenses.
9 | */
10 |
11 | #ifndef TIME_FN_H_MODULE_287987
12 | #define TIME_FN_H_MODULE_287987
13 |
14 | #if defined (__cplusplus)
15 | extern "C" {
16 | #endif
17 |
18 |
19 | /*-****************************************
20 | * Dependencies
21 | ******************************************/
22 | #include <sys/types.h> /* utime */
23 | #if defined(_MSC_VER)
24 | # include <sys/utime.h> /* utime */
25 | #else
26 | # include <utime.h> /* utime */
27 | #endif
28 | #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */
29 |
30 |
31 |
32 | /*-****************************************
33 | * Local Types
34 | ******************************************/
35 |
36 | #if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
37 | # include <stdint.h>
38 | typedef uint64_t PTime; /* Precise Time */
39 | #else
40 | typedef unsigned long long PTime; /* does not support compilers without long long support */
41 | #endif
42 |
43 |
44 |
45 | /*-****************************************
46 | * Time functions
47 | ******************************************/
48 | #if defined(_WIN32) /* Windows */
49 |
50 | #include <Windows.h> /* LARGE_INTEGER */
51 | typedef LARGE_INTEGER UTIL_time_t;
52 | #define UTIL_TIME_INITIALIZER { { 0, 0 } }
53 |
54 | #elif defined(__APPLE__) && defined(__MACH__)
55 |
56 | #include <mach/mach_time.h>
57 | typedef PTime UTIL_time_t;
58 | #define UTIL_TIME_INITIALIZER 0
59 |
60 | #elif (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */) \
61 | && defined(TIME_UTC) /* C11 requires timespec_get, but FreeBSD 11 lacks it, while still claiming C11 compliance */
62 |
63 | typedef struct timespec UTIL_time_t;
64 | #define UTIL_TIME_INITIALIZER { 0, 0 }
65 |
66 | #else /* relies on standard C90 (note : clock_t measurements can be wrong when using multi-threading) */
67 |
68 | typedef clock_t UTIL_time_t;
69 | #define UTIL_TIME_INITIALIZER 0
70 |
71 | #endif
72 |
73 |
74 | UTIL_time_t UTIL_getTime(void);
75 | PTime UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd);
76 | PTime UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd);
77 |
78 | #define SEC_TO_MICRO ((PTime)1000000)
79 | PTime UTIL_clockSpanMicro(UTIL_time_t clockStart);
80 | PTime UTIL_clockSpanNano(UTIL_time_t clockStart);
81 |
82 | void UTIL_waitForNextTick(void);
83 |
84 |
85 | #if defined (__cplusplus)
86 | }
87 | #endif
88 |
89 | #endif /* TIME_FN_H_MODULE_287987 */
90 |
--------------------------------------------------------------------------------
/tests/cli-comment-line.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Exit immediately if any command fails.
4 | # https://stackoverflow.com/a/2871034
5 | set -euxo
6 |
7 |
8 | # Default
9 | ./xxhsum ./Makefile > ./.test.xxh
10 | echo '# Test comment line' | cat - ./.test.xxh > temp && mv temp ./.test.xxh
11 | ./xxhsum --check ./.test.xxh
12 |
13 | # XXH32
14 | ./xxhsum -H32 ./Makefile > ./.test.xxh32
15 | echo '# Test comment line' | cat - ./.test.xxh32 > temp && mv temp ./.test.xxh32
16 | ./xxhsum --check ./.test.xxh32
17 |
18 | # XXH64
19 | ./xxhsum -H64 ./Makefile > ./.test.xxh64
20 | echo '# Test comment line' | cat - ./.test.xxh64 > temp && mv temp ./.test.xxh64
21 | ./xxhsum --check ./.test.xxh64
22 |
23 | # XXH128
24 | ./xxhsum -H128 ./Makefile > ./.test.xxh128
25 | echo '# Test comment line' | cat - ./.test.xxh128 > temp && mv temp ./.test.xxh128
26 | ./xxhsum --check ./.test.xxh128
27 |
28 |
29 | rm ./.test.xxh
30 | rm ./.test.xxh32
31 | rm ./.test.xxh64
32 | rm ./.test.xxh128
33 |
--------------------------------------------------------------------------------
/tests/cli-ignore-missing.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Exit immediately if any command fails.
4 | # https://stackoverflow.com/a/2871034
5 | set -e -u -x
6 |
7 |
8 | # Normal
9 | ./xxhsum ./Makefile > ./.test.xxh
10 | ./xxhsum --check ./.test.xxh
11 |
12 |
13 | # Missing, expect error
14 | # (1) Create checksum file.
15 | # (2) Remove one of them.
16 | # (3) --check it
17 | # (4) Expect NG (missing file)
18 | cp Makefile .test.makefile
19 | ./xxhsum ./.test.makefile > ./.test.xxh
20 | rm ./.test.makefile
21 | ! ./xxhsum --check ./.test.xxh # Put '!' for expecting error
22 |
23 |
24 | # Missing, --ignore-missing
25 | # (1) Create checksum file.
26 | # (2) Remove one of them.
27 | # (3) --check it with --ignore-missing.
28 | # (4) Expect OK
29 |
30 | cp Makefile .test.makefile
31 | ./xxhsum Makefile ./.test.makefile > ./.test.xxh
32 | rm ./.test.makefile
33 | ./xxhsum --check --ignore-missing ./.test.xxh
34 |
35 |
36 | # Missing, --ignore-missing, expect error
37 | # (1) Create checksum file.
38 | # (2) Remove all of them.
39 | # (3) --check it with --ignore-missing.
40 | # (4) Expect NG (no file was verified).
41 |
42 | cp Makefile .test.makefile
43 | ./xxhsum ./.test.makefile > ./.test.xxh
44 | rm ./.test.makefile
45 | ! ./xxhsum --check --ignore-missing ./.test.xxh # Put '!' for expecting error
46 |
47 |
48 | # Cleanup
49 | ( rm ./.test.* ) || true
50 |
51 | echo OK
52 |
--------------------------------------------------------------------------------
/tests/collisions/.gitignore:
--------------------------------------------------------------------------------
1 | #build artefacts
2 | collisionsTest
3 |
--------------------------------------------------------------------------------
/tests/collisions/Makefile:
--------------------------------------------------------------------------------
1 | # Brute force collision tester for 64-bit hashes
2 | # Part of xxHash project
3 | # Copyright (C) 2019-2021 Yann Collet
4 | #
5 | # GPL v2 License
6 | #
7 | # This program is free software; you can redistribute it and/or modify
8 | # it under the terms of the GNU General Public License as published by
9 | # the Free Software Foundation; either version 2 of the License, or
10 | # (at your option) any later version.
11 | #
12 | # This program is distributed in the hope that it will be useful,
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | # GNU General Public License for more details.
16 | #
17 | # You should have received a copy of the GNU General Public License along
18 | # with this program; if not, write to the Free Software Foundation, Inc.,
19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20 | #
21 | # You can contact the author at:
22 | # - xxHash homepage: https://www.xxhash.com
23 | # - xxHash source repository: https://github.com/Cyan4973/xxHash
24 | #
25 |
26 | HEADER_DIRS = ./ ../../ allcodecs/
27 | CPPFLAGS += $(addprefix -I ,$(HEADER_DIRS))
28 | CFLAGS += -Wall -Wextra -Wconversion \
29 | -std=c11
30 | CXXFLAGS += -Wall -Wextra -Wconversion \
31 | -std=c++11
32 | LDFLAGS += -pthread
33 | TESTHASHES = 3200000
34 |
35 | HASH_SRC := $(wildcard allcodecs/*.c) $(wildcard allcodecs/*.cc) $(wildcard allcodecs/*.cpp)
36 | HASH_OBJ := $(addsuffix .o,$(basename $(HASH_SRC)))
37 |
38 | .PHONY: default
39 | default: collisionsTest
40 |
41 | C_SRCDIRS = $(shell find allcodecs -type d)
42 | CXX_SRCDIRS = $(shell find allcodecs -type d)
43 | include ../../build/make/multiconf.make
44 |
45 | .PHONY: all
46 | all: collisionsTest
47 |
48 | collisionsTest: CXXFLAGS := -O3 $(CXXFLAGS)
49 | collisionsTest: CFLAGS := -O3 $(CFLAGS)
50 | $(eval $(call cxx_program,collisionsTest,main.o pool.o threading.o sort.o $(HASH_OBJ)))
51 |
52 | .PHONY: debug
53 | debug: CPPFLAGS += -DDEBUG -DXXH_NO_INLINE_HINTS
54 | debug:
55 | CFLAGS='$(CFLAGS) -g3 -Og' CXXFLAGS='$(CXXFLAGS) -g3 -Og' CPPFLAGS='$(CPPFLAGS)' $(MAKE) collisionsTest
56 |
57 | .PHONY: check
58 | check: test
59 |
60 | .PHONY: test
61 | test: debug
62 | @echo ""
63 | @echo "## $(TESTHASHES) hashes"
64 | @time ./collisionsTest --nbh=$(TESTHASHES)
65 | @echo ""
66 | @echo "## $(TESTHASHES) hashes with filter"
67 | @time ./collisionsTest --nbh=$(TESTHASHES) --filter
68 | @echo ""
69 | @echo "## $(TESTHASHES) hashes with 2 threads"
70 | @time ./collisionsTest --nbh=$(TESTHASHES) --threadlog=1
71 | @echo ""
72 |
73 | .PHONY: clean
74 | clean:
75 | $(RM) *.o allcodecs/*.o
76 |
--------------------------------------------------------------------------------
/tests/collisions/README.md:
--------------------------------------------------------------------------------
1 |
2 | __collisionsTest__ is a brute force hash analyzer
3 | which will measure a 64-bit hash algorithm's collision rate
4 | by generating billions of hashes,
5 | and comparing the result to an "ideal" target.
6 |
7 | The test requires a very large amount of memory.
8 | By default, it will generate 24 billion of 64-bit hashes,
9 | requiring __192 GB of RAM__ for their storage.
10 | The number of hashes can be modified using command `--nbh=`.
11 | Be aware that testing the collision ratio of 64-bit hashes
12 | requires a very large amount of hashes (several billion) for meaningful measurements.
13 |
14 | To reduce RAM usage, an optional filter can be requested, with `--filter`.
15 | It reduces the nb of candidates to analyze, hence associated RAM budget.
16 | Note that the filter itself requires a lot of RAM
17 | (32 GB by default, can be modified using `--filterlog=`,
18 | a too small filter will not be efficient, aim at ~2 bytes per hash),
19 | and reading and writing into filter cost a significant CPU budget,
20 | so this method is slower.
21 | It also doesn't allow advanced analysis of partial bitfields,
22 | since most hashes will be discarded and not stored.
23 |
24 | When using the filter, the RAM budget consists of the filter and a list of candidates,
25 | which will be a fraction of the original hash list.
26 | Using default settings (24 billion hashes, 32 GB filter),
27 | the number of potential candidates should be reduced to less than 2 billion,
28 | requiring ~14 GB for their storage.
29 | Such a result also depends on hash algorithm's efficiency.
30 | The number of effective candidates is likely to be lower, at ~ 1 billion,
31 | but storage must allocate an upper bound.
32 |
33 | For the default test, the expected "optimal" collision rate for a 64-bit hash function is ~18 collisions.
34 |
35 | #### How to build
36 | ```
37 | make
38 | ```
39 |
40 | Note: the code is a mix of C99 and C++14,
41 | it's not compatible with a C90-only compiler.
42 |
43 | #### Build modifier
44 |
45 | - `SLAB5`: use alternative pattern generator, friendlier for weak hash algorithms
46 | - `POOL_MT`: if `=0`, disable multi-threading code (enabled by default)
47 |
48 | #### How to integrate any hash in the tester
49 |
50 | The build script will compile files found in `./allcodecs`.
51 | Put the source code here.
52 | This also works if the hash is a single `*.h` file.
53 |
54 | The glue happens in `hashes.h`.
55 | In this file, there are 2 sections:
56 | - Adds the required `#include "header.h"`, and creates a wrapper
57 | to respect the format expected by the function pointer.
58 | - Adds the wrapper, along with the name and an indication of the output width,
59 | to the table, at the end of `hashes.h`
60 |
61 | Build with `make`. Locate your new hash with `./collisionsTest -h`,
62 | it should be listed.
63 |
64 |
65 | #### Usage
66 |
67 | ```
68 | usage: ./collisionsTest [hashName] [opt]
69 |
70 | list of hashNames: (...)
71 |
72 | Optional parameters:
73 | --nbh=NB Select nb of hashes to generate (25769803776 by default)
74 | --filter Enable the filter. Slower, but reduces memory usage for same nb of hashes.
75 | --threadlog=NB Use 2^NB threads
76 | --len=NB Select length of input (255 bytes by default)
77 | ```
78 |
79 | #### Some advises on how to setup a collisions test
80 |
81 | Most tests are primarily driven by the amount of RAM available.
82 | Here's a method to decide the size of the test.
83 |
84 | Presuming that RAM budget is not plentiful, for this example 32 GB,
85 | the `--filter` mode is actually compulsory to measure anything meaningful.
86 | Let's plan 50% of memory for the filter, that's 16 GB.
87 | This will be good enough to filter about 10% less hashes than this size.
88 | Let's round down to 14 G.
89 |
90 | By requesting 14G, the expectation is that the program will automatically
91 | size the filter to 16 GB, and expect to store ~1G candidates,
92 | leaving enough room to breeze for the system.
93 |
94 | The command line becomes:
95 | ```
96 | ./collisionsTest --nbh=14G --filter NameOfHash
97 | ```
98 |
99 | #### Examples:
100 |
101 | Here are a few results produced with this tester:
102 |
103 | | Algorithm | Input Len | Nb Hashes | Expected | Nb Collisions | Notes |
104 | | --- | --- | --- | --- | --- | --- |
105 | | __XXH3__ | 255 | 100 Gi | 312.5 | 326 | |
106 | | __XXH64__ | 255 | 100 Gi | 312.5 | 294 | |
107 | | __XXH128__ low64 | 512 | 100 Gi | 312.5 | 321 | |
108 | | __XXH128__ high64| 512 | 100 Gi | 312.5 | 325 | |
109 | | __XXH128__ | 255 | 100 Gi | 0.0 | 0 | a 128-bit hash is expected to generate 0 collisions |
110 |
111 | Test on small inputs:
112 |
113 | | Algorithm | Input Len | Nb Hashes | Expected | Nb Collisions | Notes |
114 | | --- | --- | --- | --- | --- | --- |
115 | | __XXH64__ | 8 | 100 Gi | 312.5 | __0__ | `XXH64` is bijective for `len==8` |
116 | | __XXH3__ | 8 | 100 Gi | 312.5 | __0__ | `XXH3` is also bijective for `len==8` |
117 | | __XXH3__ | 16 | 100 Gi | 312.5 | 332 | |
118 | | __XXH3__ | 32 | 14 Gi | 6.1 | 3 | |
119 | | __XXH128__ | 16 | 25 Gi | 0.0 | 0 | test range 9-16 |
120 | | __XXH128__ | 32 | 25 Gi | 0.0 | 0 | test range 17-128 |
121 | | __XXH128__ | 100 | 13 Gi | 0.0 | 0 | test range 17-128 |
122 | | __XXH128__ | 200 | 13 Gi | 0.0 | 0 | test range 129-240 |
123 |
--------------------------------------------------------------------------------
/tests/collisions/allcodecs/README.md:
--------------------------------------------------------------------------------
1 | Put in this directory all hash algorithms to test
2 |
--------------------------------------------------------------------------------
/tests/collisions/allcodecs/dummy.c:
--------------------------------------------------------------------------------
1 | /*
2 | * dummy.c, a fake hash algorithm, just to test integration capabilities.
3 | * Part of the xxHash project
4 | * Copyright (C) 2020 Yann Collet
5 | *
6 | * GPL v2 License
7 | *
8 | * This program is free software; you can redistribute it and/or modify
9 | * it under the terms of the GNU General Public License as published by
10 | * the Free Software Foundation; either version 2 of the License, or
11 | * (at your option) any later version.
12 | *
13 | * This program is distributed in the hope that it will be useful,
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 | * GNU General Public License for more details.
17 | *
18 | * You should have received a copy of the GNU General Public License along
19 | * with this program; if not, write to the Free Software Foundation, Inc.,
20 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 | *
22 | * You can contact the author at:
23 | * - xxHash homepage: https://www.xxhash.com
24 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
25 | */
26 |
27 |
28 | #include <dummy.h>
29 |
30 | unsigned badsum32(const void* input, size_t len, unsigned seed)
31 | {
32 | unsigned sum = seed;
33 | const unsigned char* in8 = input;
34 | size_t c;
35 | for (c=0; c<len; c++)
36 | sum += in8[c];
37 | return sum;
38 | }
39 |
--------------------------------------------------------------------------------
/tests/collisions/allcodecs/dummy.h:
--------------------------------------------------------------------------------
1 | /*
2 | * dummy.c,
3 | * A fake hash algorithm, just to test integration capabilities.
4 | * Part of the xxHash project
5 | * Copyright (C) 2020 Yann Collet
6 | *
7 | * GPL v2 License
8 | *
9 | * This program is free software; you can redistribute it and/or modify
10 | * it under the terms of the GNU General Public License as published by
11 | * the Free Software Foundation; either version 2 of the License, or
12 | * (at your option) any later version.
13 | *
14 | * This program is distributed in the hope that it will be useful,
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 | * GNU General Public License for more details.
18 | *
19 | * You should have received a copy of the GNU General Public License along
20 | * with this program; if not, write to the Free Software Foundation, Inc.,
21 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22 | *
23 | * You can contact the author at:
24 | * - xxHash homepage: https://www.xxhash.com
25 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
26 | */
27 |
28 | #ifndef DUMMY_H_987987
29 | #define DUMMY_H_987987
30 |
31 | #if defined (__cplusplus)
32 | extern "C" {
33 | #endif
34 |
35 |
36 | #include <stddef.h> /* size_t */
37 |
38 | unsigned badsum32(const void* input, size_t len, unsigned seed);
39 |
40 |
41 | #if defined (__cplusplus)
42 | }
43 | #endif
44 |
45 | #endif /* DUMMY_H_987987 */
46 |
--------------------------------------------------------------------------------
/tests/collisions/hashes.h:
--------------------------------------------------------------------------------
1 | /*
2 | * List of hashes for the brute force collision tester
3 | * Part of xxHash project
4 | * Copyright (C) 2019-2021 Yann Collet
5 | *
6 | * GPL v2 License
7 | *
8 | * This program is free software; you can redistribute it and/or modify
9 | * it under the terms of the GNU General Public License as published by
10 | * the Free Software Foundation; either version 2 of the License, or
11 | * (at your option) any later version.
12 | *
13 | * This program is distributed in the hope that it will be useful,
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 | * GNU General Public License for more details.
17 | *
18 | * You should have received a copy of the GNU General Public License along
19 | * with this program; if not, write to the Free Software Foundation, Inc.,
20 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21 | *
22 | * You can contact the author at:
23 | * - xxHash homepage: https://www.xxhash.com
24 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
25 | */
26 |
27 | #ifndef HASHES_H_1235465
28 | #define HASHES_H_1235465
29 |
30 | #include <stddef.h> /* size_t */
31 | #include <stdint.h> /* uint64_t */
32 | #define XXH_INLINE_ALL /* XXH128_hash_t */
33 | #include "xxhash.h"
34 |
35 |
36 | /* return type */
37 |
38 | typedef union {
39 | uint64_t h64;
40 | XXH128_hash_t h128;
41 | } UniHash;
42 |
43 | UniHash uniHash32(uint64_t v32)
44 | { UniHash unih;
45 | unih.h64 = v32;
46 | return unih;
47 | }
48 |
49 | UniHash uniHash64(uint64_t v64)
50 | { UniHash unih;
51 | unih.h64 = v64;
52 | return unih;
53 | }
54 |
55 | UniHash uniHash128(XXH128_hash_t v128)
56 | { UniHash unih;
57 | unih.h128 = v128;
58 | return unih;
59 | }
60 |
61 |
62 | /* === xxHash === */
63 |
64 | UniHash XXH3_wrapper (const void* data, size_t size)
65 | {
66 | return uniHash64( XXH3_64bits(data, size) );
67 | }
68 |
69 | UniHash XXH128_wrapper (const void* data, size_t size)
70 | {
71 | return uniHash128( XXH3_128bits(data, size) );
72 | }
73 |
74 | UniHash XXH128l_wrapper (const void* data, size_t size)
75 | {
76 | return uniHash64( XXH3_128bits(data, size).low64 );
77 | }
78 |
79 | UniHash XXH128h_wrapper (const void* data, size_t size)
80 | {
81 | return uniHash64( XXH3_128bits(data, size).high64 );
82 | }
83 |
84 | UniHash XXH64_wrapper (const void* data, size_t size)
85 | {
86 | return uniHash64 ( XXH64(data, size, 0) );
87 | }
88 |
89 | UniHash XXH32_wrapper (const void* data, size_t size)
90 | {
91 | return uniHash32( XXH32(data, size, 0) );
92 | }
93 |
94 | /* === Dummy integration example === */
95 |
96 | #include "dummy.h"
97 |
98 | UniHash badsum32_wrapper (const void* data, size_t size)
99 | {
100 | return uniHash32( badsum32(data, size, 0) );
101 | }
102 |
103 |
104 |
105 | /* === Table === */
106 |
107 | typedef UniHash (*hashfn) (const void* data, size_t size);
108 |
109 | typedef struct {
110 | const char* name;
111 | hashfn fn;
112 | int bits;
113 | } hashDescription;
114 |
115 | #define HASH_FN_TOTAL 7
116 |
117 | hashDescription hashfnTable[HASH_FN_TOTAL] = {
118 | { "xxh3" , XXH3_wrapper, 64 },
119 | { "xxh64" , XXH64_wrapper, 64 },
120 | { "xxh128", XXH128_wrapper, 128 },
121 | { "xxh128l", XXH128l_wrapper, 64 },
122 | { "xxh128h", XXH128h_wrapper, 64 },
123 | { "xxh32" , XXH32_wrapper, 32 },
124 | { "badsum32",badsum32_wrapper, 32 },
125 | };
126 |
127 | #endif /* HASHES_H_1235465 */
128 |
--------------------------------------------------------------------------------
/tests/collisions/pool.c:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (C) 2016-2021 Yann Collet, Facebook, Inc.
3 | * All rights reserved.
4 | *
5 | * This source code is licensed under both the BSD-style license (found in the
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 | * in the COPYING file in the root directory of this source tree).
8 | * You may select, at your option, one of the above-listed licenses.
9 | */
10 |
11 |
12 | /* ====== Dependencies ======= */
13 | #include <stddef.h> /* size_t */
14 | #include <stdlib.h> /* malloc, calloc, free */
15 | #include <string.h> /* memcpy */
16 | #include <assert.h>
17 |
18 | #include "pool.h"
19 |
20 |
21 | /* ====== Compiler specifics ====== */
22 | #if defined(_MSC_VER)
23 | # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
24 | #endif
25 |
26 |
27 | /* === Build Macro === */
28 |
29 | #ifndef POOL_MT // can be defined on command line
30 | # define POOL_MT 1
31 | #endif
32 |
33 |
34 | /* === Implementation === */
35 |
36 | #if POOL_MT
37 |
38 | #include "threading.h" /* pthread adaptation */
39 |
40 | /* A job is a function and an opaque argument */
41 | typedef struct POOL_job_s {
42 | POOL_function function;
43 | void *opaque;
44 | } POOL_job;
45 |
46 | struct POOL_ctx_s {
47 | /* Keep track of the threads */
48 | ZSTD_pthread_t* threads;
49 | size_t threadCapacity;
50 | size_t threadLimit;
51 |
52 | /* The queue is a circular buffer */
53 | POOL_job *queue;
54 | size_t queueHead;
55 | size_t queueTail;
56 | size_t queueSize;
57 |
58 | /* The number of threads working on jobs */
59 | size_t numThreadsBusy;
60 | /* Indicates if the queue is empty */
61 | int queueEmpty;
62 |
63 | /* The mutex protects the queue */
64 | ZSTD_pthread_mutex_t queueMutex;
65 | /* Condition variable for pushers to wait on when the queue is full */
66 | ZSTD_pthread_cond_t queuePushCond;
67 | /* Condition variables for poppers to wait on when the queue is empty */
68 | ZSTD_pthread_cond_t queuePopCond;
69 | /* Indicates if the queue is shutting down */
70 | int shutdown;
71 | };
72 |
73 | /* POOL_thread() :
74 | * Work thread for the thread pool.
75 | * Waits for jobs and executes them.
76 | * @returns : NULL on failure else non-null.
77 | */
78 | static void* POOL_thread(void* opaque)
79 | {
80 | POOL_ctx* const ctx = (POOL_ctx*)opaque;
81 | if (!ctx) { return NULL; }
82 | for (;;) {
83 | /* Lock the mutex and wait for a non-empty queue or until shutdown */
84 | ZSTD_pthread_mutex_lock(&ctx->queueMutex);
85 |
86 | while ( ctx->queueEmpty
87 | || (ctx->numThreadsBusy >= ctx->threadLimit) ) {
88 | if (ctx->shutdown) {
89 | /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit),
90 | * a few threads will be shutdown while !queueEmpty,
91 | * but enough threads will remain active to finish the queue */
92 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
93 | return opaque;
94 | }
95 | ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
96 | }
97 | /* Pop a job off the queue */
98 | { POOL_job const job = ctx->queue[ctx->queueHead];
99 | ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
100 | ctx->numThreadsBusy++;
101 | ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
102 | /* Unlock the mutex, signal a pusher, and run the job */
103 | ZSTD_pthread_cond_signal(&ctx->queuePushCond);
104 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
105 |
106 | job.function(job.opaque);
107 |
108 | /* If the intended queue size was 0, signal after finishing job */
109 | ZSTD_pthread_mutex_lock(&ctx->queueMutex);
110 | ctx->numThreadsBusy--;
111 | if (ctx->queueSize == 1) {
112 | ZSTD_pthread_cond_signal(&ctx->queuePushCond);
113 | }
114 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
115 | }
116 | } /* for (;;) */
117 | assert(0); /* Unreachable */
118 | }
119 |
120 | POOL_ctx* POOL_create(size_t numThreads, size_t queueSize)
121 | {
122 | POOL_ctx* ctx;
123 | /* Check parameters */
124 | if (!numThreads) { return NULL; }
125 | /* Allocate the context and zero initialize */
126 | ctx = (POOL_ctx*)calloc(1, sizeof(POOL_ctx));
127 | if (!ctx) { return NULL; }
128 | /* Initialize the job queue.
129 | * It needs one extra space since one space is wasted to differentiate
130 | * empty and full queues.
131 | */
132 | ctx->queueSize = queueSize + 1;
133 | ctx->queue = (POOL_job*)malloc(ctx->queueSize * sizeof(POOL_job));
134 | ctx->queueHead = 0;
135 | ctx->queueTail = 0;
136 | ctx->numThreadsBusy = 0;
137 | ctx->queueEmpty = 1;
138 | (void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
139 | (void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
140 | (void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
141 | ctx->shutdown = 0;
142 | /* Allocate space for the thread handles */
143 | ctx->threads = (ZSTD_pthread_t*)malloc(numThreads * sizeof(ZSTD_pthread_t));
144 | ctx->threadCapacity = 0;
145 | /* Check for errors */
146 | if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
147 | /* Initialize the threads */
148 | { size_t i;
149 | for (i = 0; i < numThreads; ++i) {
150 | if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
151 | ctx->threadCapacity = i;
152 | POOL_free(ctx);
153 | return NULL;
154 | } }
155 | ctx->threadCapacity = numThreads;
156 | ctx->threadLimit = numThreads;
157 | }
158 | return ctx;
159 | }
160 |
161 | /*! POOL_join() :
162 | Shutdown the queue, wake any sleeping threads, and join all of the threads.
163 | */
164 | static void POOL_join(POOL_ctx* ctx) {
165 | /* Shut down the queue */
166 | ZSTD_pthread_mutex_lock(&ctx->queueMutex);
167 | ctx->shutdown = 1;
168 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
169 |
170 | /* Wake up sleeping threads */
171 | ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
172 | ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
173 |
174 | /* Join all of the threads */
175 | { size_t i;
176 | for (i = 0; i < ctx->threadCapacity; ++i) {
177 | ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */
178 | } }
179 | }
180 |
181 | void POOL_free(POOL_ctx *ctx) {
182 | if (!ctx) { return; }
183 | POOL_join(ctx);
184 | ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
185 | ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
186 | ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
187 | free(ctx->queue);
188 | free(ctx->threads);
189 | free(ctx);
190 | }
191 |
192 |
193 |
194 | size_t POOL_sizeof(POOL_ctx *ctx) {
195 | if (ctx==NULL) return 0; /* supports sizeof NULL */
196 | return sizeof(*ctx)
197 | + ctx->queueSize * sizeof(POOL_job)
198 | + ctx->threadCapacity * sizeof(ZSTD_pthread_t);
199 | }
200 |
201 |
202 | /* @return : 0 on success, 1 on error */
203 | static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
204 | {
205 | if (numThreads <= ctx->threadCapacity) {
206 | if (!numThreads) return 1;
207 | ctx->threadLimit = numThreads;
208 | return 0;
209 | }
210 | /* numThreads > threadCapacity */
211 | { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)malloc(numThreads * sizeof(ZSTD_pthread_t));
212 | if (!threadPool) return 1;
213 | /* replace existing thread pool */
214 | memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
215 | free(ctx->threads);
216 | ctx->threads = threadPool;
217 | /* Initialize additional threads */
218 | { size_t threadId;
219 | for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
220 | if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
221 | ctx->threadCapacity = threadId;
222 | return 1;
223 | } }
224 | } }
225 | /* successfully expanded */
226 | ctx->threadCapacity = numThreads;
227 | ctx->threadLimit = numThreads;
228 | return 0;
229 | }
230 |
231 | /* @return : 0 on success, 1 on error */
232 | int POOL_resize(POOL_ctx* ctx, size_t numThreads)
233 | {
234 | int result;
235 | if (ctx==NULL) return 1;
236 | ZSTD_pthread_mutex_lock(&ctx->queueMutex);
237 | result = POOL_resize_internal(ctx, numThreads);
238 | ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
239 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
240 | return result;
241 | }
242 |
243 | /**
244 | * Returns 1 if the queue is full and 0 otherwise.
245 | *
246 | * When queueSize is 1 (pool was created with an intended queueSize of 0),
247 | * then a queue is empty if there is a thread free _and_ no job is waiting.
248 | */
249 | static int isQueueFull(POOL_ctx const* ctx) {
250 | if (ctx->queueSize > 1) {
251 | return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
252 | } else {
253 | return (ctx->numThreadsBusy == ctx->threadLimit) ||
254 | !ctx->queueEmpty;
255 | }
256 | }
257 |
258 |
259 | static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
260 | {
261 | POOL_job const job = {function, opaque};
262 | assert(ctx != NULL);
263 | if (ctx->shutdown) return;
264 |
265 | ctx->queueEmpty = 0;
266 | ctx->queue[ctx->queueTail] = job;
267 | ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
268 | ZSTD_pthread_cond_signal(&ctx->queuePopCond);
269 | }
270 |
271 | void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque)
272 | {
273 | assert(ctx != NULL);
274 | ZSTD_pthread_mutex_lock(&ctx->queueMutex);
275 | /* Wait until there is space in the queue for the new job */
276 | while (isQueueFull(ctx) && (!ctx->shutdown)) {
277 | ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
278 | }
279 | POOL_add_internal(ctx, function, opaque);
280 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
281 | }
282 |
283 |
284 | int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque)
285 | {
286 | assert(ctx != NULL);
287 | ZSTD_pthread_mutex_lock(&ctx->queueMutex);
288 | if (isQueueFull(ctx)) {
289 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
290 | return 0;
291 | }
292 | POOL_add_internal(ctx, function, opaque);
293 | ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
294 | return 1;
295 | }
296 |
297 |
298 | #else /* POOL_MT not defined */
299 |
300 | /* ========================== */
301 | /* No multi-threading support */
302 | /* ========================== */
303 |
304 |
305 | /* We don't need any data, but if it is empty, malloc() might return NULL. */
306 | struct POOL_ctx_s {
307 | int dummy;
308 | };
309 | static POOL_ctx g_ctx;
310 |
311 | POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
312 | (void)numThreads;
313 | (void)queueSize;
314 | return &g_ctx;
315 | }
316 |
317 | void POOL_free(POOL_ctx* ctx) {
318 | assert(!ctx || ctx == &g_ctx);
319 | (void)ctx;
320 | }
321 |
322 | int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
323 | (void)ctx; (void)numThreads;
324 | return 0;
325 | }
326 |
327 | void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
328 | (void)ctx;
329 | function(opaque);
330 | }
331 |
332 | int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
333 | (void)ctx;
334 | function(opaque);
335 | return 1;
336 | }
337 |
338 | size_t POOL_sizeof(POOL_ctx* ctx) {
339 | if (ctx==NULL) return 0; /* supports sizeof NULL */
340 | assert(ctx == &g_ctx);
341 | return sizeof(*ctx);
342 | }
343 |
344 | #endif /* ZSTD_MULTITHREAD */
345 |
--------------------------------------------------------------------------------
/tests/collisions/pool.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
3 | * All rights reserved.
4 | *
5 | * This source code is licensed under both the BSD-style license (found in the
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 | * in the COPYING file in the root directory of this source tree).
8 | * You may select, at your option, one of the above-listed licenses.
9 | */
10 |
11 | #ifndef POOL_H
12 | #define POOL_H
13 |
14 | #if defined (__cplusplus)
15 | extern "C" {
16 | #endif
17 |
18 |
19 | #include <stddef.h> /* size_t */
20 |
21 | typedef struct POOL_ctx_s POOL_ctx;
22 |
23 | /*! POOL_create() :
24 | * Create a thread pool with at most `numThreads` threads.
25 | * `numThreads` must be at least 1.
26 | * The maximum number of queued jobs before blocking is `queueSize`.
27 | * @return : POOL_ctx pointer on success, else NULL.
28 | */
29 | POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
30 |
31 | /*! POOL_free() :
32 | * Free a thread pool returned by POOL_create().
33 | */
34 | void POOL_free(POOL_ctx* ctx);
35 |
36 | /*! POOL_resize() :
37 | * Expands or shrinks pool's number of threads.
38 | * This is more efficient than releasing + creating a new context,
39 | * since it tries to preserve and re-use existing threads.
40 | * `numThreads` must be at least 1.
41 | * @return : 0 when resize was successful,
42 | * !0 (typically 1) if there is an error.
43 | * note : only numThreads can be resized, queueSize remains unchanged.
44 | */
45 | int POOL_resize(POOL_ctx* ctx, size_t numThreads);
46 |
47 | /*! POOL_sizeof() :
48 | * @return threadpool memory usage
49 | * note : compatible with NULL (returns 0 in this case)
50 | */
51 | size_t POOL_sizeof(POOL_ctx* ctx);
52 |
53 | /*! POOL_function :
54 | * The function type that can be added to a thread pool.
55 | */
56 | typedef void (*POOL_function)(void*);
57 |
58 | /*! POOL_add() :
59 | * Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
60 | * Possibly blocks until there is room in the queue.
61 | * Note : The function may be executed asynchronously,
62 | * therefore, `opaque` must live until function has been completed.
63 | */
64 | void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
65 |
66 |
67 | /*! POOL_tryAdd() :
68 | * Add the job `function(opaque)` to thread pool _if_ a worker is available.
69 | * Returns immediately even if not (does not block).
70 | * @return : 1 if successful, 0 if not.
71 | */
72 | int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
73 |
74 |
75 |
76 | #if defined (__cplusplus)
77 | }
78 | #endif
79 |
80 | #endif
81 |
--------------------------------------------------------------------------------
/tests/collisions/sort.cc:
--------------------------------------------------------------------------------
1 | /*
2 | * sort.cc - C++ sort functions
3 | * Copyright (C) 2019-2021 Yann Collet
4 | * GPL v2 License
5 | *
6 | * This program is free software; you can redistribute it and/or modify
7 | * it under the terms of the GNU General Public License as published by
8 | * the Free Software Foundation; either version 2 of the License, or
9 | * (at your option) any later version.
10 | *
11 | * This program is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU General Public License along
17 | * with this program; if not, write to the Free Software Foundation, Inc.,
18 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19 | *
20 | * You can contact the author at:
21 | * - xxHash homepage: https://www.xxhash.com
22 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
23 | */
24 |
25 | /*
26 | * C++ sort functions tend to run faster than C ones due to templates allowing
27 | * inline optimizations.
28 | * Also, glibc's qsort() seems to inflate memory usage, resulting in OOM
29 | * crashes on the test server.
30 | */
31 |
32 | #include <algorithm> // std::sort
33 | #define XXH_INLINE_ALL // XXH128_cmp
34 | #include <xxhash.h>
35 |
36 | #include "sort.hh"
37 |
38 | void sort64(uint64_t* table, size_t size)
39 | {
40 | std::sort(table, table + size);
41 | }
42 |
43 | #include <stdlib.h> // qsort
44 |
45 | void sort128(XXH128_hash_t* table, size_t size)
46 | {
47 | #if 0
48 | // C++ sort using a custom function object
49 | struct {
50 | bool operator()(XXH128_hash_t a, XXH128_hash_t b) const
51 | {
52 | return XXH128_cmp(&a, &b);
53 | }
54 | } customLess;
55 | std::sort(table, table + size, customLess);
56 | #else
57 | qsort(table, size, sizeof(*table), XXH128_cmp);
58 | #endif
59 | }
60 |
--------------------------------------------------------------------------------
/tests/collisions/sort.hh:
--------------------------------------------------------------------------------
1 | /*
2 | * sort.hh - headers for C++ sort functions
3 | * Copyright (C) 2019-2021 Yann Collet
4 | * GPL v2 License
5 | *
6 | * This program is free software; you can redistribute it and/or modify
7 | * it under the terms of the GNU General Public License as published by
8 | * the Free Software Foundation; either version 2 of the License, or
9 | * (at your option) any later version.
10 | *
11 | * This program is distributed in the hope that it will be useful,
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 | * GNU General Public License for more details.
15 | *
16 | * You should have received a copy of the GNU General Public License along
17 | * with this program; if not, write to the Free Software Foundation, Inc.,
18 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19 | *
20 | * You can contact the author at :
21 | * - xxHash homepage : https://www.xxhash.com
22 | * - xxHash source repository : https://github.com/Cyan4973/xxHash
23 | */
24 |
25 | #ifdef __cplusplus
26 | extern "C" {
27 | #endif
28 |
29 | #include <stddef.h> // size
30 | #include <stdint.h> // uint64_t
31 | #define XXH_STATIC_LINKING_ONLY // XXH128_hash_t
32 | #include "xxhash.h"
33 |
34 | void sort64(uint64_t* table, size_t size);
35 |
36 | void sort128(XXH128_hash_t* table, size_t size);
37 |
38 | #ifdef __cplusplus
39 | } // extern C
40 | #endif
41 |
--------------------------------------------------------------------------------
/tests/collisions/threading.c:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (c) 2016 Tino Reichardt
3 | * All rights reserved.
4 | *
5 | * This source code is licensed under both the BSD-style license (found in the
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 | * in the COPYING file in the root directory of this source tree).
8 | *
9 | * You can contact the author at:
10 | * - zstdmt source repository: https://github.com/mcmilk/zstdmt
11 | */
12 |
13 | /**
14 | * This file will hold wrapper for systems, which do not support pthreads
15 | */
16 |
17 |
18 | /* === Build Macro === */
19 |
20 | #ifndef POOL_MT // can be defined on command line
21 | # define POOL_MT 1
22 | #endif
23 |
24 |
25 | /* create fake symbol to avoid empty translation unit warning */
26 | int g_ZSTD_threading_useles_symbol;
27 |
28 | #if POOL_MT && defined(_WIN32)
29 |
30 | /**
31 | * Windows minimalist Pthread Wrapper
32 | */
33 |
34 |
35 | /* === Dependencies === */
36 | #include <process.h>
37 | #include <errno.h>
38 | #include "threading.h"
39 |
40 |
41 | /* === Implementation === */
42 |
43 | static unsigned __stdcall worker(void *arg)
44 | {
45 | ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
46 | thread->arg = thread->start_routine(thread->arg);
47 | return 0;
48 | }
49 |
50 | int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
51 | void* (*start_routine) (void*), void* arg)
52 | {
53 | (void)unused;
54 | thread->arg = arg;
55 | thread->start_routine = start_routine;
56 | thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
57 |
58 | if (!thread->handle)
59 | return errno;
60 | else
61 | return 0;
62 | }
63 |
64 | int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
65 | {
66 | DWORD result;
67 |
68 | if (!thread.handle) return 0;
69 |
70 | result = WaitForSingleObject(thread.handle, INFINITE);
71 | switch (result) {
72 | case WAIT_OBJECT_0:
73 | if (value_ptr) *value_ptr = thread.arg;
74 | return 0;
75 | case WAIT_ABANDONED:
76 | return EINVAL;
77 | default:
78 | return (int)GetLastError();
79 | }
80 | }
81 |
82 | #endif /* POOL_MT */
83 |
--------------------------------------------------------------------------------
/tests/collisions/threading.h:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright (c) 2016 Tino Reichardt
3 | * All rights reserved.
4 | *
5 | * This source code is licensed under both the BSD-style license (found in the
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 | * in the COPYING file in the root directory of this source tree).
8 | *
9 | * You can contact the author at:
10 | * - zstdmt source repository: https://github.com/mcmilk/zstdmt
11 | */
12 |
13 | #ifndef THREADING_H_938743
14 | #define THREADING_H_938743
15 |
16 | #if defined (__cplusplus)
17 | extern "C" {
18 | #endif
19 |
20 | /* === Build Macro === */
21 |
22 | #ifndef POOL_MT // can be defined on command line
23 | # define POOL_MT 1
24 | #endif
25 |
26 |
27 | /* === Implementation === */
28 |
29 | #if POOL_MT && defined(_WIN32)
30 |
31 | /**
32 | * Define windows version before include
33 | */
34 | #undef WINVER
35 | #define WINVER 0x0600
36 |
37 | #undef _WIN32_WINNT
38 | #define _WIN32_WINNT 0x0600
39 |
40 | #ifndef WIN32_LEAN_AND_MEAN
41 | # define WIN32_LEAN_AND_MEAN
42 | #endif
43 |
44 | #include <windows.h>
45 | #include <stdio.h>
46 |
47 | /* mutex */
48 | #define ZSTD_pthread_mutex_t CRITICAL_SECTION
49 | #define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0)
50 | #define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a))
51 | #define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a))
52 | #define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a))
53 |
54 | /* condition variable */
55 | #define ZSTD_pthread_cond_t CONDITION_VARIABLE
56 | #define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0)
57 | #define ZSTD_pthread_cond_destroy(a) ((void)(a))
58 | #define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE)
59 | #define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a))
60 | #define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a))
61 |
62 | /* ZSTD_pthread_create() and ZSTD_pthread_join() */
63 | typedef struct {
64 | HANDLE handle;
65 | void* (*start_routine)(void*);
66 | void* arg;
67 | } ZSTD_pthread_t;
68 |
69 | int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
70 | void* (*start_routine) (void*), void* arg);
71 |
72 | int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
73 |
74 | /**
75 | * add here more wrappers as required
76 | */
77 |
78 |
79 | #elif POOL_MT /* posix assumed ; need a better detection method */
80 | /* === POSIX Systems === */
81 | # include <pthread.h>
82 |
83 | #define ZSTD_pthread_mutex_t pthread_mutex_t
84 | #define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b))
85 | #define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a))
86 | #define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a))
87 | #define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a))
88 |
89 | #define ZSTD_pthread_cond_t pthread_cond_t
90 | #define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b))
91 | #define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a))
92 | #define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b))
93 | #define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a))
94 | #define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a))
95 |
96 | #define ZSTD_pthread_t pthread_t
97 | #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
98 | #define ZSTD_pthread_join(a, b) pthread_join((a),(b))
99 |
100 | #else /* POOL_MT == 0 */
101 | /* No multithreading support */
102 |
103 | typedef int ZSTD_pthread_mutex_t;
104 | #define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0)
105 | #define ZSTD_pthread_mutex_destroy(a) ((void)(a))
106 | #define ZSTD_pthread_mutex_lock(a) ((void)(a))
107 | #define ZSTD_pthread_mutex_unlock(a) ((void)(a))
108 |
109 | typedef int ZSTD_pthread_cond_t;
110 | #define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0)
111 | #define ZSTD_pthread_cond_destroy(a) ((void)(a))
112 | #define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b))
113 | #define ZSTD_pthread_cond_signal(a) ((void)(a))
114 | #define ZSTD_pthread_cond_broadcast(a) ((void)(a))
115 |
116 | /* do not use ZSTD_pthread_t */
117 |
118 | #endif /* POOL_MT */
119 |
120 | #if defined (__cplusplus)
121 | }
122 | #endif
123 |
124 | #endif /* THREADING_H_938743 */
125 |
--------------------------------------------------------------------------------
/tests/filename-escape.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Exit immediately if any command fails.
4 | # https://stackoverflow.com/a/2871034
5 | set -euxo pipefail
6 |
7 | echo filename-escape-test-string > #39;filename-escape-foo\nbar'
8 |
9 | ./xxhsum #39;filename-escape-foo\nbar' | tee filename-escape-xxh64.txt
10 | cat filename-escape-xxh64.txt
11 | ./xxhsum -c filename-escape-xxh64.txt
12 | hexdump -C filename-escape-xxh64.txt
13 |
14 | ./xxhsum --tag #39;filename-escape-foo\nbar' | tee filename-escape-xxh64-tag.txt
15 | cat filename-escape-xxh64-tag.txt
16 | ./xxhsum -c filename-escape-xxh64-tag.txt
17 | hexdump -C filename-escape-xxh64-tag.txt
18 |
19 | rm filename-escape-xxh64-tag.txt
20 | rm filename-escape-xxh64.txt
21 | rm #39;filename-escape-foo\nbar'
22 |
--------------------------------------------------------------------------------
/tests/generate_unicode_test.c:
--------------------------------------------------------------------------------
1 | /*
2 | * Generates a Unicode test for xxhsum without using Unicode in the source files.
3 | *
4 | * Copyright (C) 2020 Devin Hussey (easyaspi314)
5 | *
6 | * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
7 | *
8 | * Redistribution and use in source and binary forms, with or without
9 | * modification, are permitted provided that the following conditions are
10 | * met:
11 | *
12 | * * Redistributions of source code must retain the above copyright
13 | * notice, this list of conditions and the following disclaimer.
14 | * * Redistributions in binary form must reproduce the above
15 | * copyright notice, this list of conditions and the following disclaimer
16 | * in the documentation and/or other materials provided with the
17 | * distribution.
18 | *
19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 | */
31 |
32 | /*
33 | * Certain terminals don't properly handle UTF-8 (i.e. rxvt and command prompt
34 | * in the default codepage), and that can cause issues when editing text.
35 | *
36 | * We use this C file to generate a file with a Unicode filename, a file with
37 | * a checksum of said file, and both a Windows batch script and a Unix shell
38 | * script to test the file.
39 | */
40 |
41 | #define _CRT_SECURE_NO_WARNINGS /* Silence warnings on MSVC */
42 | #include <stdio.h>
43 |
44 | /* Use a Japanese filename, something that can't be cheated with ANSI.
45 | * yuniko-do.unicode (literally unicode.unicode) */
46 |
47 | /* Use raw hex values to ensure that the output is well-formed UTF-8. It is also more C90 compliant. */
48 | static const char FILENAME[] = {
49 | (char)0xe3, (char)0x83, (char)0xa6, /* U+30e6: Katakana letter yu */
50 | (char)0xe3, (char)0x83, (char)0x8b, /* U+30cb: Katakana letter ni */
51 | (char)0xe3, (char)0x82, (char)0xb3, /* U+30b3: Katakana letter ko */
52 | (char)0xe3, (char)0x83, (char)0xbc, /* U+30fc: Katakana-Hiragana prolonged sound mark (dash) */
53 | (char)0xe3, (char)0x83, (char)0x89, /* U+30c9: Katakana letter do */
54 | '.','u','n','i','c','o','d','e','\0' /* ".unicode" (so we can glob in make clean and .gitignore) */
55 | };
56 |
57 | #ifdef _WIN32
58 | /* The same text as above, but encoded in Windows UTF-16. */
59 | static const wchar_t WFILENAME[] = { 0x30e6, 0x30cb, 0x30b3, 0x30fc, 0x30c9, L'.', L'u', L'n', L'i', L'c', L'o', L'd', L'e', L'\0' };
60 | #endif
61 |
62 | int main(void)
63 | {
64 | FILE *f, *script, *checksum;
65 |
66 | /* Create our Unicode file. Use _wfopen on Windows as fopen doesn't support Unicode filenames. */
67 | #ifdef _WIN32
68 | if (!(f = _wfopen(WFILENAME, L"wb"))) return 1;
69 | #else
70 | if (!(f = fopen(FILENAME, "wb"))) return 1;
71 | #endif
72 | fprintf(f, "test\n");
73 | fclose(f);
74 |
75 | /* XXH64 checksum file with the precalculated checksum for said file. */
76 | if (!(checksum = fopen("unicode_test.xxh64", "wb")))
77 | return 1;
78 | fprintf(checksum, "2d7f1808da1fa63c %s\n", FILENAME);
79 | fclose(checksum);
80 |
81 |
82 | /* Create two scripts for both Windows and Unix. */
83 |
84 | /* Generate a Windows batch script. Always insert CRLF manually. */
85 | if (!(script = fopen("unicode_test.bat", "wb")))
86 | return 1;
87 |
88 | /* Disable echoing the commands. We do that ourselves the naive way. */
89 | fprintf(script, "@echo off\r\n");
90 |
91 | /* Change to codepage 65001 to enable UTF-8 support. */
92 | fprintf(script, "chcp 65001 >NUL 2>&1\r\n");
93 |
94 | /* First test a Unicode filename */
95 | fprintf(script, "echo Testing filename provided on command line...\r\n");
96 | fprintf(script, "echo xxhsum.exe \"%s\"\r\n", FILENAME);
97 | fprintf(script, "xxhsum.exe \"%s\"\r\n", FILENAME);
98 |
99 | /* Bail on error */
100 | fprintf(script, "if %%ERRORLEVEL%% neq 0 (\r\n");
101 | fprintf(script, " exit /B %%ERRORLEVEL%%\r\n");
102 | fprintf(script, ")\r\n");
103 |
104 | /* Then test a checksum file. */
105 | fprintf(script, "echo Testing a checksum file...\r\n");
106 | fprintf(script, "echo xxhsum.exe -c unicode_test.xxh64\r\n");
107 | fprintf(script, "xxhsum.exe -c unicode_test.xxh64\r\n");
108 |
109 | fprintf(script, "exit /B %%ERRORLEVEL%%\r\n");
110 |
111 | fclose(script);
112 |
113 | /* Generate a Unix shell script */
114 | if (!(script = fopen("unicode_test.sh", "wb")))
115 | return 1;
116 |
117 | fprintf(script, "#!/bin/sh\n");
118 | /*
119 | * Some versions of MSYS, MinGW and Cygwin do not support UTF-8, and the ones that
120 | * don't may error with something like this:
121 | *
122 | * Error: Could not open '<mojibake>.unicode': No such file or directory.
123 | *
124 | * which is an internal error that happens when it tries to convert MinGW/Cygwin
125 | * paths to Windows paths.
126 | *
127 | * In that case, we bail to cmd.exe and the batch script, which supports UTF-8
128 | * on Windows 7 and later.
129 | */
130 | fprintf(script, "case $(uname) in\n");
131 | /* MinGW/MSYS converts /c to C:\ unless you have a double slash,
132 | * Cygwin does not. */
133 | fprintf(script, " *CYGWIN*)\n");
134 | fprintf(script, " exec cmd.exe /c unicode_test.bat\n");
135 | fprintf(script, " ;;\n");
136 | fprintf(script, " *MINGW*|*MSYS*)\n");
137 | fprintf(script, " exec cmd.exe //c unicode_test.bat\n");
138 | fprintf(script, " ;;\n");
139 | fprintf(script, "esac\n");
140 |
141 | /* First test a Unicode filename */
142 | fprintf(script, "echo Testing filename provided on command line...\n");
143 | fprintf(script, "echo './xxhsum \"%s\" || exit $?'\n", FILENAME);
144 | fprintf(script, "./xxhsum \"%s\" || exit $?\n", FILENAME);
145 |
146 | /* Then test a checksum file. */
147 | fprintf(script, "echo Testing a checksum file...\n");
148 | fprintf(script, "echo './xxhsum -c unicode_test.xxh64 || exit $?'\n");
149 | fprintf(script, "./xxhsum -c unicode_test.xxh64 || exit $?\n");
150 |
151 | fclose(script);
152 |
153 | return 0;
154 | }
155 |
--------------------------------------------------------------------------------
/tests/multiInclude.c:
--------------------------------------------------------------------------------
1 | /*
2 | * Multi-include test program
3 | * Validates that xxhash.h can be included multiple times and in any order
4 | *
5 | * Copyright (C) 2020 Yann Collet
6 | *
7 | * GPL v2 License
8 | *
9 | * This program is free software; you can redistribute it and/or modify
10 | * it under the terms of the GNU General Public License as published by
11 | * the Free Software Foundation; either version 2 of the License, or
12 | * (at your option) any later version.
13 | *
14 | * This program is distributed in the hope that it will be useful,
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 | * GNU General Public License for more details.
18 | *
19 | * You should have received a copy of the GNU General Public License along
20 | * with this program; if not, write to the Free Software Foundation, Inc.,
21 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22 | *
23 | * You can contact the author at:
24 | * - xxHash homepage: https://www.xxhash.com
25 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
26 | */
27 |
28 | #include <stdio.h> /* printf */
29 |
30 | /* Normal include, gives access to public symbols */
31 | #include "../xxhash.h"
32 |
33 | /* Multiple consecutive inclusions are handled properly. */
34 | #include "../xxhash.h"
35 |
36 | /*
37 | * Advanced include, gives access to experimental symbols
38 | * This test ensures that xxhash.h can be included multiple times
39 | * and in any order. The tested order is more difficult:
40 | * without care, the declaration of experimental symbols could be skipped.
41 | */
42 | #define XXH_STATIC_LINKING_ONLY
43 | #include "../xxhash.h"
44 |
45 | /*
46 | * Inlining: redefine all identifiers, keep them private to the unit.
47 | * Note: Without specific efforts, the identifier names would collide.
48 | *
49 | * To be linked with and without xxhash.o
50 | * to test the symbol's presence and naming collisions.
51 | */
52 | #define XXH_INLINE_ALL
53 | #include "../xxhash.h"
54 |
55 | /*
56 | * Multiple consecutive inclusions with XXH_INLINE_ALL are handled properly.
57 | */
58 | #define XXH_INLINE_ALL
59 | #include "../xxhash.h"
60 |
61 |
62 | void hash_advanced(void)
63 | {
64 | const char input[] = "Hello World !";
65 | XXH3_state_t state; /* this type is part of experimental API */
66 |
67 | XXH3_64bits_reset(&state);
68 |
69 | XXH3_64bits_update(&state, input, sizeof(input));
70 |
71 | { XXH64_hash_t const h = XXH3_64bits_digest(&state);
72 | printf("hash '%s': %08x%08x \n", input, (unsigned)(h >> 32), (unsigned)h);
73 | }
74 | }
75 |
76 | int main(void)
77 | {
78 | hash_advanced();
79 | }
80 |
--------------------------------------------------------------------------------
/tests/ppc_define.c:
--------------------------------------------------------------------------------
1 | /*
2 | * Multi-include test program
3 | * ensure that pixel, bool and vector are not redefined
4 | *
5 | * Copyright (C) 2020 Yann Collet
6 | *
7 | * GPL v2 License
8 | *
9 | * This program is free software; you can redistribute it and/or modify
10 | * it under the terms of the GNU General Public License as published by
11 | * the Free Software Foundation; either version 2 of the License, or
12 | * (at your option) any later version.
13 | *
14 | * This program is distributed in the hope that it will be useful,
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 | * GNU General Public License for more details.
18 | *
19 | * You should have received a copy of the GNU General Public License along
20 | * with this program; if not, write to the Free Software Foundation, Inc.,
21 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22 | *
23 | * You can contact the author at:
24 | * - xxHash homepage: https://www.xxhash.com
25 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
26 | */
27 |
28 | /* gcc's altivec.h, included for the VSX code path,
29 | * may, in some circumstances, redefine
30 | * bool, vector and pixel keywords.
31 | *
32 | * This unit checks if it happens.
33 | * It's a compile test.
34 | * The test is mostly meaningful for PPC target using altivec.h
35 | * hence XXH_VECTOR == XXH_VSX
36 | */
37 |
38 | #define BOOL_VALUE 32123456
39 | #define bool BOOL_VALUE
40 |
41 | #define VECTOR_VALUE 374464784
42 | #define vector VECTOR_VALUE
43 |
44 | #define PIXEL_VALUE 5846841
45 | #define pixel PIXEL_VALUE
46 |
47 | #define XXH_INLINE_ALL
48 | #include "../xxhash.h"
49 |
50 | #if (bool != BOOL_VALUE)
51 | # error "bool macro was redefined !"
52 | #endif
53 |
54 | #if (vector != VECTOR_VALUE)
55 | # error "vector macro was redefined !"
56 | #endif
57 |
58 | #if (pixel != PIXEL_VALUE)
59 | # error "pixel macro was redefined !"
60 | #endif
61 |
62 | int g_nonEmptyUnit = 0;
63 |
--------------------------------------------------------------------------------
/tests/test_alias.c:
--------------------------------------------------------------------------------
1 | #define XXH_INLINE_ALL
2 |
3 | #include <inttypes.h>
4 | #include <stdio.h>
5 | #include "xxhash.h"
6 |
7 | int main() {
8 | // it seems this has to be exactly 24 bytes.
9 | union {
10 | char x[24];
11 | // force 8-byte alignment without making
12 | // aliasable with uint64_t.
13 | void *y[3];
14 | } data = {.x = "garblegarblegarblegarble"};
15 | uint64_t hash = XXH64(&data, sizeof(data), 0);
16 | printf("%016"PRIx64"\n", hash);
17 | return 0;
18 | }
19 |
--------------------------------------------------------------------------------
/tests/unicode_lint.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # `unicode_lint.sh' determines whether source files under ${dirs} directories
4 | # contain Unicode characters, and fails if any do.
5 | #
6 | # We don't recommend to call this script directly.
7 | # Instead of it, use `make lint-unicode` via root directory Makefile.
8 |
9 | # ${dirs} : target directories
10 | dirs=(./ ./cli ./tests ./tests/bench ./tests/collisions)
11 |
12 | SCRIPT_DIR="`dirname "${BASH_SOURCE[0]}"`"
13 | cd ${SCRIPT_DIR}/..
14 |
15 | echo "Ensure no unicode character is present in source files *.{c,h}"
16 | pass=true
17 |
18 | # Scan each directory in ${dirs} for Unicode in source (*.c, *.h) files
19 | i=0
20 | while [ $i -lt ${#dirs[@]} ]
21 | do
22 | dir=${dirs[$i]}
23 | echo dir=$dir
24 | result=$(
25 | find ${dir} -regex '.*\.\(c\|h\)#39; -exec grep -P -n "[^\x00-\x7F]" {} \; -exec echo "{}: FAIL" \;
26 | )
27 | if [[ $result ]]; then
28 | echo "$result"
29 | pass=false
30 | fi
31 | i=`expr $i + 1`
32 | done
33 |
34 |
35 | # Result
36 | if [ "$pass" = true ]; then
37 | echo "All tests successful: no unicode character detected"
38 | echo "Result: PASS"
39 | exit 0
40 | else
41 | echo "Result: FAIL"
42 | exit 1
43 | fi
44 |
--------------------------------------------------------------------------------
/xxh3.h:
--------------------------------------------------------------------------------
1 | /*
2 | * xxHash - Extremely Fast Hash algorithm
3 | * Development source file for `xxh3`
4 | * Copyright (C) 2019-2021 Yann Collet
5 | *
6 | * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
7 | *
8 | * Redistribution and use in source and binary forms, with or without
9 | * modification, are permitted provided that the following conditions are
10 | * met:
11 | *
12 | * * Redistributions of source code must retain the above copyright
13 | * notice, this list of conditions and the following disclaimer.
14 | * * Redistributions in binary form must reproduce the above
15 | * copyright notice, this list of conditions and the following disclaimer
16 | * in the documentation and/or other materials provided with the
17 | * distribution.
18 | *
19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 | *
31 | * You can contact the author at:
32 | * - xxHash homepage: https://www.xxhash.com
33 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
34 | */
35 |
36 | /*
37 | * Note: This file used to host the source code of XXH3_* variants.
38 | * during the development period.
39 | * The source code is now properly integrated within xxhash.h.
40 | *
41 | * xxh3.h is no longer useful,
42 | * but it is still provided for compatibility with source code
43 | * which used to include it directly.
44 | *
45 | * Programs are now highly discouraged to include xxh3.h.
46 | * Include `xxhash.h` instead, which is the officially supported interface.
47 | *
48 | * In the future, xxh3.h will start to generate warnings, then errors,
49 | * then it will be removed from source package and from include directory.
50 | */
51 |
52 | /* Simulate the same impact as including the old xxh3.h source file */
53 |
54 | #define XXH_INLINE_ALL
55 | #include "xxhash.h"
56 |
--------------------------------------------------------------------------------
/xxh_x86dispatch.h:
--------------------------------------------------------------------------------
1 | /*
2 | * xxHash - XXH3 Dispatcher for x86-based targets
3 | * Copyright (C) 2020-2024 Yann Collet
4 | *
5 | * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
6 | *
7 | * Redistribution and use in source and binary forms, with or without
8 | * modification, are permitted provided that the following conditions are
9 | * met:
10 | *
11 | * * Redistributions of source code must retain the above copyright
12 | * notice, this list of conditions and the following disclaimer.
13 | * * Redistributions in binary form must reproduce the above
14 | * copyright notice, this list of conditions and the following disclaimer
15 | * in the documentation and/or other materials provided with the
16 | * distribution.
17 | *
18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 | *
30 | * You can contact the author at:
31 | * - xxHash homepage: https://www.xxhash.com
32 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
33 | */
34 |
35 | #ifndef XXH_X86DISPATCH_H_13563687684
36 | #define XXH_X86DISPATCH_H_13563687684
37 |
38 | #include "xxhash.h" /* XXH64_hash_t, XXH3_state_t */
39 |
40 | #if defined (__cplusplus)
41 | extern "C" {
42 | #endif
43 |
44 | /*!
45 | * @brief Returns the best XXH3 implementation for x86
46 | *
47 | * @return The best @ref XXH_VECTOR implementation.
48 | * @see XXH_VECTOR_TYPES
49 | */
50 | XXH_PUBLIC_API int XXH_featureTest(void);
51 |
52 | XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_dispatch(XXH_NOESCAPE const void* input, size_t len);
53 | XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed);
54 | XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen);
55 | XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len);
56 |
57 | XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_dispatch(XXH_NOESCAPE const void* input, size_t len);
58 | XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed);
59 | XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen);
60 | XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update_dispatch(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len);
61 |
62 | #if defined (__cplusplus)
63 | }
64 | #endif
65 |
66 |
67 | /* automatic replacement of XXH3 functions.
68 | * can be disabled by setting XXH_DISPATCH_DISABLE_REPLACE */
69 | #ifndef XXH_DISPATCH_DISABLE_REPLACE
70 |
71 | # undef XXH3_64bits
72 | # define XXH3_64bits XXH3_64bits_dispatch
73 | # undef XXH3_64bits_withSeed
74 | # define XXH3_64bits_withSeed XXH3_64bits_withSeed_dispatch
75 | # undef XXH3_64bits_withSecret
76 | # define XXH3_64bits_withSecret XXH3_64bits_withSecret_dispatch
77 | # undef XXH3_64bits_update
78 | # define XXH3_64bits_update XXH3_64bits_update_dispatch
79 |
80 | # undef XXH128
81 | # define XXH128 XXH3_128bits_withSeed_dispatch
82 | # undef XXH3_128bits
83 | # define XXH3_128bits XXH3_128bits_dispatch
84 | # undef XXH3_128bits_withSeed
85 | # define XXH3_128bits_withSeed XXH3_128bits_withSeed_dispatch
86 | # undef XXH3_128bits_withSecret
87 | # define XXH3_128bits_withSecret XXH3_128bits_withSecret_dispatch
88 | # undef XXH3_128bits_update
89 | # define XXH3_128bits_update XXH3_128bits_update_dispatch
90 |
91 | #endif /* XXH_DISPATCH_DISABLE_REPLACE */
92 |
93 | #endif /* XXH_X86DISPATCH_H_13563687684 */
94 |
--------------------------------------------------------------------------------
/xxhash.c:
--------------------------------------------------------------------------------
1 | /*
2 | * xxHash - Extremely Fast Hash algorithm
3 | * Copyright (C) 2012-2023 Yann Collet
4 | *
5 | * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
6 | *
7 | * Redistribution and use in source and binary forms, with or without
8 | * modification, are permitted provided that the following conditions are
9 | * met:
10 | *
11 | * * Redistributions of source code must retain the above copyright
12 | * notice, this list of conditions and the following disclaimer.
13 | * * Redistributions in binary form must reproduce the above
14 | * copyright notice, this list of conditions and the following disclaimer
15 | * in the documentation and/or other materials provided with the
16 | * distribution.
17 | *
18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 | *
30 | * You can contact the author at:
31 | * - xxHash homepage: https://www.xxhash.com
32 | * - xxHash source repository: https://github.com/Cyan4973/xxHash
33 | */
34 |
35 | /*
36 | * xxhash.c instantiates functions defined in xxhash.h
37 | */
38 |
39 | #define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
40 | #define XXH_IMPLEMENTATION /* access definitions */
41 |
42 | #include "xxhash.h"
43 |
--------------------------------------------------------------------------------