├── b3sum ├── .gitignore ├── LICENSE_A2 ├── LICENSE_CC0 ├── LICENSE_A2LLVM ├── Cargo.toml ├── README.md ├── src │ └── unit_tests.rs └── what_does_check_do.md ├── .gitignore ├── .cargo └── config.toml ├── c ├── dependencies │ ├── CMakeLists.txt │ └── tbb │ │ └── CMakeLists.txt ├── .gitignore ├── cmake │ └── BLAKE3 │ │ ├── Testing.cmake │ │ ├── Examples.cmake │ │ ├── Utils.cmake │ │ └── ContinuousIntegration.cmake ├── blake3_c_rust_bindings │ ├── README.md │ ├── Cargo.toml │ ├── cross_test.sh │ ├── build.rs │ └── src │ │ └── lib.rs ├── blake3-config.cmake.in ├── libblake3.pc.in ├── example.c ├── blake3_tbb.cpp ├── example_tbb.c ├── CMakePresets.json ├── Makefile.testing ├── blake3.h ├── test.py ├── main.c ├── blake3_portable.c └── blake3_dispatch.c ├── .git-blame-ignore-revs ├── test_vectors ├── src │ ├── bin │ │ └── generate.rs │ └── lib.rs ├── Cargo.toml └── cross_test.sh ├── tools ├── instruction_set_support │ ├── Cargo.toml │ └── src │ │ └── main.rs ├── compiler_version │ ├── Cargo.toml │ ├── build.rs │ └── src │ │ └── main.rs └── release.md ├── reference_impl ├── Cargo.toml └── README.md ├── CONTRIBUTING.md ├── .github └── workflows │ ├── build_b3sum.py │ ├── tag.yml │ └── upload_github_release_asset.py ├── src ├── guts.rs ├── ffi_avx2.rs ├── ffi_neon.rs ├── join.rs ├── io.rs ├── ffi_sse2.rs ├── ffi_sse41.rs ├── ffi_avx512.rs ├── portable.rs └── traits.rs ├── media ├── B3.svg └── BLAKE3.svg ├── Cargo.toml ├── LICENSE_CC0 ├── README.md ├── LICENSE_A2 └── LICENSE_A2LLVM /b3sum/.gitignore: -------------------------------------------------------------------------------- 1 | !Cargo.lock 2 | -------------------------------------------------------------------------------- /b3sum/LICENSE_A2: -------------------------------------------------------------------------------- 1 | ../LICENSE_A2 -------------------------------------------------------------------------------- /b3sum/LICENSE_CC0: -------------------------------------------------------------------------------- 1 | ../LICENSE_CC0 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Cargo.lock 2 | target 3 | -------------------------------------------------------------------------------- /b3sum/LICENSE_A2LLVM: -------------------------------------------------------------------------------- 1 | ../LICENSE_A2LLVM -------------------------------------------------------------------------------- /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [target.wasm32-wasip1] 2 | runner = "wasmtime" 3 | -------------------------------------------------------------------------------- /c/dependencies/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if(BLAKE3_USE_TBB) 2 | add_subdirectory(tbb) 3 | endif() 4 | -------------------------------------------------------------------------------- /c/.gitignore: -------------------------------------------------------------------------------- 1 | blake3 2 | example 3 | example_tbb 4 | build/ 5 | *.o 6 | 7 | CMakeUserPresets.json 8 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # CMakeLists.txt whitespace fixups 2 | 3e14f865d30271c74fc68d417af488ea91b66d48 3 | -------------------------------------------------------------------------------- /c/cmake/BLAKE3/Testing.cmake: -------------------------------------------------------------------------------- 1 | if(BLAKE3_TESTING_CI) 2 | include(BLAKE3/ContinuousIntegration) 3 | endif() 4 | -------------------------------------------------------------------------------- /test_vectors/src/bin/generate.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | // The trailing newline is included. 3 | print!("{}", test_vectors::generate_json()); 4 | } 5 | -------------------------------------------------------------------------------- /tools/instruction_set_support/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "instruction_set_support" 3 | version = "0.0.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | -------------------------------------------------------------------------------- /tools/compiler_version/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "compiler_version" 3 | version = "0.0.0" 4 | edition = "2021" 5 | 6 | [build-dependencies] 7 | cc = "1.0.50" 8 | -------------------------------------------------------------------------------- /reference_impl/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "reference_impl" 3 | version = "0.0.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | name = "reference_impl" 8 | path = "reference_impl.rs" 9 | -------------------------------------------------------------------------------- /tools/compiler_version/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let build = cc::Build::new(); 3 | let compiler = build.get_compiler(); 4 | let compiler_path = compiler.path().to_string_lossy(); 5 | println!("cargo:rustc-env=COMPILER_PATH={}", compiler_path); 6 | } 7 | -------------------------------------------------------------------------------- /c/blake3_c_rust_bindings/README.md: -------------------------------------------------------------------------------- 1 | These are Rust bindings for the C implementation of BLAKE3. As there is 2 | a native Rust implementation of BLAKE3 provided in this same repo, these 3 | bindings are not expected to be used in production. They're intended for 4 | testing and benchmarking. 5 | -------------------------------------------------------------------------------- /c/blake3-config.cmake.in: -------------------------------------------------------------------------------- 1 | @PACKAGE_INIT@ 2 | 3 | include(CMakeFindDependencyMacro) 4 | 5 | # Remember TBB option state 6 | set(BLAKE3_USE_TBB @BLAKE3_USE_TBB@) 7 | 8 | if(BLAKE3_USE_TBB) 9 | find_dependency(TBB @TBB_VERSION@) 10 | endif() 11 | 12 | include("${CMAKE_CURRENT_LIST_DIR}/blake3-targets.cmake") 13 | 14 | check_required_components(blake3) 15 | -------------------------------------------------------------------------------- /c/libblake3.pc.in: -------------------------------------------------------------------------------- 1 | prefix="@CMAKE_INSTALL_PREFIX@" 2 | exec_prefix="${prefix}" 3 | libdir="@PKG_CONFIG_INSTALL_LIBDIR@" 4 | includedir="@PKG_CONFIG_INSTALL_INCLUDEDIR@" 5 | 6 | Name: @PROJECT_NAME@ 7 | Description: @PROJECT_DESCRIPTION@ 8 | Version: @PROJECT_VERSION@ 9 | 10 | Requires: @PKG_CONFIG_REQUIRES@ 11 | Libs: -L"${libdir}" -lblake3 @PKG_CONFIG_LIBS@ 12 | Cflags: -I"${includedir}" @PKG_CONFIG_CFLAGS@ 13 | -------------------------------------------------------------------------------- /tools/instruction_set_support/src/main.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] 3 | { 4 | dbg!(is_x86_feature_detected!("sse2")); 5 | dbg!(is_x86_feature_detected!("sse4.1")); 6 | dbg!(is_x86_feature_detected!("avx2")); 7 | dbg!(is_x86_feature_detected!("avx512f")); 8 | dbg!(is_x86_feature_detected!("avx512vl")); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /c/cmake/BLAKE3/Examples.cmake: -------------------------------------------------------------------------------- 1 | if(NOT WIN32) 2 | add_executable(blake3-example 3 | example.c) 4 | target_link_libraries(blake3-example PRIVATE blake3) 5 | install(TARGETS blake3-example) 6 | 7 | if(BLAKE3_USE_TBB) 8 | add_executable(blake3-example-tbb 9 | example_tbb.c) 10 | target_link_libraries(blake3-example-tbb PRIVATE blake3) 11 | install(TARGETS blake3-example-tbb) 12 | endif() 13 | endif() 14 | -------------------------------------------------------------------------------- /test_vectors/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "test_vectors" 3 | version = "0.0.0" 4 | edition = "2021" 5 | 6 | [features] 7 | neon = ["blake3/neon"] 8 | prefer_intrinsics = ["blake3/prefer_intrinsics"] 9 | pure = ["blake3/pure"] 10 | wasm32_simd = ["blake3/wasm32_simd"] 11 | 12 | [dependencies] 13 | # If you ever change these path dependencies, you'll probably need to update 14 | # cross_test.sh, or CI will break. I'm sorry >.< 15 | blake3 = { path = "../" } 16 | hex = "0.4.0" 17 | reference_impl = { path = "../reference_impl" } 18 | serde = { version = "1.0", features = ["derive"] } 19 | serde_json = "1.0" 20 | -------------------------------------------------------------------------------- /reference_impl/README.md: -------------------------------------------------------------------------------- 1 | This is the reference implementation of BLAKE3. It is used for testing and 2 | as a readable example of the algorithms involved. Section 5.1 of [the BLAKE3 3 | spec](https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf) 4 | discusses this implementation. You can render docs for this implementation 5 | by running `cargo doc --open` in this directory. 6 | 7 | This implementation is a single file 8 | ([`reference_impl.rs`](reference_impl.rs)) with no dependencies. It is 9 | not optimized for performance. 10 | 11 | There are ports of this reference implementation to other languages: 12 | 13 | - [C](https://github.com/oconnor663/blake3_reference_impl_c) 14 | - [Python](https://github.com/oconnor663/pure_python_blake3) 15 | -------------------------------------------------------------------------------- /tools/release.md: -------------------------------------------------------------------------------- 1 | # Release checklist 2 | 3 | - Make sure `cargo outdated -R` is clean in the root and in b3sum/. 4 | - Bump the version in the root Cargo.toml. 5 | - Bump the version in b3sum/Cargo.toml. 6 | - Bump the dependency version too, if new features are used. 7 | - Delete b3sum/Cargo.lock and recreate it with `cargo build` or similar. 8 | - Update the `-h` output in b3sum/README.md if it's changed. 9 | - Bump `BLAKE3_VERSION_STRING` in c/blake3.h. 10 | - Bump `VERSION` in c/CMakeLists.txt. 11 | - Make a version bump commit with change notes. 12 | - `git push` and make sure CI is green. 13 | - `git tag` the version bump commit with the new version number. 14 | - `git push --tags` 15 | - `cargo publish` in the root. 16 | - `cargo publish` in b3sum/. 17 | -------------------------------------------------------------------------------- /c/dependencies/tbb/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package(TBB 2021.11.0 QUIET) 2 | 3 | if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.11) 4 | include(FetchContent) 5 | 6 | if(NOT TBB_FOUND AND BLAKE3_FETCH_TBB) 7 | set(CMAKE_C_STANDARD 99) 8 | set(CMAKE_C_EXTENSIONS OFF) 9 | 10 | set(CMAKE_CXX_STANDARD 20) 11 | set(CMAKE_CXX_EXTENSIONS ON) 12 | 13 | option(TBB_TEST OFF "") 14 | option(TBBMALLOC_BUILD OFF "") 15 | 16 | mark_as_advanced(TBB_TEST) 17 | mark_as_advanced(TBBMALLOC_BUILD) 18 | 19 | FetchContent_Declare( 20 | TBB 21 | GIT_REPOSITORY https://github.com/uxlfoundation/oneTBB 22 | GIT_TAG 0c0ff192a2304e114bc9e6557582dfba101360ff # v2022.0.0 23 | GIT_SHALLOW TRUE 24 | ) 25 | 26 | FetchContent_MakeAvailable(TBB) 27 | endif() 28 | endif() 29 | -------------------------------------------------------------------------------- /b3sum/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "b3sum" 3 | version = "1.8.2" 4 | authors = ["Jack O'Connor "] 5 | description = "a command line implementation of the BLAKE3 hash function" 6 | repository = "https://github.com/BLAKE3-team/BLAKE3" 7 | license = "CC0-1.0 OR Apache-2.0 OR Apache-2.0 WITH LLVM-exception" 8 | readme = "README.md" 9 | edition = "2021" 10 | 11 | [features] 12 | neon = ["blake3/neon"] 13 | prefer_intrinsics = ["blake3/prefer_intrinsics"] 14 | pure = ["blake3/pure"] 15 | 16 | [dependencies] 17 | anyhow = "1.0.25" 18 | blake3 = { version = "1.8", path = "..", features = ["mmap", "rayon"] } 19 | clap = { version = "4.0.8", features = ["derive", "wrap_help"] } 20 | hex = "0.4.0" 21 | rayon-core = "1.12.1" 22 | wild = "2.0.3" 23 | 24 | [dev-dependencies] 25 | duct = "1.0.0" 26 | tempfile = "3.1.0" 27 | -------------------------------------------------------------------------------- /tools/compiler_version/src/main.rs: -------------------------------------------------------------------------------- 1 | use std::process::Command; 2 | 3 | fn main() { 4 | // Print the rustc version. 5 | Command::new(env!("CARGO")) 6 | .args(&["rustc", "--quiet", "--", "--version"]) 7 | .status() 8 | .unwrap(); 9 | println!(); 10 | 11 | // Print the Cargo version. 12 | Command::new(env!("CARGO")) 13 | .args(&["--version"]) 14 | .status() 15 | .unwrap(); 16 | println!(); 17 | 18 | // Print the C compiler version. This relies on C compiler detection done 19 | // in build.rs, which sets the COMPILER_PATH variable. 20 | let compiler_path = env!("COMPILER_PATH"); 21 | let mut compiler_command = Command::new(compiler_path); 22 | // Use the --version flag on everything other than MSVC. 23 | if !cfg!(target_env = "msvc") { 24 | compiler_command.arg("--version"); 25 | } 26 | let _ = compiler_command.status().unwrap(); 27 | } 28 | -------------------------------------------------------------------------------- /c/example.c: -------------------------------------------------------------------------------- 1 | #include "blake3.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | int main(void) { 8 | // Initialize the hasher. 9 | blake3_hasher hasher; 10 | blake3_hasher_init(&hasher); 11 | 12 | // Read input bytes from stdin. 13 | unsigned char buf[65536]; 14 | while (1) { 15 | ssize_t n = read(STDIN_FILENO, buf, sizeof(buf)); 16 | if (n > 0) { 17 | blake3_hasher_update(&hasher, buf, n); 18 | } else if (n == 0) { 19 | break; // end of file 20 | } else { 21 | fprintf(stderr, "read failed: %s\n", strerror(errno)); 22 | return 1; 23 | } 24 | } 25 | 26 | // Finalize the hash. BLAKE3_OUT_LEN is the default output length, 32 bytes. 27 | uint8_t output[BLAKE3_OUT_LEN]; 28 | blake3_hasher_finalize(&hasher, output, BLAKE3_OUT_LEN); 29 | 30 | // Print the hash as hexadecimal. 31 | for (size_t i = 0; i < BLAKE3_OUT_LEN; i++) { 32 | printf("%02x", output[i]); 33 | } 34 | printf("\n"); 35 | return 0; 36 | } 37 | -------------------------------------------------------------------------------- /c/cmake/BLAKE3/Utils.cmake: -------------------------------------------------------------------------------- 1 | if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.10) 2 | include_guard(GLOBAL) 3 | endif() 4 | 5 | if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.14) 6 | set(BLAKE3_CXX_COMPILER_FRONTEND_VARIANT "${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}") 7 | else() 8 | # Get the C++ compiler name without extension 9 | get_filename_component(BLAKE3_CMAKE_CXX_COMPILER_NAME "${CMAKE_CXX_COMPILER}" NAME_WE) 10 | # Strip any trailing versioning from the C++ compiler name 11 | string(REGEX MATCH "^.*(clang\\+\\+|clang-cl)" BLAKE3_CMAKE_CXX_COMPILER_NAME "${BLAKE3_CMAKE_CXX_COMPILER_NAME}") 12 | # Guess the frontend variant from the C++ compiler name 13 | if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND BLAKE3_CMAKE_CXX_COMPILER_NAME STREQUAL "clang-cl") 14 | set(BLAKE3_CXX_COMPILER_FRONTEND_VARIANT "MSVC") 15 | elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") 16 | set(BLAKE3_CXX_COMPILER_FRONTEND_VARIANT "MSVC") 17 | else() 18 | set(BLAKE3_CXX_COMPILER_FRONTEND_VARIANT "GNU") 19 | endif() 20 | unset(BLAKE3_CMAKE_CXX_COMPILER_NAME) 21 | endif() 22 | -------------------------------------------------------------------------------- /test_vectors/cross_test.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | # This hacky script works around the fact that `cross test` does not support 4 | # path dependencies. (It uses a docker shared folder to let the guest access 5 | # project files, so parent directories aren't available.) Solve this problem by 6 | # copying the entire project to a temp dir and rearranging paths to put 7 | # "blake3" and "reference_impl" underneath "test_vectors", so that everything 8 | # is accessible. Hopefully this will just run on CI forever and no one will 9 | # ever read this and discover my deep shame. 10 | 11 | set -e -u -o pipefail 12 | 13 | project_root="$(realpath "$(dirname "$BASH_SOURCE")/..")" 14 | tmpdir="$(mktemp -d)" 15 | echo "Running cross tests in $tmpdir" 16 | cd "$tmpdir" 17 | git clone "$project_root" blake3 18 | mv blake3/test_vectors . 19 | mv blake3/reference_impl test_vectors 20 | mv blake3 test_vectors 21 | cd test_vectors 22 | sed -i 's|blake3 = { path = "../" }|blake3 = { path = "./blake3" }|' Cargo.toml 23 | sed -i 's|reference_impl = { path = "../reference_impl" }|reference_impl = { path = "reference_impl" }|' Cargo.toml 24 | 25 | cross test "$@" 26 | -------------------------------------------------------------------------------- /c/blake3_c_rust_bindings/Cargo.toml: -------------------------------------------------------------------------------- 1 | # These are Rust bindings for the C implementation of BLAKE3. As there is a 2 | # native (and faster) Rust implementation of BLAKE3 provided in this same repo, 3 | # these bindings are not expected to be used in production. They're intended 4 | # for testing and benchmarking. 5 | 6 | [package] 7 | name = "blake3_c_rust_bindings" 8 | version = "0.0.0" 9 | description = "TESTING ONLY Rust bindings for the BLAKE3 C implementation" 10 | edition = "2021" 11 | 12 | [features] 13 | # By default the x86-64 build uses assembly implementations. This feature makes 14 | # the build use the C intrinsics implementations instead. 15 | prefer_intrinsics = [] 16 | # Activate NEON bindings. We don't currently do any CPU feature detection for 17 | # this. If this Cargo feature is on, the NEON gets used. 18 | neon = [] 19 | # Enable TBB-based multithreading. 20 | tbb = [] 21 | 22 | [dev-dependencies] 23 | arrayref = "0.3.5" 24 | arrayvec = { version = "0.7.0", default-features = false } 25 | page_size = "0.6.0" 26 | rand = "0.9.0" 27 | rand_chacha = "0.9.0" 28 | reference_impl = { path = "../../reference_impl" } 29 | 30 | [build-dependencies] 31 | cc = "1.0.48" 32 | ignore = "0.4.23" 33 | -------------------------------------------------------------------------------- /c/blake3_c_rust_bindings/cross_test.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | # This hacky script works around the fact that `cross test` does not support 4 | # path dependencies. (It uses a docker shared folder to let the guest access 5 | # project files, so parent directories aren't available.) Solve this problem by 6 | # copying the entire project to a temp dir and rearranging paths to put "c" and 7 | # "reference_impl" underneath "blake3_c_rust_bindings", so that everything is 8 | # accessible. Hopefully this will just run on CI forever and no one will ever 9 | # read this and discover my deep shame. 10 | 11 | set -e -u -o pipefail 12 | 13 | project_root="$(realpath "$(dirname "$BASH_SOURCE")/../..")" 14 | tmpdir="$(mktemp -d)" 15 | echo "Running cross tests in $tmpdir" 16 | cd "$tmpdir" 17 | git clone "$project_root" blake3 18 | mv blake3/c/blake3_c_rust_bindings . 19 | mv blake3/reference_impl blake3_c_rust_bindings 20 | mv blake3/c blake3_c_rust_bindings 21 | cd blake3_c_rust_bindings 22 | sed -i 's|reference_impl = { path = "../../reference_impl" }|reference_impl = { path = "reference_impl" }|' Cargo.toml 23 | 24 | export BLAKE3_C_DIR_OVERRIDE="./c" 25 | cat > Cross.toml << EOF 26 | [build.env] 27 | passthrough = [ 28 | "BLAKE3_C_DIR_OVERRIDE", 29 | ] 30 | EOF 31 | cross test "$@" 32 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We welcome and encourage third-party contributions to BLAKE3, be it reports of issues encountered while using the software or proposals of patches. 4 | 5 | ## Bug reports 6 | 7 | Bugs and other problems should be reported on [GitHub Issues](https://github.com/BLAKE3/BLAKE3/issues). 8 | 9 | If you report a bug, please: 10 | 11 | * Check that it's not already reported in the [GitHub Issues](https://github.com/BLAKE3/BLAKE3/issues). 12 | * Provide information to help us diagnose and ideally reproduce the bug. 13 | 14 | ## Patches 15 | 16 | We encourage you to fix a bug via a [GitHub Pull request](https://github.com/BLAKE3/BLAKE3/pulls), preferably after creating a related issue and referring it in the PR. 17 | 18 | If you contribute code and submit a patch, please note the following: 19 | 20 | * We use Rust's stable branch for developing BLAKE3. 21 | * Pull requests should target the `master` branch. 22 | * Try to follow the established Rust [style guidelines](https://doc.rust-lang.org/1.0.0/style/). 23 | 24 | Also please make sure to create new unit tests covering your code additions. You can execute the tests by running: 25 | 26 | ```bash 27 | cargo test 28 | ``` 29 | 30 | All third-party contributions will be recognized in the list of contributors. 31 | 32 | -------------------------------------------------------------------------------- /.github/workflows/build_b3sum.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from pathlib import Path 4 | import platform 5 | import shutil 6 | import subprocess 7 | import sys 8 | 9 | ROOT = Path(__file__).parent.parent.parent 10 | RUST_TARGET = sys.argv[1] 11 | 12 | subprocess.run( 13 | ["cargo", "build", "--target", sys.argv[1], "--release"], cwd=ROOT / "b3sum" 14 | ) 15 | 16 | if platform.system() == "Windows": 17 | original_exe_name = "b3sum.exe" 18 | else: 19 | original_exe_name = "b3sum" 20 | 21 | if platform.system() == "Windows": 22 | new_exe_name = "b3sum_windows_x64_bin.exe" 23 | elif platform.system() == "Darwin": 24 | new_exe_name = "b3sum_macos_x64_bin" 25 | elif platform.system() == "Linux": 26 | new_exe_name = "b3sum_linux_x64_bin" 27 | else: 28 | raise RuntimeError("Unexpected platform: " + platform.system()) 29 | 30 | # Copy the built binary so that it has the upload name we want. 31 | out_dir = ROOT / "b3sum/target" / RUST_TARGET / "release" 32 | original_exe_path = str(out_dir / original_exe_name) 33 | new_exe_path = str(out_dir / new_exe_name) 34 | print("copying", repr(original_exe_path), "to", repr(new_exe_path)) 35 | shutil.copyfile(original_exe_path, new_exe_path) 36 | 37 | # This lets the subsequent upload step get the filepath. 38 | print("::set-output name=bin_path::" + new_exe_path) 39 | -------------------------------------------------------------------------------- /c/blake3_tbb.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | 6 | #include "blake3_impl.h" 7 | 8 | static_assert(TBB_USE_EXCEPTIONS == 0, 9 | "This file should be compiled with C++ exceptions disabled."); 10 | 11 | extern "C" void blake3_compress_subtree_wide_join_tbb( 12 | // shared params 13 | const uint32_t key[8], uint8_t flags, bool use_tbb, 14 | // left-hand side params 15 | const uint8_t *l_input, size_t l_input_len, uint64_t l_chunk_counter, 16 | uint8_t *l_cvs, size_t *l_n, 17 | // right-hand side params 18 | const uint8_t *r_input, size_t r_input_len, uint64_t r_chunk_counter, 19 | uint8_t *r_cvs, size_t *r_n) noexcept { 20 | if (!use_tbb) { 21 | *l_n = blake3_compress_subtree_wide(l_input, l_input_len, key, 22 | l_chunk_counter, flags, l_cvs, use_tbb); 23 | *r_n = blake3_compress_subtree_wide(r_input, r_input_len, key, 24 | r_chunk_counter, flags, r_cvs, use_tbb); 25 | return; 26 | } 27 | 28 | oneapi::tbb::parallel_invoke( 29 | [=]() { 30 | *l_n = blake3_compress_subtree_wide( 31 | l_input, l_input_len, key, l_chunk_counter, flags, l_cvs, use_tbb); 32 | }, 33 | [=]() { 34 | *r_n = blake3_compress_subtree_wide( 35 | r_input, r_input_len, key, r_chunk_counter, flags, r_cvs, use_tbb); 36 | }); 37 | } 38 | -------------------------------------------------------------------------------- /.github/workflows/tag.yml: -------------------------------------------------------------------------------- 1 | name: publish_b3sum_binaries 2 | 3 | on: 4 | push: 5 | tags: 6 | - "*" 7 | 8 | env: 9 | BLAKE3_CI: "1" 10 | RUSTFLAGS: "-D warnings" 11 | 12 | jobs: 13 | cargo_tests: 14 | name: ${{ matrix.target.name }} 15 | runs-on: ${{ matrix.target.os }} 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | target: [ 20 | { "os": "ubuntu-latest", "rust-target": "x86_64-unknown-linux-musl", "name": "Linux" }, 21 | { "os": "macOS-latest", "rust-target": "x86_64-apple-darwin", "name": "macOS" }, 22 | { "os": "windows-latest", "rust-target": "x86_64-pc-windows-msvc", "name": "Windows" }, 23 | ] 24 | 25 | steps: 26 | - uses: actions/checkout@v4 27 | - uses: actions/setup-python@v4 28 | with: 29 | python-version: "3.x" 30 | - run: pip install PyGithub 31 | - run: sudo apt-get install musl-tools 32 | if: matrix.target.os == 'ubuntu-latest' 33 | - uses: dtolnay/rust-toolchain@stable 34 | with: 35 | targets: ${{ matrix.target.rust-target }} 36 | - name: build b3sum 37 | id: build_b3sum 38 | run: python -u .github/workflows/build_b3sum.py ${{ matrix.target.rust-target }} 39 | - name: upload release asset 40 | env: 41 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 42 | GITHUB_TAG: ${{ github.ref }} 43 | run: python -u .github/workflows/upload_github_release_asset.py ${{ steps.build_b3sum.outputs.bin_path }} 44 | -------------------------------------------------------------------------------- /src/guts.rs: -------------------------------------------------------------------------------- 1 | //! Deprecated in favor of [`hazmat`](crate::hazmat) 2 | 3 | pub use crate::{BLOCK_LEN, CHUNK_LEN}; 4 | 5 | #[derive(Clone, Debug)] 6 | pub struct ChunkState(crate::ChunkState); 7 | 8 | impl ChunkState { 9 | // Currently this type only supports the regular hash mode. If an 10 | // incremental user needs keyed_hash or derive_key, we can add that. 11 | pub fn new(chunk_counter: u64) -> Self { 12 | Self(crate::ChunkState::new( 13 | crate::IV, 14 | chunk_counter, 15 | 0, 16 | crate::platform::Platform::detect(), 17 | )) 18 | } 19 | 20 | #[inline] 21 | pub fn len(&self) -> usize { 22 | self.0.count() 23 | } 24 | 25 | #[inline] 26 | pub fn update(&mut self, input: &[u8]) -> &mut Self { 27 | self.0.update(input); 28 | self 29 | } 30 | 31 | pub fn finalize(&self, is_root: bool) -> crate::Hash { 32 | let output = self.0.output(); 33 | if is_root { 34 | output.root_hash() 35 | } else { 36 | output.chaining_value().into() 37 | } 38 | } 39 | } 40 | 41 | // As above, this currently assumes the regular hash mode. If an incremental 42 | // user needs keyed_hash or derive_key, we can add that. 43 | pub fn parent_cv( 44 | left_child: &crate::Hash, 45 | right_child: &crate::Hash, 46 | is_root: bool, 47 | ) -> crate::Hash { 48 | let output = crate::parent_node_output( 49 | left_child.as_bytes(), 50 | right_child.as_bytes(), 51 | crate::IV, 52 | 0, 53 | crate::platform::Platform::detect(), 54 | ); 55 | if is_root { 56 | output.root_hash() 57 | } else { 58 | output.chaining_value().into() 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /c/example_tbb.c: -------------------------------------------------------------------------------- 1 | #include "blake3.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | int main(int argc, char **argv) { 11 | // For each filepath argument, memory map it and hash it. 12 | for (int i = 1; i < argc; i++) { 13 | // Open and memory map the file. 14 | int fd = open(argv[i], O_RDONLY); 15 | if (fd == -1) { 16 | fprintf(stderr, "open failed: %s\n", strerror(errno)); 17 | return 1; 18 | } 19 | struct stat statbuf; 20 | if (fstat(fd, &statbuf) == -1) { 21 | fprintf(stderr, "stat failed: %s\n", strerror(errno)); 22 | return 1; 23 | } 24 | void *mapped = mmap(NULL, statbuf.st_size, PROT_READ, MAP_PRIVATE, fd, 0); 25 | if (mapped == MAP_FAILED) { 26 | fprintf(stderr, "mmap failed: %s\n", strerror(errno)); 27 | return 1; 28 | } 29 | 30 | // Initialize the hasher. 31 | blake3_hasher hasher; 32 | blake3_hasher_init(&hasher); 33 | 34 | // Hash the mapped file using multiple threads. 35 | blake3_hasher_update_tbb(&hasher, mapped, statbuf.st_size); 36 | 37 | // Unmap and close the file. 38 | if (munmap(mapped, statbuf.st_size) == -1) { 39 | fprintf(stderr, "munmap failed: %s\n", strerror(errno)); 40 | return 1; 41 | } 42 | if (close(fd) == -1) { 43 | fprintf(stderr, "close failed: %s\n", strerror(errno)); 44 | return 1; 45 | } 46 | 47 | // Finalize the hash. BLAKE3_OUT_LEN is the default output length, 32 bytes. 48 | uint8_t output[BLAKE3_OUT_LEN]; 49 | blake3_hasher_finalize(&hasher, output, BLAKE3_OUT_LEN); 50 | 51 | // Print the hash as hexadecimal. 52 | for (size_t i = 0; i < BLAKE3_OUT_LEN; i++) { 53 | printf("%02x", output[i]); 54 | } 55 | printf("\n"); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/ffi_avx2.rs: -------------------------------------------------------------------------------- 1 | use crate::{CVWords, IncrementCounter, BLOCK_LEN, OUT_LEN}; 2 | 3 | // Note that there is no AVX2 implementation of compress_in_place or 4 | // compress_xof. 5 | 6 | // Unsafe because this may only be called on platforms supporting AVX2. 7 | pub unsafe fn hash_many( 8 | inputs: &[&[u8; N]], 9 | key: &CVWords, 10 | counter: u64, 11 | increment_counter: IncrementCounter, 12 | flags: u8, 13 | flags_start: u8, 14 | flags_end: u8, 15 | out: &mut [u8], 16 | ) { 17 | unsafe { 18 | // The Rust hash_many implementations do bounds checking on the `out` 19 | // array, but the C implementations don't. Even though this is an unsafe 20 | // function, assert the bounds here. 21 | assert!(out.len() >= inputs.len() * OUT_LEN); 22 | ffi::blake3_hash_many_avx2( 23 | inputs.as_ptr() as *const *const u8, 24 | inputs.len(), 25 | N / BLOCK_LEN, 26 | key.as_ptr(), 27 | counter, 28 | increment_counter.yes(), 29 | flags, 30 | flags_start, 31 | flags_end, 32 | out.as_mut_ptr(), 33 | ) 34 | } 35 | } 36 | 37 | pub mod ffi { 38 | extern "C" { 39 | pub fn blake3_hash_many_avx2( 40 | inputs: *const *const u8, 41 | num_inputs: usize, 42 | blocks: usize, 43 | key: *const u32, 44 | counter: u64, 45 | increment_counter: bool, 46 | flags: u8, 47 | flags_start: u8, 48 | flags_end: u8, 49 | out: *mut u8, 50 | ); 51 | } 52 | } 53 | 54 | #[cfg(test)] 55 | mod test { 56 | use super::*; 57 | 58 | #[test] 59 | fn test_hash_many() { 60 | if !crate::platform::avx2_detected() { 61 | return; 62 | } 63 | crate::test::test_hash_many_fn(hash_many, hash_many); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /c/CMakePresets.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 3, 3 | "cmakeMinimumRequired": { 4 | "major": 3, 5 | "minor": 22, 6 | "patch": 0 7 | }, 8 | "configurePresets": [ 9 | { 10 | "name": "base", 11 | "hidden": true, 12 | "binaryDir": "${sourceDir}/build/${presetName}" 13 | }, 14 | { 15 | "name": "msvc", 16 | "hidden": true, 17 | "generator": "Visual Studio 17 2022", 18 | "vendor": { 19 | "microsoft.com/VisualStudioSettings/CMake/1.0": { 20 | "hostOS": [ 21 | "Windows" 22 | ] 23 | } 24 | } 25 | }, 26 | { 27 | "name": "x64-windows-msvc", 28 | "inherits": [ 29 | "msvc", 30 | "base" 31 | ], 32 | "architecture": "x64" 33 | }, 34 | { 35 | "name": "x86-windows-msvc", 36 | "inherits": [ 37 | "msvc", 38 | "base" 39 | ], 40 | "architecture": "Win32" 41 | }, 42 | { 43 | "name": "arm64-windows-msvc", 44 | "inherits": [ 45 | "msvc", 46 | "base" 47 | ], 48 | "architecture": "ARM64" 49 | } 50 | ], 51 | "buildPresets": [ 52 | { 53 | "name": "x64-windows-msvc-debug", 54 | "configurePreset": "x64-windows-msvc", 55 | "configuration": "Debug" 56 | }, 57 | { 58 | "name": "x64-windows-msvc-release", 59 | "configurePreset": "x64-windows-msvc", 60 | "configuration": "RelWithDebInfo" 61 | }, 62 | { 63 | "name": "x86-windows-msvc-debug", 64 | "configurePreset": "x86-windows-msvc", 65 | "configuration": "Debug" 66 | }, 67 | { 68 | "name": "x86-windows-msvc-release", 69 | "configurePreset": "x86-windows-msvc", 70 | "configuration": "RelWithDebInfo" 71 | } 72 | ] 73 | } -------------------------------------------------------------------------------- /c/Makefile.testing: -------------------------------------------------------------------------------- 1 | # This Makefile is only for testing. C callers should follow the instructions 2 | # in ./README.md to incorporate these C files into their existing build. 3 | 4 | NAME=blake3 5 | CC=gcc 6 | CFLAGS=-O3 -Wall -Wextra -std=c11 -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2 -fPIE -fvisibility=hidden 7 | LDFLAGS=-pie -Wl,-z,relro,-z,now 8 | TARGETS= 9 | ASM_TARGETS= 10 | EXTRAFLAGS=-Wa,--noexecstack 11 | 12 | ifdef BLAKE3_NO_SSE2 13 | EXTRAFLAGS += -DBLAKE3_NO_SSE2 14 | else 15 | TARGETS += blake3_sse2.o 16 | ASM_TARGETS += blake3_sse2_x86-64_unix.S 17 | endif 18 | 19 | ifdef BLAKE3_NO_SSE41 20 | EXTRAFLAGS += -DBLAKE3_NO_SSE41 21 | else 22 | TARGETS += blake3_sse41.o 23 | ASM_TARGETS += blake3_sse41_x86-64_unix.S 24 | endif 25 | 26 | ifdef BLAKE3_NO_AVX2 27 | EXTRAFLAGS += -DBLAKE3_NO_AVX2 28 | else 29 | TARGETS += blake3_avx2.o 30 | ASM_TARGETS += blake3_avx2_x86-64_unix.S 31 | endif 32 | 33 | ifdef BLAKE3_NO_AVX512 34 | EXTRAFLAGS += -DBLAKE3_NO_AVX512 35 | else 36 | TARGETS += blake3_avx512.o 37 | ASM_TARGETS += blake3_avx512_x86-64_unix.S 38 | endif 39 | 40 | ifdef BLAKE3_USE_NEON 41 | EXTRAFLAGS += -DBLAKE3_USE_NEON=1 42 | TARGETS += blake3_neon.o 43 | endif 44 | 45 | ifdef BLAKE3_NO_NEON 46 | EXTRAFLAGS += -DBLAKE3_USE_NEON=0 47 | endif 48 | 49 | all: blake3.c blake3_dispatch.c blake3_portable.c main.c $(TARGETS) 50 | $(CC) $(CFLAGS) $(EXTRAFLAGS) $^ -o $(NAME) $(LDFLAGS) 51 | 52 | blake3_sse2.o: blake3_sse2.c 53 | $(CC) $(CFLAGS) $(EXTRAFLAGS) -c $^ -o $@ -msse2 54 | 55 | blake3_sse41.o: blake3_sse41.c 56 | $(CC) $(CFLAGS) $(EXTRAFLAGS) -c $^ -o $@ -msse4.1 57 | 58 | blake3_avx2.o: blake3_avx2.c 59 | $(CC) $(CFLAGS) $(EXTRAFLAGS) -c $^ -o $@ -mavx2 60 | 61 | blake3_avx512.o: blake3_avx512.c 62 | $(CC) $(CFLAGS) $(EXTRAFLAGS) -c $^ -o $@ -mavx512f -mavx512vl 63 | 64 | blake3_neon.o: blake3_neon.c 65 | $(CC) $(CFLAGS) $(EXTRAFLAGS) -c $^ -o $@ 66 | 67 | test: CFLAGS += -DBLAKE3_TESTING -fsanitize=address,undefined 68 | test: all 69 | ./test.py 70 | 71 | asm: blake3.c blake3_dispatch.c blake3_portable.c main.c $(ASM_TARGETS) 72 | $(CC) $(CFLAGS) $(EXTRAFLAGS) $^ -o $(NAME) $(LDFLAGS) 73 | 74 | test_asm: CFLAGS += -DBLAKE3_TESTING -fsanitize=address,undefined 75 | test_asm: asm 76 | ./test.py 77 | 78 | example: example.c blake3.c blake3_dispatch.c blake3_portable.c $(ASM_TARGETS) 79 | $(CC) $(CFLAGS) $(EXTRAFLAGS) $^ -o $@ $(LDFLAGS) 80 | 81 | clean: 82 | rm -f $(NAME) *.o 83 | -------------------------------------------------------------------------------- /src/ffi_neon.rs: -------------------------------------------------------------------------------- 1 | use crate::{CVWords, IncrementCounter, BLOCK_LEN, OUT_LEN}; 2 | 3 | // Unsafe because this may only be called on platforms supporting NEON. 4 | pub unsafe fn hash_many( 5 | inputs: &[&[u8; N]], 6 | key: &CVWords, 7 | counter: u64, 8 | increment_counter: IncrementCounter, 9 | flags: u8, 10 | flags_start: u8, 11 | flags_end: u8, 12 | out: &mut [u8], 13 | ) { 14 | // The Rust hash_many implementations do bounds checking on the `out` 15 | // array, but the C implementations don't. Even though this is an unsafe 16 | // function, assert the bounds here. 17 | assert!(out.len() >= inputs.len() * OUT_LEN); 18 | unsafe { 19 | ffi::blake3_hash_many_neon( 20 | inputs.as_ptr() as *const *const u8, 21 | inputs.len(), 22 | N / BLOCK_LEN, 23 | key.as_ptr(), 24 | counter, 25 | increment_counter.yes(), 26 | flags, 27 | flags_start, 28 | flags_end, 29 | out.as_mut_ptr(), 30 | ) 31 | } 32 | } 33 | 34 | // blake3_neon.c normally depends on blake3_portable.c, because the NEON 35 | // implementation only provides 4x compression, and it relies on the portable 36 | // implementation for 1x compression. However, we expose the portable Rust 37 | // implementation here instead, to avoid linking in unnecessary code. 38 | #[no_mangle] 39 | pub extern "C" fn blake3_compress_in_place_portable( 40 | cv: *mut u32, 41 | block: *const u8, 42 | block_len: u8, 43 | counter: u64, 44 | flags: u8, 45 | ) { 46 | unsafe { 47 | crate::portable::compress_in_place( 48 | &mut *(cv as *mut [u32; 8]), 49 | &*(block as *const [u8; 64]), 50 | block_len, 51 | counter, 52 | flags, 53 | ) 54 | } 55 | } 56 | 57 | pub mod ffi { 58 | extern "C" { 59 | pub fn blake3_hash_many_neon( 60 | inputs: *const *const u8, 61 | num_inputs: usize, 62 | blocks: usize, 63 | key: *const u32, 64 | counter: u64, 65 | increment_counter: bool, 66 | flags: u8, 67 | flags_start: u8, 68 | flags_end: u8, 69 | out: *mut u8, 70 | ); 71 | } 72 | } 73 | 74 | #[cfg(test)] 75 | mod test { 76 | use super::*; 77 | 78 | #[test] 79 | fn test_hash_many() { 80 | // This entire file is gated on feature="neon", so NEON support is 81 | // assumed here. 82 | crate::test::test_hash_many_fn(hash_many, hash_many); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /.github/workflows/upload_github_release_asset.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import github 4 | import os 5 | import sys 6 | import time 7 | 8 | RETRIES = 10 9 | 10 | g = github.Github(os.environ["GITHUB_TOKEN"]) 11 | tag_name = os.environ["GITHUB_TAG"] 12 | tag_prefix = "refs/tags/" 13 | if tag_name.startswith(tag_prefix): 14 | tag_name = tag_name[len(tag_prefix) :] 15 | assert len(sys.argv) == 2 16 | asset_path = sys.argv[1] 17 | asset_name = os.path.basename(asset_path) 18 | 19 | repo = g.get_repo(os.environ["GITHUB_REPOSITORY"]) 20 | 21 | tags = list(repo.get_tags()) 22 | 23 | for tag in tags: 24 | if tag.name == tag_name: 25 | break 26 | else: 27 | raise RuntimeError("no tag named " + repr(tag_name)) 28 | 29 | try: 30 | print("Creating GitHub release for tag " + repr(tag_name) + "...") 31 | repo.create_git_release(tag_name, tag_name, tag.commit.commit.message) 32 | except github.GithubException as github_error: 33 | if github_error.data["errors"][0]["code"] == "already_exists": 34 | print("Release for tag " + repr(tag_name) + " already exists.") 35 | else: 36 | raise 37 | 38 | 39 | def get_release(): 40 | for i in range(RETRIES): 41 | releases = list(repo.get_releases()) 42 | for release in releases: 43 | if release.tag_name == tag_name: 44 | return release 45 | print(f"Release for tag {repr(tag_name)} not found. Retrying...") 46 | time.sleep(1) 47 | raise RuntimeError("no release for tag " + repr(tag_name)) 48 | 49 | 50 | release = get_release() 51 | 52 | print("Uploading " + repr(asset_path) + "...") 53 | for i in range(RETRIES): 54 | try: 55 | print("Upload attempt #{} of {}...".format(i + 1, RETRIES)) 56 | release.upload_asset(asset_path) 57 | break 58 | except github.GithubException as github_error: 59 | # Unfortunately the asset upload API is flaky. Even worse, it often 60 | # partially succeeds, returning an error to the caller but leaving the 61 | # release in a state where subsequent uploads of the same asset will 62 | # fail with an "already_exists" error. (Though the asset is not visible 63 | # on github.com, so we can't just declare victory and move on.) If we 64 | # detect this case, explicitly delete the asset and continue retrying. 65 | print(github_error) 66 | for asset in release.get_assets(): 67 | if asset.name == asset_name: 68 | print("Found uploaded asset after failure. Deleting...") 69 | asset.delete_asset() 70 | else: 71 | raise RuntimeError("All upload attempts failed.") 72 | 73 | print("Success!") 74 | -------------------------------------------------------------------------------- /b3sum/README.md: -------------------------------------------------------------------------------- 1 | # b3sum 2 | 3 | A command line utility for calculating 4 | [BLAKE3](https://github.com/BLAKE3-team/BLAKE3) hashes, similar to 5 | Coreutils tools like `b2sum` or `md5sum`. 6 | 7 | ``` 8 | Usage: b3sum [OPTIONS] [FILE]... 9 | 10 | Arguments: 11 | [FILE]... Files to hash, or checkfiles to check 12 | 13 | Options: 14 | --keyed Use the keyed mode, reading the 32-byte key from stdin 15 | --derive-key Use the key derivation mode, with the given context string 16 | -l, --length The number of output bytes, before hex encoding [default: 32] 17 | --seek The starting output byte offset, before hex encoding [default: 0] 18 | --num-threads The maximum number of threads to use 19 | --no-mmap Disable memory mapping 20 | --no-names Omit filenames in the output 21 | --raw Write raw output bytes to stdout, rather than hex 22 | --tag Output BSD-style checksums: BLAKE3 ([FILE]) = [HASH] 23 | -c, --check Read BLAKE3 sums from the [FILE]s and check them 24 | --quiet Skip printing OK for each checked file 25 | -h, --help Print help (see more with '--help') 26 | -V, --version Print version 27 | ``` 28 | 29 | See also [this document about how the `--check` flag 30 | works](https://github.com/BLAKE3-team/BLAKE3/blob/master/b3sum/what_does_check_do.md). 31 | 32 | # Example 33 | 34 | Hash the file `foo.txt`: 35 | 36 | ```bash 37 | b3sum foo.txt 38 | ``` 39 | 40 | Time hashing a gigabyte of data, to see how fast it is: 41 | 42 | ```bash 43 | # Create a 1 GB file. 44 | head -c 1000000000 /dev/zero > /tmp/bigfile 45 | # Hash it with SHA-256. 46 | time openssl sha256 /tmp/bigfile 47 | # Hash it with BLAKE3. 48 | time b3sum /tmp/bigfile 49 | ``` 50 | 51 | 52 | # Installation 53 | 54 | Prebuilt binaries are available for Linux, Windows, and macOS (requiring 55 | the [unidentified developer 56 | workaround](https://support.apple.com/guide/mac-help/open-a-mac-app-from-an-unidentified-developer-mh40616/mac)) 57 | on the [releases page](https://github.com/BLAKE3-team/BLAKE3/releases). 58 | If you've [installed Rust and 59 | Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html), 60 | you can also build `b3sum` yourself with: 61 | 62 | ``` 63 | cargo install b3sum 64 | ``` 65 | 66 | On Linux for example, Cargo will put the compiled binary in 67 | `~/.cargo/bin`. You might want to add that directory to your `$PATH`, or 68 | `rustup` might have done it for you when you installed Cargo. 69 | 70 | If you want to install directly from this directory, you can run `cargo 71 | install --path .`. Or you can just build with `cargo build --release`, 72 | which puts the binary at `./target/release/b3sum`. 73 | -------------------------------------------------------------------------------- /c/blake3.h: -------------------------------------------------------------------------------- 1 | #ifndef BLAKE3_H 2 | #define BLAKE3_H 3 | 4 | #include 5 | #include 6 | 7 | #if !defined(BLAKE3_API) 8 | # if defined(_WIN32) || defined(__CYGWIN__) 9 | # if defined(BLAKE3_DLL) 10 | # if defined(BLAKE3_DLL_EXPORTS) 11 | # define BLAKE3_API __declspec(dllexport) 12 | # else 13 | # define BLAKE3_API __declspec(dllimport) 14 | # endif 15 | # define BLAKE3_PRIVATE 16 | # else 17 | # define BLAKE3_API 18 | # define BLAKE3_PRIVATE 19 | # endif 20 | # elif __GNUC__ >= 4 21 | # define BLAKE3_API __attribute__((visibility("default"))) 22 | # define BLAKE3_PRIVATE __attribute__((visibility("hidden"))) 23 | # else 24 | # define BLAKE3_API 25 | # define BLAKE3_PRIVATE 26 | # endif 27 | #endif 28 | 29 | #ifdef __cplusplus 30 | extern "C" { 31 | #endif 32 | 33 | #define BLAKE3_VERSION_STRING "1.8.2" 34 | #define BLAKE3_KEY_LEN 32 35 | #define BLAKE3_OUT_LEN 32 36 | #define BLAKE3_BLOCK_LEN 64 37 | #define BLAKE3_CHUNK_LEN 1024 38 | #define BLAKE3_MAX_DEPTH 54 39 | 40 | // This struct is a private implementation detail. It has to be here because 41 | // it's part of the blake3_hasher structure defined below. 42 | typedef struct { 43 | uint32_t cv[8]; 44 | uint64_t chunk_counter; 45 | uint8_t buf[BLAKE3_BLOCK_LEN]; 46 | uint8_t buf_len; 47 | uint8_t blocks_compressed; 48 | uint8_t flags; 49 | } blake3_chunk_state; 50 | 51 | typedef struct { 52 | uint32_t key[8]; 53 | blake3_chunk_state chunk; 54 | uint8_t cv_stack_len; 55 | // The stack size is MAX_DEPTH + 1 because we do lazy merging. For example, 56 | // with 7 chunks, we have 3 entries in the stack. Adding an 8th chunk 57 | // requires a 4th entry, rather than merging everything down to 1, because we 58 | // don't know whether more input is coming. This is different from how the 59 | // reference implementation does things. 60 | uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN]; 61 | } blake3_hasher; 62 | 63 | BLAKE3_API const char *blake3_version(void); 64 | BLAKE3_API void blake3_hasher_init(blake3_hasher *self); 65 | BLAKE3_API void blake3_hasher_init_keyed(blake3_hasher *self, 66 | const uint8_t key[BLAKE3_KEY_LEN]); 67 | BLAKE3_API void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context); 68 | BLAKE3_API void blake3_hasher_init_derive_key_raw(blake3_hasher *self, const void *context, 69 | size_t context_len); 70 | BLAKE3_API void blake3_hasher_update(blake3_hasher *self, const void *input, 71 | size_t input_len); 72 | #if defined(BLAKE3_USE_TBB) 73 | BLAKE3_API void blake3_hasher_update_tbb(blake3_hasher *self, const void *input, 74 | size_t input_len); 75 | #endif // BLAKE3_USE_TBB 76 | BLAKE3_API void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, 77 | size_t out_len); 78 | BLAKE3_API void blake3_hasher_finalize_seek(const blake3_hasher *self, uint64_t seek, 79 | uint8_t *out, size_t out_len); 80 | BLAKE3_API void blake3_hasher_reset(blake3_hasher *self); 81 | 82 | #ifdef __cplusplus 83 | } 84 | #endif 85 | 86 | #endif /* BLAKE3_H */ 87 | -------------------------------------------------------------------------------- /src/join.rs: -------------------------------------------------------------------------------- 1 | //! The multi-threading abstractions used by `Hasher::update_with_join`. 2 | //! 3 | //! Different implementations of the `Join` trait determine whether 4 | //! `Hasher::update_with_join` performs multi-threading on sufficiently large 5 | //! inputs. The `SerialJoin` implementation is single-threaded, and the 6 | //! `RayonJoin` implementation (gated by the `rayon` feature) is multi-threaded. 7 | //! Interfaces other than `Hasher::update_with_join`, like [`hash`](crate::hash) 8 | //! and [`Hasher::update`](crate::Hasher::update), always use `SerialJoin` 9 | //! internally. 10 | //! 11 | //! The `Join` trait is an almost exact copy of the [`rayon::join`] API, and 12 | //! `RayonJoin` is the only non-trivial implementation. Previously this trait 13 | //! was public, but currently it's been re-privatized, as it's both 1) of no 14 | //! value to most callers and 2) a pretty big implementation detail to commit 15 | //! to. 16 | //! 17 | //! [`rayon::join`]: https://docs.rs/rayon/1.3.0/rayon/fn.join.html 18 | 19 | /// The trait that abstracts over single-threaded and multi-threaded recursion. 20 | /// 21 | /// See the [`join` module docs](index.html) for more details. 22 | pub trait Join { 23 | fn join(oper_a: A, oper_b: B) -> (RA, RB) 24 | where 25 | A: FnOnce() -> RA + Send, 26 | B: FnOnce() -> RB + Send, 27 | RA: Send, 28 | RB: Send; 29 | } 30 | 31 | /// The trivial, serial implementation of `Join`. The left and right sides are 32 | /// executed one after the other, on the calling thread. The standalone hashing 33 | /// functions and the `Hasher::update` method use this implementation 34 | /// internally. 35 | /// 36 | /// See the [`join` module docs](index.html) for more details. 37 | pub enum SerialJoin {} 38 | 39 | impl Join for SerialJoin { 40 | #[inline] 41 | fn join(oper_a: A, oper_b: B) -> (RA, RB) 42 | where 43 | A: FnOnce() -> RA + Send, 44 | B: FnOnce() -> RB + Send, 45 | RA: Send, 46 | RB: Send, 47 | { 48 | (oper_a(), oper_b()) 49 | } 50 | } 51 | 52 | /// The Rayon-based implementation of `Join`. The left and right sides are 53 | /// executed on the Rayon thread pool, potentially in parallel. This 54 | /// implementation is gated by the `rayon` feature, which is off by default. 55 | /// 56 | /// See the [`join` module docs](index.html) for more details. 57 | #[cfg(feature = "rayon")] 58 | pub enum RayonJoin {} 59 | 60 | #[cfg(feature = "rayon")] 61 | impl Join for RayonJoin { 62 | #[inline] 63 | fn join(oper_a: A, oper_b: B) -> (RA, RB) 64 | where 65 | A: FnOnce() -> RA + Send, 66 | B: FnOnce() -> RB + Send, 67 | RA: Send, 68 | RB: Send, 69 | { 70 | rayon_core::join(oper_a, oper_b) 71 | } 72 | } 73 | 74 | #[cfg(test)] 75 | mod test { 76 | use super::*; 77 | 78 | #[test] 79 | fn test_serial_join() { 80 | let oper_a = || 1 + 1; 81 | let oper_b = || 2 + 2; 82 | assert_eq!((2, 4), SerialJoin::join(oper_a, oper_b)); 83 | } 84 | 85 | #[test] 86 | #[cfg(feature = "rayon")] 87 | fn test_rayon_join() { 88 | let oper_a = || 1 + 1; 89 | let oper_b = || 2 + 2; 90 | assert_eq!((2, 4), RayonJoin::join(oper_a, oper_b)); 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/io.rs: -------------------------------------------------------------------------------- 1 | //! Helper functions for efficient IO. 2 | 3 | #[cfg(feature = "std")] 4 | pub(crate) fn copy_wide( 5 | mut reader: impl std::io::Read, 6 | hasher: &mut crate::Hasher, 7 | ) -> std::io::Result { 8 | let mut buffer = [0; 65536]; 9 | let mut total = 0; 10 | loop { 11 | match reader.read(&mut buffer) { 12 | Ok(0) => return Ok(total), 13 | Ok(n) => { 14 | hasher.update(&buffer[..n]); 15 | total += n as u64; 16 | } 17 | // see test_update_reader_interrupted 18 | Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, 19 | Err(e) => return Err(e), 20 | } 21 | } 22 | } 23 | 24 | // Mmap a file, if it looks like a good idea. Return None in cases where we know mmap will fail, or 25 | // if the file is short enough that mmapping isn't worth it. However, if we do try to mmap and it 26 | // fails, return the error. 27 | // 28 | // SAFETY: Mmaps are fundamentally unsafe, because you can call invariant-checking functions like 29 | // str::from_utf8 on them and then have them change out from under you. Letting a safe caller get 30 | // their hands on an mmap, or even a &[u8] that's backed by an mmap, is unsound. However, because 31 | // this function is crate-private, we can guarantee that all can ever happen in the event of a race 32 | // condition is that we either hash nonsense bytes or crash with SIGBUS or similar, neither of 33 | // which should risk memory corruption in a safe caller. 34 | // 35 | // PARANOIA: But a data race...is a data race...is a data race...right? Even if we know that no 36 | // platform in the "real world" is ever going to do anything other than compute the "wrong answer" 37 | // if we race on this mmap while we hash it, aren't we still supposed to feel bad about doing this? 38 | // Well, maybe. This is IO, and IO gets special carve-outs in the memory model. Consider a 39 | // memory-mapped register that returns random 32-bit words. (This is actually realistic if you have 40 | // a hardware RNG.) It's probably sound to construct a *const i32 pointing to that register and do 41 | // some raw pointer reads from it. Those reads should be volatile if you don't want the compiler to 42 | // coalesce them, but either way the compiler isn't allowed to just _go nuts_ and insert 43 | // should-never-happen branches to wipe your hard drive if two adjacent reads happen to give 44 | // different values. As far as I'm aware, there's no such thing as a read that's allowed if it's 45 | // volatile but prohibited if it's not (unlike atomics). As mentioned above, it's not ok to 46 | // construct a safe &i32 to the register if you're going to leak that reference to unknown callers. 47 | // But if you "know what you're doing," I don't think *const i32 and &i32 are fundamentally 48 | // different here. Feedback needed. 49 | #[cfg(feature = "mmap")] 50 | pub(crate) fn maybe_mmap_file(file: &std::fs::File) -> std::io::Result> { 51 | let metadata = file.metadata()?; 52 | let file_size = metadata.len(); 53 | if !metadata.is_file() { 54 | // Not a real file. 55 | Ok(None) 56 | } else if file_size < 16 * 1024 { 57 | // Mapping small files is not worth it, and some special files that can't be mapped report 58 | // a size of zero. 59 | Ok(None) 60 | } else { 61 | let map = unsafe { memmap2::Mmap::map(file)? }; 62 | Ok(Some(map)) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/ffi_sse2.rs: -------------------------------------------------------------------------------- 1 | use crate::{CVWords, IncrementCounter, BLOCK_LEN, OUT_LEN}; 2 | 3 | // Unsafe because this may only be called on platforms supporting SSE2. 4 | pub unsafe fn compress_in_place( 5 | cv: &mut CVWords, 6 | block: &[u8; BLOCK_LEN], 7 | block_len: u8, 8 | counter: u64, 9 | flags: u8, 10 | ) { 11 | unsafe { 12 | ffi::blake3_compress_in_place_sse2( 13 | cv.as_mut_ptr(), 14 | block.as_ptr(), 15 | block_len, 16 | counter, 17 | flags, 18 | ) 19 | } 20 | } 21 | 22 | // Unsafe because this may only be called on platforms supporting SSE2. 23 | pub unsafe fn compress_xof( 24 | cv: &CVWords, 25 | block: &[u8; BLOCK_LEN], 26 | block_len: u8, 27 | counter: u64, 28 | flags: u8, 29 | ) -> [u8; 64] { 30 | unsafe { 31 | let mut out = [0u8; 64]; 32 | ffi::blake3_compress_xof_sse2( 33 | cv.as_ptr(), 34 | block.as_ptr(), 35 | block_len, 36 | counter, 37 | flags, 38 | out.as_mut_ptr(), 39 | ); 40 | out 41 | } 42 | } 43 | 44 | // Unsafe because this may only be called on platforms supporting SSE2. 45 | pub unsafe fn hash_many( 46 | inputs: &[&[u8; N]], 47 | key: &CVWords, 48 | counter: u64, 49 | increment_counter: IncrementCounter, 50 | flags: u8, 51 | flags_start: u8, 52 | flags_end: u8, 53 | out: &mut [u8], 54 | ) { 55 | unsafe { 56 | // The Rust hash_many implementations do bounds checking on the `out` 57 | // array, but the C implementations don't. Even though this is an unsafe 58 | // function, assert the bounds here. 59 | assert!(out.len() >= inputs.len() * OUT_LEN); 60 | ffi::blake3_hash_many_sse2( 61 | inputs.as_ptr() as *const *const u8, 62 | inputs.len(), 63 | N / BLOCK_LEN, 64 | key.as_ptr(), 65 | counter, 66 | increment_counter.yes(), 67 | flags, 68 | flags_start, 69 | flags_end, 70 | out.as_mut_ptr(), 71 | ) 72 | } 73 | } 74 | 75 | pub mod ffi { 76 | extern "C" { 77 | pub fn blake3_compress_in_place_sse2( 78 | cv: *mut u32, 79 | block: *const u8, 80 | block_len: u8, 81 | counter: u64, 82 | flags: u8, 83 | ); 84 | pub fn blake3_compress_xof_sse2( 85 | cv: *const u32, 86 | block: *const u8, 87 | block_len: u8, 88 | counter: u64, 89 | flags: u8, 90 | out: *mut u8, 91 | ); 92 | pub fn blake3_hash_many_sse2( 93 | inputs: *const *const u8, 94 | num_inputs: usize, 95 | blocks: usize, 96 | key: *const u32, 97 | counter: u64, 98 | increment_counter: bool, 99 | flags: u8, 100 | flags_start: u8, 101 | flags_end: u8, 102 | out: *mut u8, 103 | ); 104 | } 105 | } 106 | 107 | #[cfg(test)] 108 | mod test { 109 | use super::*; 110 | 111 | #[test] 112 | fn test_compress() { 113 | if !crate::platform::sse2_detected() { 114 | return; 115 | } 116 | crate::test::test_compress_fn(compress_in_place, compress_xof); 117 | } 118 | 119 | #[test] 120 | fn test_hash_many() { 121 | if !crate::platform::sse2_detected() { 122 | return; 123 | } 124 | crate::test::test_hash_many_fn(hash_many, hash_many); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/ffi_sse41.rs: -------------------------------------------------------------------------------- 1 | use crate::{CVWords, IncrementCounter, BLOCK_LEN, OUT_LEN}; 2 | 3 | // Unsafe because this may only be called on platforms supporting SSE4.1. 4 | pub unsafe fn compress_in_place( 5 | cv: &mut CVWords, 6 | block: &[u8; BLOCK_LEN], 7 | block_len: u8, 8 | counter: u64, 9 | flags: u8, 10 | ) { 11 | unsafe { 12 | ffi::blake3_compress_in_place_sse41( 13 | cv.as_mut_ptr(), 14 | block.as_ptr(), 15 | block_len, 16 | counter, 17 | flags, 18 | ) 19 | } 20 | } 21 | 22 | // Unsafe because this may only be called on platforms supporting SSE4.1. 23 | pub unsafe fn compress_xof( 24 | cv: &CVWords, 25 | block: &[u8; BLOCK_LEN], 26 | block_len: u8, 27 | counter: u64, 28 | flags: u8, 29 | ) -> [u8; 64] { 30 | unsafe { 31 | let mut out = [0u8; 64]; 32 | ffi::blake3_compress_xof_sse41( 33 | cv.as_ptr(), 34 | block.as_ptr(), 35 | block_len, 36 | counter, 37 | flags, 38 | out.as_mut_ptr(), 39 | ); 40 | out 41 | } 42 | } 43 | 44 | // Unsafe because this may only be called on platforms supporting SSE4.1. 45 | pub unsafe fn hash_many( 46 | inputs: &[&[u8; N]], 47 | key: &CVWords, 48 | counter: u64, 49 | increment_counter: IncrementCounter, 50 | flags: u8, 51 | flags_start: u8, 52 | flags_end: u8, 53 | out: &mut [u8], 54 | ) { 55 | unsafe { 56 | // The Rust hash_many implementations do bounds checking on the `out` 57 | // array, but the C implementations don't. Even though this is an unsafe 58 | // function, assert the bounds here. 59 | assert!(out.len() >= inputs.len() * OUT_LEN); 60 | ffi::blake3_hash_many_sse41( 61 | inputs.as_ptr() as *const *const u8, 62 | inputs.len(), 63 | N / BLOCK_LEN, 64 | key.as_ptr(), 65 | counter, 66 | increment_counter.yes(), 67 | flags, 68 | flags_start, 69 | flags_end, 70 | out.as_mut_ptr(), 71 | ) 72 | } 73 | } 74 | 75 | pub mod ffi { 76 | extern "C" { 77 | pub fn blake3_compress_in_place_sse41( 78 | cv: *mut u32, 79 | block: *const u8, 80 | block_len: u8, 81 | counter: u64, 82 | flags: u8, 83 | ); 84 | pub fn blake3_compress_xof_sse41( 85 | cv: *const u32, 86 | block: *const u8, 87 | block_len: u8, 88 | counter: u64, 89 | flags: u8, 90 | out: *mut u8, 91 | ); 92 | pub fn blake3_hash_many_sse41( 93 | inputs: *const *const u8, 94 | num_inputs: usize, 95 | blocks: usize, 96 | key: *const u32, 97 | counter: u64, 98 | increment_counter: bool, 99 | flags: u8, 100 | flags_start: u8, 101 | flags_end: u8, 102 | out: *mut u8, 103 | ); 104 | } 105 | } 106 | 107 | #[cfg(test)] 108 | mod test { 109 | use super::*; 110 | 111 | #[test] 112 | fn test_compress() { 113 | if !crate::platform::sse41_detected() { 114 | return; 115 | } 116 | crate::test::test_compress_fn(compress_in_place, compress_xof); 117 | } 118 | 119 | #[test] 120 | fn test_hash_many() { 121 | if !crate::platform::sse41_detected() { 122 | return; 123 | } 124 | crate::test::test_hash_many_fn(hash_many, hash_many); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /c/test.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from binascii import hexlify 4 | import json 5 | from os import path 6 | import subprocess 7 | 8 | HERE = path.dirname(__file__) 9 | TEST_VECTORS_PATH = path.join(HERE, "..", "test_vectors", "test_vectors.json") 10 | TEST_VECTORS = json.load(open(TEST_VECTORS_PATH)) 11 | 12 | 13 | def run_blake3(args, input): 14 | output = subprocess.run([path.join(HERE, "blake3")] + args, 15 | input=input, 16 | stdout=subprocess.PIPE, 17 | check=True) 18 | return output.stdout.decode().strip() 19 | 20 | 21 | # Fill the input with a repeating byte pattern. We use a cycle length of 251, 22 | # because that's the largest prime number less than 256. This makes it unlikely 23 | # to swapping any two adjacent input blocks or chunks will give the same 24 | # answer. 25 | def make_test_input(length): 26 | i = 0 27 | buf = bytearray() 28 | while len(buf) < length: 29 | buf.append(i) 30 | i = (i + 1) % 251 31 | return buf 32 | 33 | 34 | def main(): 35 | for case in TEST_VECTORS["cases"]: 36 | input_len = case["input_len"] 37 | input = make_test_input(input_len) 38 | hex_key = hexlify(TEST_VECTORS["key"].encode()) 39 | context_string = TEST_VECTORS["context_string"] 40 | expected_hash_xof = case["hash"] 41 | expected_hash = expected_hash_xof[:64] 42 | expected_keyed_hash_xof = case["keyed_hash"] 43 | expected_keyed_hash = expected_keyed_hash_xof[:64] 44 | expected_derive_key_xof = case["derive_key"] 45 | expected_derive_key = expected_derive_key_xof[:64] 46 | 47 | # Test the default hash. 48 | test_hash = run_blake3([], input) 49 | for line in test_hash.splitlines(): 50 | assert expected_hash == line, \ 51 | "hash({}): {} != {}".format(input_len, expected_hash, line) 52 | 53 | # Test the extended hash. 54 | xof_len = len(expected_hash_xof) // 2 55 | test_hash_xof = run_blake3(["--length", str(xof_len)], input) 56 | for line in test_hash_xof.splitlines(): 57 | assert expected_hash_xof == line, \ 58 | "hash_xof({}): {} != {}".format( 59 | input_len, expected_hash_xof, line) 60 | 61 | # Test the default keyed hash. 62 | test_keyed_hash = run_blake3(["--keyed", hex_key], input) 63 | for line in test_keyed_hash.splitlines(): 64 | assert expected_keyed_hash == line, \ 65 | "keyed_hash({}): {} != {}".format( 66 | input_len, expected_keyed_hash, line) 67 | 68 | # Test the extended keyed hash. 69 | xof_len = len(expected_keyed_hash_xof) // 2 70 | test_keyed_hash_xof = run_blake3( 71 | ["--keyed", hex_key, "--length", 72 | str(xof_len)], input) 73 | for line in test_keyed_hash_xof.splitlines(): 74 | assert expected_keyed_hash_xof == line, \ 75 | "keyed_hash_xof({}): {} != {}".format( 76 | input_len, expected_keyed_hash_xof, line) 77 | 78 | # Test the default derive key. 79 | test_derive_key = run_blake3(["--derive-key", context_string], input) 80 | for line in test_derive_key.splitlines(): 81 | assert expected_derive_key == line, \ 82 | "derive_key({}): {} != {}".format( 83 | input_len, expected_derive_key, line) 84 | 85 | # Test the extended derive key. 86 | xof_len = len(expected_derive_key_xof) // 2 87 | test_derive_key_xof = run_blake3( 88 | ["--derive-key", context_string, "--length", 89 | str(xof_len)], input) 90 | for line in test_derive_key_xof.splitlines(): 91 | assert expected_derive_key_xof == line, \ 92 | "derive_key_xof({}): {} != {}".format( 93 | input_len, expected_derive_key_xof, line) 94 | 95 | 96 | if __name__ == "__main__": 97 | main() 98 | -------------------------------------------------------------------------------- /media/B3.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 21 | 39 | 41 | 42 | 44 | image/svg+xml 45 | 47 | 48 | 49 | 50 | 51 | 56 | 60 | 64 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /c/main.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This main file is intended for testing via `make test`. It does not build in 3 | * other settings. See README.md in this directory for examples of how to build 4 | * C code. 5 | */ 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "blake3.h" 15 | #include "blake3_impl.h" 16 | 17 | #define HASH_MODE 0 18 | #define KEYED_HASH_MODE 1 19 | #define DERIVE_KEY_MODE 2 20 | 21 | static void hex_char_value(uint8_t c, uint8_t *value, bool *valid) { 22 | if ('0' <= c && c <= '9') { 23 | *value = c - '0'; 24 | *valid = true; 25 | } else if ('a' <= c && c <= 'f') { 26 | *value = 10 + c - 'a'; 27 | *valid = true; 28 | } else { 29 | *valid = false; 30 | } 31 | } 32 | 33 | static int parse_key(char *hex_key, uint8_t out[BLAKE3_KEY_LEN]) { 34 | size_t hex_len = strlen(hex_key); 35 | if (hex_len != 64) { 36 | fprintf(stderr, "Expected a 64-char hexadecimal key, got %zu chars.\n", 37 | hex_len); 38 | return 1; 39 | } 40 | for (size_t i = 0; i < 64; i++) { 41 | uint8_t value; 42 | bool valid; 43 | hex_char_value(hex_key[i], &value, &valid); 44 | if (!valid) { 45 | fprintf(stderr, "Invalid hex char.\n"); 46 | return 1; 47 | } 48 | if (i % 2 == 0) { 49 | out[i / 2] = 0; 50 | value <<= 4; 51 | } 52 | out[i / 2] += value; 53 | } 54 | return 0; 55 | } 56 | 57 | /* A little repetition here */ 58 | enum cpu_feature { 59 | SSE2 = 1 << 0, 60 | SSSE3 = 1 << 1, 61 | SSE41 = 1 << 2, 62 | AVX = 1 << 3, 63 | AVX2 = 1 << 4, 64 | AVX512F = 1 << 5, 65 | AVX512VL = 1 << 6, 66 | /* ... */ 67 | UNDEFINED = 1 << 30 68 | }; 69 | 70 | extern enum cpu_feature g_cpu_features; 71 | enum cpu_feature get_cpu_features(void); 72 | 73 | int main(int argc, char **argv) { 74 | size_t out_len = BLAKE3_OUT_LEN; 75 | uint8_t key[BLAKE3_KEY_LEN]; 76 | char *context = ""; 77 | uint8_t mode = HASH_MODE; 78 | while (argc > 1) { 79 | if (argc <= 2) { 80 | fprintf(stderr, "Odd number of arguments.\n"); 81 | return 1; 82 | } 83 | if (strcmp("--length", argv[1]) == 0) { 84 | char *endptr = NULL; 85 | errno = 0; 86 | unsigned long long out_len_ll = strtoull(argv[2], &endptr, 10); 87 | if (errno != 0 || out_len_ll > SIZE_MAX || endptr == argv[2] || 88 | *endptr != 0) { 89 | fprintf(stderr, "Bad length argument.\n"); 90 | return 1; 91 | } 92 | out_len = (size_t)out_len_ll; 93 | } else if (strcmp("--keyed", argv[1]) == 0) { 94 | mode = KEYED_HASH_MODE; 95 | int ret = parse_key(argv[2], key); 96 | if (ret != 0) { 97 | return ret; 98 | } 99 | } else if (strcmp("--derive-key", argv[1]) == 0) { 100 | mode = DERIVE_KEY_MODE; 101 | context = argv[2]; 102 | } else { 103 | fprintf(stderr, "Unknown flag.\n"); 104 | return 1; 105 | } 106 | argc -= 2; 107 | argv += 2; 108 | } 109 | 110 | /* 111 | * We're going to hash the input multiple times, so we need to buffer it all. 112 | * This is just for test cases, so go ahead and assume that the input is less 113 | * than 1 MiB. 114 | */ 115 | size_t buf_capacity = 1 << 20; 116 | uint8_t *buf = malloc(buf_capacity); 117 | assert(buf != NULL); 118 | size_t buf_len = 0; 119 | while (1) { 120 | size_t n = fread(&buf[buf_len], 1, buf_capacity - buf_len, stdin); 121 | if (n == 0) { 122 | break; 123 | } 124 | buf_len += n; 125 | assert(buf_len < buf_capacity); 126 | } 127 | 128 | const int mask = get_cpu_features(); 129 | int feature = 0; 130 | do { 131 | fprintf(stderr, "Testing 0x%08X\n", feature); 132 | g_cpu_features = feature; 133 | blake3_hasher hasher; 134 | switch (mode) { 135 | case HASH_MODE: 136 | blake3_hasher_init(&hasher); 137 | break; 138 | case KEYED_HASH_MODE: 139 | blake3_hasher_init_keyed(&hasher, key); 140 | break; 141 | case DERIVE_KEY_MODE: 142 | blake3_hasher_init_derive_key(&hasher, context); 143 | break; 144 | default: 145 | abort(); 146 | } 147 | 148 | blake3_hasher_update(&hasher, buf, buf_len); 149 | 150 | /* TODO: An incremental output reader API to avoid this allocation. */ 151 | uint8_t *out = malloc(out_len); 152 | if (out_len > 0 && out == NULL) { 153 | fprintf(stderr, "malloc() failed.\n"); 154 | return 1; 155 | } 156 | blake3_hasher_finalize(&hasher, out, out_len); 157 | for (size_t i = 0; i < out_len; i++) { 158 | printf("%02x", out[i]); 159 | } 160 | printf("\n"); 161 | free(out); 162 | feature = (feature - mask) & mask; 163 | } while (feature != 0); 164 | free(buf); 165 | return 0; 166 | } 167 | -------------------------------------------------------------------------------- /src/ffi_avx512.rs: -------------------------------------------------------------------------------- 1 | use crate::{CVWords, IncrementCounter, BLOCK_LEN, OUT_LEN}; 2 | 3 | // Unsafe because this may only be called on platforms supporting AVX-512. 4 | pub unsafe fn compress_in_place( 5 | cv: &mut CVWords, 6 | block: &[u8; BLOCK_LEN], 7 | block_len: u8, 8 | counter: u64, 9 | flags: u8, 10 | ) { 11 | unsafe { 12 | ffi::blake3_compress_in_place_avx512( 13 | cv.as_mut_ptr(), 14 | block.as_ptr(), 15 | block_len, 16 | counter, 17 | flags, 18 | ) 19 | } 20 | } 21 | 22 | // Unsafe because this may only be called on platforms supporting AVX-512. 23 | pub unsafe fn compress_xof( 24 | cv: &CVWords, 25 | block: &[u8; BLOCK_LEN], 26 | block_len: u8, 27 | counter: u64, 28 | flags: u8, 29 | ) -> [u8; 64] { 30 | unsafe { 31 | let mut out = [0u8; 64]; 32 | ffi::blake3_compress_xof_avx512( 33 | cv.as_ptr(), 34 | block.as_ptr(), 35 | block_len, 36 | counter, 37 | flags, 38 | out.as_mut_ptr(), 39 | ); 40 | out 41 | } 42 | } 43 | 44 | // Unsafe because this may only be called on platforms supporting AVX-512. 45 | pub unsafe fn hash_many( 46 | inputs: &[&[u8; N]], 47 | key: &CVWords, 48 | counter: u64, 49 | increment_counter: IncrementCounter, 50 | flags: u8, 51 | flags_start: u8, 52 | flags_end: u8, 53 | out: &mut [u8], 54 | ) { 55 | unsafe { 56 | // The Rust hash_many implementations do bounds checking on the `out` 57 | // array, but the C implementations don't. Even though this is an unsafe 58 | // function, assert the bounds here. 59 | assert!(out.len() >= inputs.len() * OUT_LEN); 60 | ffi::blake3_hash_many_avx512( 61 | inputs.as_ptr() as *const *const u8, 62 | inputs.len(), 63 | N / BLOCK_LEN, 64 | key.as_ptr(), 65 | counter, 66 | increment_counter.yes(), 67 | flags, 68 | flags_start, 69 | flags_end, 70 | out.as_mut_ptr(), 71 | ) 72 | } 73 | } 74 | 75 | // Unsafe because this may only be called on platforms supporting AVX-512. 76 | #[cfg(unix)] 77 | pub unsafe fn xof_many( 78 | cv: &CVWords, 79 | block: &[u8; BLOCK_LEN], 80 | block_len: u8, 81 | counter: u64, 82 | flags: u8, 83 | out: &mut [u8], 84 | ) { 85 | unsafe { 86 | debug_assert_eq!(0, out.len() % BLOCK_LEN, "whole blocks only"); 87 | ffi::blake3_xof_many_avx512( 88 | cv.as_ptr(), 89 | block.as_ptr(), 90 | block_len, 91 | counter, 92 | flags, 93 | out.as_mut_ptr(), 94 | out.len() / BLOCK_LEN, 95 | ); 96 | } 97 | } 98 | 99 | pub mod ffi { 100 | extern "C" { 101 | pub fn blake3_compress_in_place_avx512( 102 | cv: *mut u32, 103 | block: *const u8, 104 | block_len: u8, 105 | counter: u64, 106 | flags: u8, 107 | ); 108 | pub fn blake3_compress_xof_avx512( 109 | cv: *const u32, 110 | block: *const u8, 111 | block_len: u8, 112 | counter: u64, 113 | flags: u8, 114 | out: *mut u8, 115 | ); 116 | pub fn blake3_hash_many_avx512( 117 | inputs: *const *const u8, 118 | num_inputs: usize, 119 | blocks: usize, 120 | key: *const u32, 121 | counter: u64, 122 | increment_counter: bool, 123 | flags: u8, 124 | flags_start: u8, 125 | flags_end: u8, 126 | out: *mut u8, 127 | ); 128 | #[cfg(unix)] 129 | pub fn blake3_xof_many_avx512( 130 | cv: *const u32, 131 | block: *const u8, 132 | block_len: u8, 133 | counter: u64, 134 | flags: u8, 135 | out: *mut u8, 136 | outblocks: usize, 137 | ); 138 | } 139 | } 140 | 141 | #[cfg(test)] 142 | mod test { 143 | use super::*; 144 | 145 | #[test] 146 | fn test_compress() { 147 | if !crate::platform::avx512_detected() { 148 | return; 149 | } 150 | crate::test::test_compress_fn(compress_in_place, compress_xof); 151 | } 152 | 153 | #[test] 154 | fn test_hash_many() { 155 | if !crate::platform::avx512_detected() { 156 | return; 157 | } 158 | crate::test::test_hash_many_fn(hash_many, hash_many); 159 | } 160 | 161 | #[cfg(unix)] 162 | #[test] 163 | fn test_xof_many() { 164 | if !crate::platform::avx512_detected() { 165 | return; 166 | } 167 | crate::test::test_xof_many_fn(xof_many); 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /src/portable.rs: -------------------------------------------------------------------------------- 1 | use crate::{ 2 | counter_high, counter_low, CVBytes, CVWords, IncrementCounter, BLOCK_LEN, IV, MSG_SCHEDULE, 3 | OUT_LEN, 4 | }; 5 | use arrayref::{array_mut_ref, array_ref}; 6 | 7 | #[inline(always)] 8 | fn g(state: &mut [u32; 16], a: usize, b: usize, c: usize, d: usize, x: u32, y: u32) { 9 | state[a] = state[a].wrapping_add(state[b]).wrapping_add(x); 10 | state[d] = (state[d] ^ state[a]).rotate_right(16); 11 | state[c] = state[c].wrapping_add(state[d]); 12 | state[b] = (state[b] ^ state[c]).rotate_right(12); 13 | state[a] = state[a].wrapping_add(state[b]).wrapping_add(y); 14 | state[d] = (state[d] ^ state[a]).rotate_right(8); 15 | state[c] = state[c].wrapping_add(state[d]); 16 | state[b] = (state[b] ^ state[c]).rotate_right(7); 17 | } 18 | 19 | #[inline(always)] 20 | fn round(state: &mut [u32; 16], msg: &[u32; 16], round: usize) { 21 | // Select the message schedule based on the round. 22 | let schedule = MSG_SCHEDULE[round]; 23 | 24 | // Mix the columns. 25 | g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); 26 | g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); 27 | g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); 28 | g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); 29 | 30 | // Mix the diagonals. 31 | g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); 32 | g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); 33 | g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); 34 | g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); 35 | } 36 | 37 | #[inline(always)] 38 | fn compress_pre( 39 | cv: &CVWords, 40 | block: &[u8; BLOCK_LEN], 41 | block_len: u8, 42 | counter: u64, 43 | flags: u8, 44 | ) -> [u32; 16] { 45 | let block_words = crate::platform::words_from_le_bytes_64(block); 46 | 47 | let mut state = [ 48 | cv[0], 49 | cv[1], 50 | cv[2], 51 | cv[3], 52 | cv[4], 53 | cv[5], 54 | cv[6], 55 | cv[7], 56 | IV[0], 57 | IV[1], 58 | IV[2], 59 | IV[3], 60 | counter_low(counter), 61 | counter_high(counter), 62 | block_len as u32, 63 | flags as u32, 64 | ]; 65 | 66 | round(&mut state, &block_words, 0); 67 | round(&mut state, &block_words, 1); 68 | round(&mut state, &block_words, 2); 69 | round(&mut state, &block_words, 3); 70 | round(&mut state, &block_words, 4); 71 | round(&mut state, &block_words, 5); 72 | round(&mut state, &block_words, 6); 73 | 74 | state 75 | } 76 | 77 | pub fn compress_in_place( 78 | cv: &mut CVWords, 79 | block: &[u8; BLOCK_LEN], 80 | block_len: u8, 81 | counter: u64, 82 | flags: u8, 83 | ) { 84 | let state = compress_pre(cv, block, block_len, counter, flags); 85 | 86 | cv[0] = state[0] ^ state[8]; 87 | cv[1] = state[1] ^ state[9]; 88 | cv[2] = state[2] ^ state[10]; 89 | cv[3] = state[3] ^ state[11]; 90 | cv[4] = state[4] ^ state[12]; 91 | cv[5] = state[5] ^ state[13]; 92 | cv[6] = state[6] ^ state[14]; 93 | cv[7] = state[7] ^ state[15]; 94 | } 95 | 96 | pub fn compress_xof( 97 | cv: &CVWords, 98 | block: &[u8; BLOCK_LEN], 99 | block_len: u8, 100 | counter: u64, 101 | flags: u8, 102 | ) -> [u8; 64] { 103 | let mut state = compress_pre(cv, block, block_len, counter, flags); 104 | state[0] ^= state[8]; 105 | state[1] ^= state[9]; 106 | state[2] ^= state[10]; 107 | state[3] ^= state[11]; 108 | state[4] ^= state[12]; 109 | state[5] ^= state[13]; 110 | state[6] ^= state[14]; 111 | state[7] ^= state[15]; 112 | state[8] ^= cv[0]; 113 | state[9] ^= cv[1]; 114 | state[10] ^= cv[2]; 115 | state[11] ^= cv[3]; 116 | state[12] ^= cv[4]; 117 | state[13] ^= cv[5]; 118 | state[14] ^= cv[6]; 119 | state[15] ^= cv[7]; 120 | crate::platform::le_bytes_from_words_64(&state) 121 | } 122 | 123 | pub fn hash1( 124 | input: &[u8; N], 125 | key: &CVWords, 126 | counter: u64, 127 | flags: u8, 128 | flags_start: u8, 129 | flags_end: u8, 130 | out: &mut CVBytes, 131 | ) { 132 | debug_assert_eq!(N % BLOCK_LEN, 0, "uneven blocks"); 133 | let mut cv = *key; 134 | let mut block_flags = flags | flags_start; 135 | let mut slice = &input[..]; 136 | while slice.len() >= BLOCK_LEN { 137 | if slice.len() == BLOCK_LEN { 138 | block_flags |= flags_end; 139 | } 140 | compress_in_place( 141 | &mut cv, 142 | array_ref!(slice, 0, BLOCK_LEN), 143 | BLOCK_LEN as u8, 144 | counter, 145 | block_flags, 146 | ); 147 | block_flags = flags; 148 | slice = &slice[BLOCK_LEN..]; 149 | } 150 | *out = crate::platform::le_bytes_from_words_32(&cv); 151 | } 152 | 153 | pub fn hash_many( 154 | inputs: &[&[u8; N]], 155 | key: &CVWords, 156 | mut counter: u64, 157 | increment_counter: IncrementCounter, 158 | flags: u8, 159 | flags_start: u8, 160 | flags_end: u8, 161 | out: &mut [u8], 162 | ) { 163 | debug_assert!(out.len() >= inputs.len() * OUT_LEN, "out too short"); 164 | for (&input, output) in inputs.iter().zip(out.chunks_exact_mut(OUT_LEN)) { 165 | hash1( 166 | input, 167 | key, 168 | counter, 169 | flags, 170 | flags_start, 171 | flags_end, 172 | array_mut_ref!(output, 0, OUT_LEN), 173 | ); 174 | if increment_counter.yes() { 175 | counter += 1; 176 | } 177 | } 178 | } 179 | 180 | #[cfg(test)] 181 | pub mod test { 182 | use super::*; 183 | 184 | // This is basically testing the portable implementation against itself, 185 | // but it also checks that compress_in_place and compress_xof are 186 | // consistent. And there are tests against the reference implementation and 187 | // against hardcoded test vectors elsewhere. 188 | #[test] 189 | fn test_compress() { 190 | crate::test::test_compress_fn(compress_in_place, compress_xof); 191 | } 192 | 193 | // Ditto. 194 | #[test] 195 | fn test_hash_many() { 196 | crate::test::test_hash_many_fn(hash_many, hash_many); 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /c/blake3_portable.c: -------------------------------------------------------------------------------- 1 | #include "blake3_impl.h" 2 | #include 3 | 4 | INLINE uint32_t rotr32(uint32_t w, uint32_t c) { 5 | return (w >> c) | (w << (32 - c)); 6 | } 7 | 8 | INLINE void g(uint32_t *state, size_t a, size_t b, size_t c, size_t d, 9 | uint32_t x, uint32_t y) { 10 | state[a] = state[a] + state[b] + x; 11 | state[d] = rotr32(state[d] ^ state[a], 16); 12 | state[c] = state[c] + state[d]; 13 | state[b] = rotr32(state[b] ^ state[c], 12); 14 | state[a] = state[a] + state[b] + y; 15 | state[d] = rotr32(state[d] ^ state[a], 8); 16 | state[c] = state[c] + state[d]; 17 | state[b] = rotr32(state[b] ^ state[c], 7); 18 | } 19 | 20 | INLINE void round_fn(uint32_t state[16], const uint32_t *msg, size_t round) { 21 | // Select the message schedule based on the round. 22 | const uint8_t *schedule = MSG_SCHEDULE[round]; 23 | 24 | // Mix the columns. 25 | g(state, 0, 4, 8, 12, msg[schedule[0]], msg[schedule[1]]); 26 | g(state, 1, 5, 9, 13, msg[schedule[2]], msg[schedule[3]]); 27 | g(state, 2, 6, 10, 14, msg[schedule[4]], msg[schedule[5]]); 28 | g(state, 3, 7, 11, 15, msg[schedule[6]], msg[schedule[7]]); 29 | 30 | // Mix the rows. 31 | g(state, 0, 5, 10, 15, msg[schedule[8]], msg[schedule[9]]); 32 | g(state, 1, 6, 11, 12, msg[schedule[10]], msg[schedule[11]]); 33 | g(state, 2, 7, 8, 13, msg[schedule[12]], msg[schedule[13]]); 34 | g(state, 3, 4, 9, 14, msg[schedule[14]], msg[schedule[15]]); 35 | } 36 | 37 | INLINE void compress_pre(uint32_t state[16], const uint32_t cv[8], 38 | const uint8_t block[BLAKE3_BLOCK_LEN], 39 | uint8_t block_len, uint64_t counter, uint8_t flags) { 40 | uint32_t block_words[16]; 41 | block_words[0] = load32(block + 4 * 0); 42 | block_words[1] = load32(block + 4 * 1); 43 | block_words[2] = load32(block + 4 * 2); 44 | block_words[3] = load32(block + 4 * 3); 45 | block_words[4] = load32(block + 4 * 4); 46 | block_words[5] = load32(block + 4 * 5); 47 | block_words[6] = load32(block + 4 * 6); 48 | block_words[7] = load32(block + 4 * 7); 49 | block_words[8] = load32(block + 4 * 8); 50 | block_words[9] = load32(block + 4 * 9); 51 | block_words[10] = load32(block + 4 * 10); 52 | block_words[11] = load32(block + 4 * 11); 53 | block_words[12] = load32(block + 4 * 12); 54 | block_words[13] = load32(block + 4 * 13); 55 | block_words[14] = load32(block + 4 * 14); 56 | block_words[15] = load32(block + 4 * 15); 57 | 58 | state[0] = cv[0]; 59 | state[1] = cv[1]; 60 | state[2] = cv[2]; 61 | state[3] = cv[3]; 62 | state[4] = cv[4]; 63 | state[5] = cv[5]; 64 | state[6] = cv[6]; 65 | state[7] = cv[7]; 66 | state[8] = IV[0]; 67 | state[9] = IV[1]; 68 | state[10] = IV[2]; 69 | state[11] = IV[3]; 70 | state[12] = counter_low(counter); 71 | state[13] = counter_high(counter); 72 | state[14] = (uint32_t)block_len; 73 | state[15] = (uint32_t)flags; 74 | 75 | round_fn(state, &block_words[0], 0); 76 | round_fn(state, &block_words[0], 1); 77 | round_fn(state, &block_words[0], 2); 78 | round_fn(state, &block_words[0], 3); 79 | round_fn(state, &block_words[0], 4); 80 | round_fn(state, &block_words[0], 5); 81 | round_fn(state, &block_words[0], 6); 82 | } 83 | 84 | void blake3_compress_in_place_portable(uint32_t cv[8], 85 | const uint8_t block[BLAKE3_BLOCK_LEN], 86 | uint8_t block_len, uint64_t counter, 87 | uint8_t flags) { 88 | uint32_t state[16]; 89 | compress_pre(state, cv, block, block_len, counter, flags); 90 | cv[0] = state[0] ^ state[8]; 91 | cv[1] = state[1] ^ state[9]; 92 | cv[2] = state[2] ^ state[10]; 93 | cv[3] = state[3] ^ state[11]; 94 | cv[4] = state[4] ^ state[12]; 95 | cv[5] = state[5] ^ state[13]; 96 | cv[6] = state[6] ^ state[14]; 97 | cv[7] = state[7] ^ state[15]; 98 | } 99 | 100 | void blake3_compress_xof_portable(const uint32_t cv[8], 101 | const uint8_t block[BLAKE3_BLOCK_LEN], 102 | uint8_t block_len, uint64_t counter, 103 | uint8_t flags, uint8_t out[64]) { 104 | uint32_t state[16]; 105 | compress_pre(state, cv, block, block_len, counter, flags); 106 | 107 | store32(&out[0 * 4], state[0] ^ state[8]); 108 | store32(&out[1 * 4], state[1] ^ state[9]); 109 | store32(&out[2 * 4], state[2] ^ state[10]); 110 | store32(&out[3 * 4], state[3] ^ state[11]); 111 | store32(&out[4 * 4], state[4] ^ state[12]); 112 | store32(&out[5 * 4], state[5] ^ state[13]); 113 | store32(&out[6 * 4], state[6] ^ state[14]); 114 | store32(&out[7 * 4], state[7] ^ state[15]); 115 | store32(&out[8 * 4], state[8] ^ cv[0]); 116 | store32(&out[9 * 4], state[9] ^ cv[1]); 117 | store32(&out[10 * 4], state[10] ^ cv[2]); 118 | store32(&out[11 * 4], state[11] ^ cv[3]); 119 | store32(&out[12 * 4], state[12] ^ cv[4]); 120 | store32(&out[13 * 4], state[13] ^ cv[5]); 121 | store32(&out[14 * 4], state[14] ^ cv[6]); 122 | store32(&out[15 * 4], state[15] ^ cv[7]); 123 | } 124 | 125 | INLINE void hash_one_portable(const uint8_t *input, size_t blocks, 126 | const uint32_t key[8], uint64_t counter, 127 | uint8_t flags, uint8_t flags_start, 128 | uint8_t flags_end, uint8_t out[BLAKE3_OUT_LEN]) { 129 | uint32_t cv[8]; 130 | memcpy(cv, key, BLAKE3_KEY_LEN); 131 | uint8_t block_flags = flags | flags_start; 132 | while (blocks > 0) { 133 | if (blocks == 1) { 134 | block_flags |= flags_end; 135 | } 136 | blake3_compress_in_place_portable(cv, input, BLAKE3_BLOCK_LEN, counter, 137 | block_flags); 138 | input = &input[BLAKE3_BLOCK_LEN]; 139 | blocks -= 1; 140 | block_flags = flags; 141 | } 142 | store_cv_words(out, cv); 143 | } 144 | 145 | void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs, 146 | size_t blocks, const uint32_t key[8], 147 | uint64_t counter, bool increment_counter, 148 | uint8_t flags, uint8_t flags_start, 149 | uint8_t flags_end, uint8_t *out) { 150 | while (num_inputs > 0) { 151 | hash_one_portable(inputs[0], blocks, key, counter, flags, flags_start, 152 | flags_end, out); 153 | if (increment_counter) { 154 | counter += 1; 155 | } 156 | inputs += 1; 157 | num_inputs -= 1; 158 | out = &out[BLAKE3_OUT_LEN]; 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "blake3" 3 | version = "1.8.2" 4 | authors = ["Jack O'Connor ", "Samuel Neves"] 5 | description = "the BLAKE3 hash function" 6 | repository = "https://github.com/BLAKE3-team/BLAKE3" 7 | license = "CC0-1.0 OR Apache-2.0 OR Apache-2.0 WITH LLVM-exception" 8 | documentation = "https://docs.rs/blake3" 9 | readme = "README.md" 10 | edition = "2021" 11 | 12 | [features] 13 | default = ["std"] 14 | 15 | # The NEON implementation does not participate in dynamic feature detection, 16 | # which is currently x86-only. If "neon" is on, NEON support is assumed. Note 17 | # that AArch64 always supports NEON, but support on ARMv7 varies. The NEON 18 | # implementation uses C intrinsics and requires a C compiler. 19 | neon = [] 20 | 21 | # The Wasm SIMD implementation does not participate in dynamic feature detection, 22 | # which is currently x86-only. If "wasm_simd" is on, Wasm SIMD support is assumed. 23 | # Note that not all Wasm implementations support the Wasm SIMD specification. 24 | # This may become the default in the future. 25 | wasm32_simd = [] 26 | 27 | # Enables std::io traits, including impl Write for Hasher, impl Read and Seek 28 | # for OutputReader, and the Hasher::update_reader method. This feature is 29 | # enabled by default. (Previously this also controlled runtime CPU feature 30 | # detection on x86, but now we use the no-std-compatible `cpufeatures` crate 31 | # for that.) 32 | std = [] 33 | 34 | # The `rayon` feature (disabled by default, but enabled for docs.rs) adds the 35 | # `update_rayon` and (in combination with `mmap` below) `update_mmap_rayon` 36 | # methods, for multithreaded hashing. However, even if this feature is enabled, 37 | # all other APIs remain single-threaded. 38 | # 39 | # Implementation detail: We take a dependency on rayon-core instead of rayon, 40 | # because it builds faster and still includes all the APIs we need. 41 | rayon = ["dep:rayon-core"] 42 | 43 | # The `mmap` feature (disabled by default, but enabled for docs.rs) adds the 44 | # `update_mmap` and (in combination with `rayon` above) `update_mmap_rayon` 45 | # helper methods for memory-mapped IO. 46 | mmap = ["std", "dep:memmap2"] 47 | 48 | # Implement the zeroize::Zeroize trait for types in this crate. 49 | zeroize = ["dep:zeroize", "arrayvec/zeroize"] 50 | 51 | # This crate implements traits from the RustCrypto project, exposed here as the 52 | # "traits-preview" feature. However, these traits aren't stable, and they're 53 | # expected to change in incompatible ways before they reach 1.0. For that 54 | # reason, this crate makes no SemVer guarantees for this feature, and callers 55 | # who use it should expect breaking changes between patch versions of this 56 | # crate. (The "*-preview" feature name follows the conventions of the RustCrypto 57 | # "signature" crate.) 58 | traits-preview = ["dep:digest"] 59 | 60 | # ---------- Features below this line are undocumented and unstable. ---------- 61 | # The following features are mainly intended for testing and benchmarking, and 62 | # they might change or disappear at any time without a major version bump. 63 | 64 | # It wasn't originally intended to expose "digest" as its own feature, but the 65 | # traits-preview feature above predated the "dep:" syntax in Cargo. Version 66 | # 1.5.2 of this crate started using "dep:" syntax, but that broke some callers 67 | # in the wild (https://solana.stackexchange.com/q/17787/29050). This feature 68 | # unbreaks those callers. When Cargo gains the ability to deprecate features, 69 | # this feature will be deprecated. Note that the relevant trait implementations 70 | # are still gated by "traits-preview". 71 | digest = ["dep:digest"] 72 | 73 | # By default on x86_64, this crate uses Samuel Neves' hand-written assembly 74 | # implementations for SSE4.1, AVX2, and AVX512. (These provide both the best 75 | # runtime performance, and the fastest build times.) And by default on 32-bit 76 | # x86, this crate uses Rust intrinsics implementations for SSE4.1 and AVX2, and 77 | # a C intrinsics implementation for AVX-512. In both cases, if a C compiler is 78 | # not detected, or if AVX-512 support is missing from the detected compiler, 79 | # build.rs automatically falls back to a pure Rust build. This feature forces 80 | # that fallback, for testing purposes. (Note that in CI testing, we set the 81 | # BLAKE3_CI environment variable, which instructs build.rs to error out rather 82 | # than doing an automatic fallback.) 83 | pure = [] 84 | 85 | # As described above, on x86_64 this crate use assembly implementations by 86 | # default. Enabling the "prefer_intrinsics" feature makes this crate use 87 | # intrinsics implementations on both 32-bit and 64-bit x86, again for testing 88 | # purposes. 89 | prefer_intrinsics = [] 90 | 91 | # Disable individual instruction sets. CI testing uses these flags to simulate 92 | # different levels of hardware SIMD support. Note that code for the 93 | # corresponding instruction set is still compiled; only detection is disabled. 94 | # 95 | # As noted above, these flags are *for testing only* and are not stable. It's 96 | # possible that some users might find that their particular use case performs 97 | # better if e.g. AVX-512 is disabled, because of issues like CPU downclocking. 98 | # If that comes up, and if disabling the instruction set here at the feature 99 | # level turns out to be the right approach, then we can design a stable 100 | # feature. Until then, we reserve the right to break these features in a patch 101 | # release. 102 | no_sse2 = [] 103 | no_sse41 = [] 104 | no_avx2 = [] 105 | no_avx512 = [] 106 | no_neon = [] 107 | 108 | [package.metadata.docs.rs] 109 | # Document the rayon/mmap methods and the Serialize/Deserialize/Zeroize impls on docs.rs. 110 | features = ["mmap", "rayon", "serde", "zeroize"] 111 | 112 | [dependencies] 113 | arrayref = "0.3.5" 114 | arrayvec = { version = "0.7.4", default-features = false } 115 | constant_time_eq = { version = "0.3.1", default-features = false } 116 | cfg-if = "1.0.0" 117 | digest = { version = "0.10.1", features = ["mac"], optional = true } 118 | memmap2 = { version = "0.9", optional = true } 119 | rayon-core = { version = "1.12.1", optional = true } 120 | serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } 121 | zeroize = { version = "1", default-features = false, optional = true } 122 | 123 | [target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies] 124 | cpufeatures = "0.2.17" 125 | 126 | [dev-dependencies] 127 | hmac = "0.12.0" 128 | hex = "0.4.2" 129 | page_size = "0.6.0" 130 | rand = "0.9.0" 131 | rand_chacha = "0.9.0" 132 | reference_impl = { path = "./reference_impl" } 133 | tempfile = "3.8.0" 134 | serde_json = "1.0.107" 135 | ciborium = "0.2.2" 136 | 137 | [build-dependencies] 138 | cc = "1.1.12" 139 | -------------------------------------------------------------------------------- /media/BLAKE3.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 19 | 21 | 39 | 41 | 42 | 44 | image/svg+xml 45 | 47 | 48 | 49 | 50 | 55 | 59 | 63 | 67 | 71 | 75 | 79 | 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /LICENSE_CC0: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | -------------------------------------------------------------------------------- /src/traits.rs: -------------------------------------------------------------------------------- 1 | //! Implementations of commonly used traits like `Digest` and `Mac` from the 2 | //! [`digest`](https://crates.io/crates/digest) crate. 3 | 4 | pub use digest; 5 | 6 | use crate::{Hasher, OutputReader}; 7 | use digest::crypto_common; 8 | use digest::generic_array::{typenum::U32, typenum::U64, GenericArray}; 9 | 10 | impl digest::HashMarker for Hasher {} 11 | 12 | impl digest::Update for Hasher { 13 | #[inline] 14 | fn update(&mut self, data: &[u8]) { 15 | self.update(data); 16 | } 17 | } 18 | 19 | impl digest::Reset for Hasher { 20 | #[inline] 21 | fn reset(&mut self) { 22 | self.reset(); // the inherent method 23 | } 24 | } 25 | 26 | impl digest::OutputSizeUser for Hasher { 27 | type OutputSize = U32; 28 | } 29 | 30 | impl digest::FixedOutput for Hasher { 31 | #[inline] 32 | fn finalize_into(self, out: &mut GenericArray) { 33 | out.copy_from_slice(self.finalize().as_bytes()); 34 | } 35 | } 36 | 37 | impl digest::FixedOutputReset for Hasher { 38 | #[inline] 39 | fn finalize_into_reset(&mut self, out: &mut GenericArray) { 40 | out.copy_from_slice(self.finalize().as_bytes()); 41 | self.reset(); 42 | } 43 | } 44 | 45 | impl digest::ExtendableOutput for Hasher { 46 | type Reader = OutputReader; 47 | 48 | #[inline] 49 | fn finalize_xof(self) -> Self::Reader { 50 | Hasher::finalize_xof(&self) 51 | } 52 | } 53 | 54 | impl digest::ExtendableOutputReset for Hasher { 55 | #[inline] 56 | fn finalize_xof_reset(&mut self) -> Self::Reader { 57 | let reader = Hasher::finalize_xof(self); 58 | self.reset(); 59 | reader 60 | } 61 | } 62 | 63 | impl digest::XofReader for OutputReader { 64 | #[inline] 65 | fn read(&mut self, buffer: &mut [u8]) { 66 | self.fill(buffer); 67 | } 68 | } 69 | 70 | impl crypto_common::KeySizeUser for Hasher { 71 | type KeySize = U32; 72 | } 73 | 74 | impl crypto_common::BlockSizeUser for Hasher { 75 | type BlockSize = U64; 76 | } 77 | 78 | impl digest::MacMarker for Hasher {} 79 | 80 | impl digest::KeyInit for Hasher { 81 | #[inline] 82 | fn new(key: &digest::Key) -> Self { 83 | let key_bytes: [u8; 32] = (*key).into(); 84 | Hasher::new_keyed(&key_bytes) 85 | } 86 | } 87 | 88 | #[cfg(test)] 89 | mod test { 90 | use super::*; 91 | 92 | #[test] 93 | fn test_digest_traits() { 94 | // Inherent methods. 95 | let mut hasher1 = crate::Hasher::new(); 96 | hasher1.update(b"foo"); 97 | hasher1.update(b"bar"); 98 | hasher1.update(b"baz"); 99 | let out1 = hasher1.finalize(); 100 | let mut xof1 = [0; 301]; 101 | hasher1.finalize_xof().fill(&mut xof1); 102 | assert_eq!(out1.as_bytes(), &xof1[..32]); 103 | 104 | // Trait implementations. 105 | let mut hasher2: crate::Hasher = digest::Digest::new(); 106 | digest::Digest::update(&mut hasher2, b"xxx"); 107 | digest::Digest::reset(&mut hasher2); 108 | digest::Digest::update(&mut hasher2, b"foo"); 109 | digest::Digest::update(&mut hasher2, b"bar"); 110 | digest::Digest::update(&mut hasher2, b"baz"); 111 | let out2 = digest::Digest::finalize(hasher2.clone()); 112 | let mut xof2 = [0; 301]; 113 | digest::XofReader::read( 114 | &mut digest::ExtendableOutput::finalize_xof(hasher2.clone()), 115 | &mut xof2, 116 | ); 117 | assert_eq!(out1.as_bytes(), &out2[..]); 118 | assert_eq!(xof1[..], xof2[..]); 119 | 120 | // Again with the resetting variants. 121 | let mut hasher3: crate::Hasher = digest::Digest::new(); 122 | digest::Digest::update(&mut hasher3, b"foobarbaz"); 123 | let mut out3 = [0; 32]; 124 | digest::FixedOutputReset::finalize_into_reset( 125 | &mut hasher3, 126 | GenericArray::from_mut_slice(&mut out3), 127 | ); 128 | digest::Digest::update(&mut hasher3, b"foobarbaz"); 129 | let mut out4 = [0; 32]; 130 | digest::FixedOutputReset::finalize_into_reset( 131 | &mut hasher3, 132 | GenericArray::from_mut_slice(&mut out4), 133 | ); 134 | digest::Digest::update(&mut hasher3, b"foobarbaz"); 135 | let mut xof3 = [0; 301]; 136 | digest::XofReader::read( 137 | &mut digest::ExtendableOutputReset::finalize_xof_reset(&mut hasher3), 138 | &mut xof3, 139 | ); 140 | digest::Digest::update(&mut hasher3, b"foobarbaz"); 141 | let mut xof4 = [0; 301]; 142 | digest::XofReader::read( 143 | &mut digest::ExtendableOutputReset::finalize_xof_reset(&mut hasher3), 144 | &mut xof4, 145 | ); 146 | assert_eq!(out1.as_bytes(), &out3[..]); 147 | assert_eq!(out1.as_bytes(), &out4[..]); 148 | assert_eq!(xof1[..], xof3[..]); 149 | assert_eq!(xof1[..], xof4[..]); 150 | } 151 | 152 | #[test] 153 | fn test_mac_trait() { 154 | // Inherent methods. 155 | let key = b"some super secret key bytes fooo"; 156 | let mut hasher1 = crate::Hasher::new_keyed(key); 157 | hasher1.update(b"foo"); 158 | hasher1.update(b"bar"); 159 | hasher1.update(b"baz"); 160 | let out1 = hasher1.finalize(); 161 | 162 | // Trait implementation. 163 | let generic_key = (*key).into(); 164 | let mut hasher2: crate::Hasher = digest::Mac::new(&generic_key); 165 | digest::Mac::update(&mut hasher2, b"xxx"); 166 | digest::Mac::reset(&mut hasher2); 167 | digest::Mac::update(&mut hasher2, b"foo"); 168 | digest::Mac::update(&mut hasher2, b"bar"); 169 | digest::Mac::update(&mut hasher2, b"baz"); 170 | let out2 = digest::Mac::finalize(hasher2); 171 | assert_eq!(out1.as_bytes(), out2.into_bytes().as_slice()); 172 | } 173 | 174 | fn expected_hmac_blake3(key: &[u8], input: &[u8]) -> [u8; 32] { 175 | // See https://en.wikipedia.org/wiki/HMAC. 176 | let key_hash; 177 | let key_prime = if key.len() <= 64 { 178 | key 179 | } else { 180 | key_hash = *crate::hash(key).as_bytes(); 181 | &key_hash 182 | }; 183 | let mut ipad = [0x36; 64]; 184 | let mut opad = [0x5c; 64]; 185 | for i in 0..key_prime.len() { 186 | ipad[i] ^= key_prime[i]; 187 | opad[i] ^= key_prime[i]; 188 | } 189 | let mut inner_state = crate::Hasher::new(); 190 | inner_state.update(&ipad); 191 | inner_state.update(input); 192 | let mut outer_state = crate::Hasher::new(); 193 | outer_state.update(&opad); 194 | outer_state.update(inner_state.finalize().as_bytes()); 195 | outer_state.finalize().into() 196 | } 197 | 198 | #[test] 199 | fn test_hmac_compatibility() { 200 | use hmac::{Mac, SimpleHmac}; 201 | 202 | // Test a short key. 203 | let mut x = SimpleHmac::::new_from_slice(b"key").unwrap(); 204 | hmac::digest::Update::update(&mut x, b"data"); 205 | let output = x.finalize().into_bytes(); 206 | assert_ne!(output.len(), 0); 207 | let expected = expected_hmac_blake3(b"key", b"data"); 208 | assert_eq!(expected, output.as_ref()); 209 | 210 | // Test a range of key and data lengths, particularly to exercise the long-key logic. 211 | let mut input_bytes = [0; crate::test::TEST_CASES_MAX]; 212 | crate::test::paint_test_input(&mut input_bytes); 213 | for &input_len in crate::test::TEST_CASES { 214 | #[cfg(feature = "std")] 215 | dbg!(input_len); 216 | let input = &input_bytes[..input_len]; 217 | 218 | let mut x = SimpleHmac::::new_from_slice(input).unwrap(); 219 | hmac::digest::Update::update(&mut x, input); 220 | let output = x.finalize().into_bytes(); 221 | assert_ne!(output.len(), 0); 222 | 223 | let expected = expected_hmac_blake3(input, input); 224 | assert_eq!(expected, output.as_ref()); 225 | } 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /b3sum/src/unit_tests.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | 3 | #[test] 4 | fn test_parse_check_line() { 5 | // ========================= 6 | // ===== Success Cases ===== 7 | // ========================= 8 | 9 | // the basic case 10 | let crate::ParsedCheckLine { 11 | file_string, 12 | is_escaped, 13 | file_path, 14 | expected_hash, 15 | } = crate::parse_check_line( 16 | "0909090909090909090909090909090909090909090909090909090909090909 foo", 17 | ) 18 | .unwrap(); 19 | assert_eq!(expected_hash, blake3::Hash::from([0x09; 32])); 20 | assert!(!is_escaped); 21 | assert_eq!(file_string, "foo"); 22 | assert_eq!(file_path, Path::new("foo")); 23 | 24 | // regular whitespace 25 | let crate::ParsedCheckLine { 26 | file_string, 27 | is_escaped, 28 | file_path, 29 | expected_hash, 30 | } = crate::parse_check_line( 31 | "fafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafafa \t\r\n\n\r \t\r\n\n\r", 32 | ) 33 | .unwrap(); 34 | assert_eq!(expected_hash, blake3::Hash::from([0xfa; 32])); 35 | assert!(!is_escaped); 36 | assert_eq!(file_string, " \t\r\n\n\r \t"); 37 | assert_eq!(file_path, Path::new(" \t\r\n\n\r \t")); 38 | 39 | // path is one space 40 | let crate::ParsedCheckLine { 41 | file_string, 42 | is_escaped, 43 | file_path, 44 | expected_hash, 45 | } = crate::parse_check_line( 46 | "4242424242424242424242424242424242424242424242424242424242424242 ", 47 | ) 48 | .unwrap(); 49 | assert_eq!(expected_hash, blake3::Hash::from([0x42; 32])); 50 | assert!(!is_escaped); 51 | assert_eq!(file_string, " "); 52 | assert_eq!(file_path, Path::new(" ")); 53 | 54 | // *Unescaped* backslashes. Note that this line does *not* start with a 55 | // backslash, so something like "\" + "n" is interpreted as *two* 56 | // characters. We forbid all backslashes on Windows, so this test is 57 | // Unix-only. 58 | if cfg!(not(windows)) { 59 | let crate::ParsedCheckLine { 60 | file_string, 61 | is_escaped, 62 | file_path, 63 | expected_hash, 64 | } = crate::parse_check_line( 65 | "4343434343434343434343434343434343434343434343434343434343434343 fo\\a\\no", 66 | ) 67 | .unwrap(); 68 | assert_eq!(expected_hash, blake3::Hash::from([0x43; 32])); 69 | assert!(!is_escaped); 70 | assert_eq!(file_string, "fo\\a\\no"); 71 | assert_eq!(file_path, Path::new("fo\\a\\no")); 72 | } 73 | 74 | // escaped newlines 75 | let crate::ParsedCheckLine { 76 | file_string, 77 | is_escaped, 78 | file_path, 79 | expected_hash, 80 | } = crate::parse_check_line( 81 | "\\4444444444444444444444444444444444444444444444444444444444444444 fo\\r\\n\\n\\ro", 82 | ) 83 | .unwrap(); 84 | assert_eq!(expected_hash, blake3::Hash::from([0x44; 32])); 85 | assert!(is_escaped); 86 | assert_eq!(file_string, "fo\\r\\n\\n\\ro"); 87 | assert_eq!(file_path, Path::new("fo\r\n\n\ro")); 88 | 89 | // Escaped newline and backslash. Again because backslash is not allowed on 90 | // Windows, this test is Unix-only. 91 | if cfg!(not(windows)) { 92 | let crate::ParsedCheckLine { 93 | file_string, 94 | is_escaped, 95 | file_path, 96 | expected_hash, 97 | } = crate::parse_check_line( 98 | "\\4545454545454545454545454545454545454545454545454545454545454545 fo\\n\\\\o", 99 | ) 100 | .unwrap(); 101 | assert_eq!(expected_hash, blake3::Hash::from([0x45; 32])); 102 | assert!(is_escaped); 103 | assert_eq!(file_string, "fo\\n\\\\o"); 104 | assert_eq!(file_path, Path::new("fo\n\\o")); 105 | } 106 | 107 | // non-ASCII path 108 | let crate::ParsedCheckLine { 109 | file_string, 110 | is_escaped, 111 | file_path, 112 | expected_hash, 113 | } = crate::parse_check_line( 114 | "4646464646464646464646464646464646464646464646464646464646464646 否认", 115 | ) 116 | .unwrap(); 117 | assert_eq!(expected_hash, blake3::Hash::from([0x46; 32])); 118 | assert!(!is_escaped); 119 | assert_eq!(file_string, "否认"); 120 | assert_eq!(file_path, Path::new("否认")); 121 | 122 | // untagged separator " " in the file name 123 | let crate::ParsedCheckLine { 124 | file_string, 125 | is_escaped, 126 | file_path, 127 | expected_hash, 128 | } = crate::parse_check_line( 129 | "4747474747474747474747474747474747474747474747474747474747474747 foo bar", 130 | ) 131 | .unwrap(); 132 | assert_eq!(expected_hash, blake3::Hash::from([0x47; 32])); 133 | assert!(!is_escaped); 134 | assert_eq!(file_string, "foo bar"); 135 | assert_eq!(file_path, Path::new("foo bar")); 136 | 137 | // tagged separator ") = " in the file name 138 | let crate::ParsedCheckLine { 139 | file_string, 140 | is_escaped, 141 | file_path, 142 | expected_hash, 143 | } = crate::parse_check_line( 144 | "BLAKE3 (foo) = bar) = 4848484848484848484848484848484848484848484848484848484848484848", 145 | ) 146 | .unwrap(); 147 | assert_eq!(expected_hash, blake3::Hash::from([0x48; 32])); 148 | assert!(!is_escaped); 149 | assert_eq!(file_string, "foo) = bar"); 150 | assert_eq!(file_path, Path::new("foo) = bar")); 151 | 152 | // ========================= 153 | // ===== Failure Cases ===== 154 | // ========================= 155 | 156 | // too short 157 | crate::parse_check_line("").unwrap_err(); 158 | crate::parse_check_line("0").unwrap_err(); 159 | crate::parse_check_line("00").unwrap_err(); 160 | crate::parse_check_line("0000000000000000000000000000000000000000000000000000000000000000") 161 | .unwrap_err(); 162 | crate::parse_check_line("0000000000000000000000000000000000000000000000000000000000000000 ") 163 | .unwrap_err(); 164 | 165 | // not enough spaces 166 | crate::parse_check_line("0000000000000000000000000000000000000000000000000000000000000000 foo") 167 | .unwrap_err(); 168 | 169 | // capital letter hex 170 | crate::parse_check_line( 171 | "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA foo", 172 | ) 173 | .unwrap_err(); 174 | 175 | // non-hex hex 176 | crate::parse_check_line( 177 | "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx foo", 178 | ) 179 | .unwrap_err(); 180 | 181 | // non-ASCII hex 182 | crate::parse_check_line("你好, 我叫杰克. 认识你很高兴. 要不要吃个香蕉? foo").unwrap_err(); 183 | 184 | // invalid escape sequence 185 | crate::parse_check_line( 186 | "\\0000000000000000000000000000000000000000000000000000000000000000 fo\\o", 187 | ) 188 | .unwrap_err(); 189 | 190 | // truncated escape sequence 191 | crate::parse_check_line( 192 | "\\0000000000000000000000000000000000000000000000000000000000000000 foo\\", 193 | ) 194 | .unwrap_err(); 195 | 196 | // null char 197 | crate::parse_check_line( 198 | "0000000000000000000000000000000000000000000000000000000000000000 fo\0o", 199 | ) 200 | .unwrap_err(); 201 | 202 | // Unicode replacement char 203 | crate::parse_check_line( 204 | "0000000000000000000000000000000000000000000000000000000000000000 fo�o", 205 | ) 206 | .unwrap_err(); 207 | 208 | // On Windows only, backslashes are not allowed, escaped or otherwise. 209 | if cfg!(windows) { 210 | crate::parse_check_line( 211 | "0000000000000000000000000000000000000000000000000000000000000000 fo\\o", 212 | ) 213 | .unwrap_err(); 214 | crate::parse_check_line( 215 | "\\0000000000000000000000000000000000000000000000000000000000000000 fo\\\\o", 216 | ) 217 | .unwrap_err(); 218 | } 219 | } 220 | 221 | #[test] 222 | fn test_filepath_to_string() { 223 | let output = crate::filepath_to_string(Path::new("foo")); 224 | assert_eq!(output.filepath_string, "foo"); 225 | assert!(!output.is_escaped); 226 | 227 | let output = crate::filepath_to_string(Path::new("f\\ \t\r\noo")); 228 | if cfg!(windows) { 229 | // We normalize backslashes to forward slashes on Windows. 230 | assert_eq!(output.filepath_string, "f/ \t\\r\\noo"); 231 | } else { 232 | assert_eq!(output.filepath_string, "f\\\\ \t\\r\\noo"); 233 | } 234 | assert!(output.is_escaped); 235 | } 236 | -------------------------------------------------------------------------------- /b3sum/what_does_check_do.md: -------------------------------------------------------------------------------- 1 | # How does `b3sum --check` behave exactly?
or: Are filepaths...text? 2 | 3 | Most of the time, `b3sum --check` is a drop-in replacement for `md5sum --check` 4 | and other Coreutils hashing tools. It consumes a checkfile (the output of a 5 | regular `b3sum` command), re-hashes all the files listed there, and returns 6 | success if all of those hashes are still correct. What makes this more 7 | complicated than it might seem, is that representing filepaths as text means we 8 | need to consider many possible edge cases of unrepresentable filepaths. This 9 | document describes all of these edge cases in detail. 10 | 11 | > [!CAUTION] 12 | > `b3sum --check` (like all the Coreutils `--check` features) tells you whether 13 | > some _filepaths_ have changed, but it can't tell you whether a _directory_ 14 | > has changed in general. If you create a checkfile with something like `b3sum 15 | > my_dir/* > CHECKFILE`, then `b3sum --check CHECKFILE` will succeed even after 16 | > _new files_ are added to `my_dir`. Adding new files without changing anything 17 | > else is often enough to execute arbitrary code, for example by shadowing an 18 | > `import` in Python, or by installing something in `.git/hooks`. This is 19 | > confusing enough that I don't recommend using `--check` as a security tool in 20 | > new code. 21 | 22 | ## The simple case 23 | 24 | Here's the result of running `b3sum a b c/d` in a directory that contains 25 | those three files: 26 | 27 | ```bash 28 | $ echo hi > a 29 | $ echo lo > b 30 | $ mkdir c 31 | $ echo stuff > c/d 32 | $ b3sum a b c/d 33 | 0b8b60248fad7ac6dfac221b7e01a8b91c772421a15b387dd1fb2d6a94aee438 a 34 | 6ae4a57bbba24f79c461d30bcb4db973b9427d9207877e34d2d74528daa84115 b 35 | 2d477356c962e54784f1c5dc5297718d92087006f6ee96b08aeaf7f3cd252377 c/d 36 | ``` 37 | 38 | If we pipe that output into `b3sum --check`, it will exit with status zero 39 | (success) and print: 40 | 41 | ```bash 42 | $ b3sum a b c/d | b3sum --check 43 | a: OK 44 | b: OK 45 | c/d: OK 46 | ``` 47 | 48 | If we delete `b` and change the contents of `c/d`, and then use the same 49 | checkfile as above, `b3sum --check` will exit with a non-zero status (failure) 50 | and print: 51 | 52 | ```bash 53 | $ b3sum a b c/d > checkfile 54 | $ rm b 55 | $ echo more stuff >> c/d 56 | $ b3sum --check checkfile 57 | a: OK 58 | b: FAILED (No such file or directory (os error 2)) 59 | c/d: FAILED 60 | ``` 61 | 62 | In these typical cases, `b3sum` and `md5sum` have identical output for success 63 | and very similar output for failure. 64 | 65 | ## Escaping newlines and backslashes 66 | 67 | Since the checkfile format (the regular output format of `b3sum`) is 68 | newline-separated text, we need to worry about what happens when a filepath 69 | contains a newline, or worse. Suppose we create a file named `x[newline]x` 70 | (3 characters). One way to create such a file is with a Python one-liner like 71 | this: 72 | 73 | ```python 74 | >>> open("x\nx", "w") 75 | ``` 76 | 77 | Here's what happens when we hash that file with `b3sum`: 78 | 79 | ```bash 80 | $ b3sum x* 81 | \af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262 x\nx 82 | ``` 83 | 84 | Notice two things. First, `b3sum` puts a single `\` character at the front of 85 | the line. This indicates that the filepath contains escape sequences that 86 | `b3sum --check` will need to unescape. Then, `b3sum` replaces the newline 87 | character in the filepath with the two-character escape sequence `\n`. 88 | Similarly, if the filepath contained carriage returns or backslashes, `b3sum` 89 | would escape those as `\r` and `\\` in the output. So far, all of this behavior 90 | is still identical to `md5sum`. (Note: Coreutils [introduced `\r` 91 | escaping](https://github.com/coreutils/coreutils/commit/ed1c58427d574fb4ff0cb8f915eb0d554000ceeb) 92 | in v9.0, September 2021.) 93 | 94 | ## Invalid Unicode 95 | 96 | This is where `b3sum` and `md5sum` diverge. Apart from the newline and 97 | backslash escapes described above, `md5sum` copies all other filepath bytes 98 | verbatim to its output. That means its output encoding is "ASCII plus whatever 99 | bytes we got from the command line". This creates two problems: 100 | 101 | 1. Printing something that isn't UTF-8 is kind of gross. 102 | 2. Windows support. 103 | 104 | What's the deal with Windows? To start with, there's a fundamental difference 105 | in how Unix and Windows represent filepaths. Unix filepaths are "usually UTF-8" 106 | and Windows filepaths are "usually UTF-16". That means that a file named `abc` 107 | is typically represented as the bytes `[97, 98, 99]` on Unix and as the bytes 108 | `[97, 0, 98, 0, 99, 0]` on Windows. The `md5sum` approach won't work if we plan 109 | on creating a checkfile on Unix and checking it on Windows, or vice versa. 110 | 111 | A more portable approach is to convert platform-specific bytes into some 112 | consistent Unicode encoding. (In practice this is going to be UTF-8, but in 113 | theory it could be anything.) Then when `--check` needs to open a file, we 114 | convert the Unicode representation back into platform-specific bytes. This 115 | makes important common cases like `abc`, and in fact even `abc[newline]def`, 116 | work as expected. Great! 117 | 118 | But...what did we mean above when we said *usually* UTF-8 and *usually* UTF-16? 119 | It turns out that not every possible sequence of bytes is valid UTF-8, and not 120 | every possible sequence of 16-bit wide chars is valid UTF-16. For example, the 121 | byte 0xFF (255) can never appear in any UTF-8 string. If we ask Python to 122 | decode it, it yells at us: 123 | 124 | ```python 125 | >>> b"\xFF".decode("UTF-8") 126 | UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte 127 | ``` 128 | 129 | However, tragically, we *can* create a file with that byte in its name (on 130 | Linux at least, though not usually on macOS): 131 | 132 | ```python 133 | >>> open(b"y\xFFy", "w") 134 | ``` 135 | 136 | So some filepaths aren't representable in Unicode at all. Our plan to "convert 137 | platform-specific bytes into some consistent Unicode encoding" isn't going to 138 | work for everything. What does `b3sum` do with the file above? 139 | 140 | ```bash 141 | $ b3sum y* 142 | af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262 y�y 143 | ``` 144 | 145 | That � in there is a "Unicode replacement character". When we run into 146 | filepaths that we can't represent in Unicode, we replace the unrepresentable 147 | parts with these characters. On the checking side, to avoid any possible 148 | confusion between two different invalid filepaths, we automatically fail if we 149 | see a replacement character. Together with a few more details covered in the 150 | next section, this gives us an important set of properties: 151 | 152 | 1. Any file can be hashed locally. 153 | 2. Any file with a valid Unicode name not containing the � character can be 154 | checked. 155 | 3. Checking ambiguous or unrepresentable filepaths always fails. 156 | 4. Checkfiles are always valid UTF-8. 157 | 5. Checkfiles are portable between Unix and Windows. 158 | 159 | ## Formal Rules 160 | 161 | 1. When hashing, filepaths are represented in a platform-specific encoding, 162 | which can accommodate any filepath on the current platform. In Rust, this is 163 | `OsStr`/`OsString`. 164 | 2. In output, filepaths are first converted to UTF-8. Any non-Unicode segments 165 | are replaced with Unicode replacement characters (U+FFFD). In Rust, this is 166 | `OsStr::to_string_lossy`. 167 | 3. Then, if a filepath contains any backslashes (U+005C) or newlines (U+000A), 168 | these characters are escaped as `\\` and `\n` respectively. 169 | 4. Finally, any output line containing an escape sequence is prefixed with a 170 | single backslash. 171 | 5. When checking, each line is parsed as UTF-8, separated by a newline 172 | (U+000A). Invalid UTF-8 is an error. 173 | 6. Then, if a line begins with a backslash, the filepath component is 174 | unescaped. Any escape sequence other than `\\` or `\n` is an error. If a 175 | line does not begin with a backslash, unescaping is not performed, and any 176 | backslashes in the filepath component are interpreted literally. (`b3sum` 177 | output never contains unescaped backslashes, but they can occur in 178 | checkfiles assembled by hand.) 179 | 7. Finally, if a filepath contains a Unicode replacement character (U+FFFD) or 180 | a null character (U+0000), it is an error. 181 | 182 | **Additionally, on Windows only:** 183 | 184 | 8. In output, all backslashes (U+005C) are replaced with forward slashes 185 | (U+002F). 186 | 9. When checking, after unescaping, if a filepath contains a backslash, it is 187 | an error. 188 | -------------------------------------------------------------------------------- /c/cmake/BLAKE3/ContinuousIntegration.cmake: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.13 FATAL_ERROR) 2 | 3 | if(BUILD_SHARED_LIBS) 4 | message(FATAL_ERROR "BUILD_SHARED_LIBS is incompatible with BLAKE3_TESTING_CI") 5 | endif() 6 | 7 | include(CTest) 8 | 9 | # Declare a testing specific variant of the `blake3` library target. 10 | # 11 | # We use a separate library target in order to be able to perform compilation with various 12 | # combinations of features which are too noisy to specify in the main CMake config as options for 13 | # the normal `blake3` target. 14 | # 15 | # Initially this target has no properties but eventually we will populate them by copying all of the 16 | # relevant properties from the normal `blake3` target. 17 | add_library(blake3-testing 18 | blake3.c 19 | blake3_dispatch.c 20 | blake3_portable.c 21 | ) 22 | 23 | if(BLAKE3_USE_TBB AND TBB_FOUND) 24 | target_sources(blake3-testing 25 | PRIVATE 26 | blake3_tbb.cpp) 27 | endif() 28 | 29 | if(BLAKE3_SIMD_TYPE STREQUAL "amd64-asm") 30 | # Conditionally add amd64 asm files to `blake3-testing` sources 31 | if(MSVC) 32 | if(NOT BLAKE3_NO_AVX2) 33 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_avx2_x86-64_windows_msvc.asm) 34 | endif() 35 | if(NOT BLAKE3_NO_AVX512) 36 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_avx512_x86-64_windows_msvc.asm) 37 | endif() 38 | if(NOT BLAKE3_NO_SSE2) 39 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_sse2_x86-64_windows_msvc.asm) 40 | endif() 41 | if(NOT BLAKE3_NO_SSE41) 42 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_sse41_x86-64_windows_msvc.asm) 43 | endif() 44 | elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU" 45 | OR CMAKE_C_COMPILER_ID STREQUAL "Clang" 46 | OR CMAKE_C_COMPILER_ID STREQUAL "AppleClang") 47 | if (WIN32) 48 | if(NOT BLAKE3_NO_AVX2) 49 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_avx2_x86-64_windows_gnu.S) 50 | endif() 51 | if(NOT BLAKE3_NO_AVX512) 52 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_avx512_x86-64_windows_gnu.S) 53 | endif() 54 | if(NOT BLAKE3_NO_SSE2) 55 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_sse2_x86-64_windows_gnu.S) 56 | endif() 57 | if(NOT BLAKE3_NO_SSE41) 58 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_sse41_x86-64_windows_gnu.S) 59 | endif() 60 | elseif(UNIX) 61 | if(NOT BLAKE3_NO_AVX2) 62 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_avx2_x86-64_unix.S) 63 | endif() 64 | if(NOT BLAKE3_NO_AVX512) 65 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_avx512_x86-64_unix.S) 66 | endif() 67 | if(NOT BLAKE3_NO_SSE2) 68 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_sse2_x86-64_unix.S) 69 | endif() 70 | if(NOT BLAKE3_NO_SSE41) 71 | list(APPEND BLAKE3_TESTING_AMD64_ASM_SOURCES blake3_sse41_x86-64_unix.S) 72 | endif() 73 | endif() 74 | endif() 75 | target_sources(blake3-testing PRIVATE ${BLAKE3_AMD64_ASM_SOURCES}) 76 | elseif(BLAKE3_SIMD_TYPE STREQUAL "x86-intrinsics") 77 | # Conditionally add amd64 C files to `blake3-testing` sources 78 | if (NOT DEFINED BLAKE3_CFLAGS_SSE2 79 | OR NOT DEFINED BLAKE3_CFLAGS_SSE4.1 80 | OR NOT DEFINED BLAKE3_CFLAGS_AVX2 81 | OR NOT DEFINED BLAKE3_CFLAGS_AVX512) 82 | message(WARNING "BLAKE3_SIMD_TYPE is set to 'x86-intrinsics' but no compiler flags are available for the target architecture.") 83 | else() 84 | set(BLAKE3_SIMD_X86_INTRINSICS ON) 85 | endif() 86 | 87 | if(NOT BLAKE3_NO_AVX2) 88 | target_sources(blake3-testing PRIVATE blake3_avx2.c) 89 | set_source_files_properties(blake3_avx2.c PROPERTIES COMPILE_FLAGS "${BLAKE3_CFLAGS_AVX2}") 90 | endif() 91 | if(NOT BLAKE3_NO_AVX512) 92 | target_sources(blake3-testing PRIVATE blake3_avx512.c) 93 | set_source_files_properties(blake3_avx512.c PROPERTIES COMPILE_FLAGS "${BLAKE3_CFLAGS_AVX512}") 94 | endif() 95 | if(NOT BLAKE3_NO_SSE2) 96 | target_sources(blake3-testing PRIVATE blake3_sse2.c) 97 | set_source_files_properties(blake3_sse2.c PROPERTIES COMPILE_FLAGS "${BLAKE3_CFLAGS_SSE2}") 98 | endif() 99 | if(NOT BLAKE3_NO_SSE41) 100 | target_sources(blake3-testing PRIVATE blake3_sse41.c) 101 | set_source_files_properties(blake3_sse41.c PROPERTIES COMPILE_FLAGS "${BLAKE3_CFLAGS_SSE4.1}") 102 | endif() 103 | 104 | elseif(BLAKE3_SIMD_TYPE STREQUAL "neon-intrinsics") 105 | # Conditionally add neon C files to `blake3-testing` sources 106 | 107 | target_sources(blake3-testing PRIVATE 108 | blake3_neon.c 109 | ) 110 | target_compile_definitions(blake3-testing PRIVATE 111 | BLAKE3_USE_NEON=1 112 | ) 113 | 114 | if (DEFINED BLAKE3_CFLAGS_NEON) 115 | set_source_files_properties(blake3_neon.c PROPERTIES COMPILE_FLAGS "${BLAKE3_CFLAGS_NEON}") 116 | endif() 117 | 118 | elseif(BLAKE3_SIMD_TYPE STREQUAL "none") 119 | # Disable neon if simd type is "none". We check for individual amd64 features further below. 120 | 121 | target_compile_definitions(blake3-testing PRIVATE 122 | BLAKE3_USE_NEON=0 123 | ) 124 | 125 | endif() 126 | 127 | if(BLAKE3_NO_AVX2) 128 | target_compile_definitions(blake3-testing PRIVATE BLAKE3_NO_AVX2) 129 | endif() 130 | if(BLAKE3_NO_AVX512) 131 | target_compile_definitions(blake3-testing PRIVATE BLAKE3_NO_AVX512) 132 | endif() 133 | if(BLAKE3_NO_SSE2) 134 | target_compile_definitions(blake3-testing PRIVATE BLAKE3_NO_SSE2) 135 | endif() 136 | if(BLAKE3_NO_SSE41) 137 | target_compile_definitions(blake3-testing PRIVATE BLAKE3_NO_SSE41) 138 | endif() 139 | 140 | target_compile_definitions(blake3-testing PUBLIC BLAKE3_TESTING) 141 | 142 | get_target_property(BLAKE3_COMPILE_DEFINITIONS blake3 COMPILE_DEFINITIONS) 143 | if(BLAKE3_COMPILE_DEFINITIONS) 144 | target_compile_definitions(blake3-testing PUBLIC 145 | ${BLAKE3_COMPILE_DEFINITIONS}) 146 | endif() 147 | 148 | get_target_property(BLAKE3_COMPILE_OPTIONS blake3 COMPILE_OPTIONS) 149 | if(BLAKE3_COMPILE_OPTIONS) 150 | target_compile_options(blake3-testing PRIVATE 151 | ${BLAKE3_COMPILE_OPTIONS} 152 | -O3 153 | -Wall 154 | -Wextra 155 | -pedantic 156 | -fstack-protector-strong 157 | -D_FORTIFY_SOURCE=2 158 | -fPIE 159 | -fvisibility=hidden 160 | -fsanitize=address,undefined 161 | ) 162 | endif() 163 | 164 | get_target_property(BLAKE3_INCLUDE_DIRECTORIES blake3 INCLUDE_DIRECTORIES) 165 | if(BLAKE3_INCLUDE_DIRECTORIES) 166 | target_include_directories(blake3-testing PUBLIC 167 | $ 168 | $ 169 | ) 170 | endif() 171 | 172 | get_target_property(BLAKE3_LINK_LIBRARIES blake3 LINK_LIBRARIES) 173 | if(BLAKE3_LINK_LIBRARIES) 174 | target_link_libraries(blake3-testing PRIVATE ${BLAKE3_LINK_LIBRARIES}) 175 | endif() 176 | 177 | get_target_property(BLAKE3_LINK_OPTIONS blake3 LINK_OPTIONS) 178 | if(BLAKE3_LINK_OPTIONS) 179 | target_link_options(blake3-testing PRIVATE 180 | ${BLAKE3_LINK_OPTIONS} 181 | -fsanitize=address,undefined 182 | -pie 183 | -Wl,-z,relro,-z,now 184 | ) 185 | endif() 186 | 187 | # test asm target 188 | add_executable(blake3-asm-test 189 | main.c 190 | ) 191 | set_target_properties(blake3-asm-test PROPERTIES 192 | OUTPUT_NAME blake3 193 | RUNTIME_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}) 194 | target_link_libraries(blake3-asm-test PRIVATE blake3-testing) 195 | target_compile_definitions(blake3-asm-test PRIVATE BLAKE3_TESTING) 196 | target_compile_options(blake3-asm-test PRIVATE 197 | -O3 198 | -Wall 199 | -Wextra 200 | -pedantic 201 | -fstack-protector-strong 202 | -D_FORTIFY_SOURCE=2 203 | -fPIE 204 | -fvisibility=hidden 205 | -fsanitize=address,undefined 206 | ) 207 | target_link_options(blake3-asm-test PRIVATE 208 | -fsanitize=address,undefined 209 | -pie 210 | -Wl,-z,relro,-z,now 211 | ) 212 | 213 | add_test(NAME blake3-testing 214 | COMMAND "${CMAKE_CTEST_COMMAND}" 215 | --verbose 216 | --extra-verbose 217 | --build-and-test "${CMAKE_SOURCE_DIR}" "${CMAKE_BINARY_DIR}" 218 | --build-generator "${CMAKE_GENERATOR}" 219 | --build-makeprogram "${CMAKE_MAKE_PROGRAM}" 220 | --build-project libblake3 221 | --build-target blake3-asm-test 222 | --build-options 223 | --fresh 224 | "-DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}" 225 | "-DBLAKE3_TESTING=${BLAKE3_TESTING}" 226 | "-DBLAKE3_TESTING_CI=${BLAKE3_TESTING_CI}" 227 | "-DBLAKE3_USE_TBB=${BLAKE3_USE_TBB}" 228 | "-DBLAKE3_SIMD_TYPE=${BLAKE3_SIMD_TYPE}" 229 | "-DBLAKE3_NO_SSE2=${BLAKE3_NO_SSE2}" 230 | "-DBLAKE3_NO_SSE41=${BLAKE3_NO_SSE41}" 231 | "-DBLAKE3_NO_AVX2=${BLAKE3_NO_AVX2}" 232 | "-DBLAKE3_NO_AVX512=${BLAKE3_NO_AVX512}" 233 | --test-command 234 | "${CMAKE_SOURCE_DIR}/test.py" 235 | ) 236 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BLAKE3 2 | 3 | BLAKE3 is a cryptographic hash function that is: 4 | 5 | - **Much faster** than MD5, SHA-1, SHA-2, SHA-3, and BLAKE2. 6 | - **Secure**, unlike MD5 and SHA-1. And secure against length extension, 7 | unlike SHA-2. 8 | - **Highly parallelizable** across any number of threads and SIMD lanes, 9 | because it's a Merkle tree on the inside. 10 | - Capable of **verified streaming** and **incremental updates**, again 11 | because it's a Merkle tree. 12 | - A **PRF**, **MAC**, **KDF**, and **XOF**, as well as a regular hash. 13 | - **One algorithm with no variants**, which is fast on x86-64 and also 14 | on smaller architectures. 15 | 16 | The [chart below](https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/benchmarks/bar_chart.py) 17 | is an example benchmark of 16 KiB inputs on a Cascade Lake-SP 8275CL server CPU 18 | from 2019. For more detailed benchmarks, see the 19 | [BLAKE3 paper](https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf). 20 | 21 |

22 | performance graph 23 |

24 | 25 | BLAKE3 is based on an optimized instance of the established hash 26 | function [BLAKE2](https://blake2.net) and on the [original Bao tree 27 | mode](https://github.com/oconnor663/bao/blob/master/docs/spec_0.9.1.md). 28 | The specifications and design rationale are available in the [BLAKE3 29 | paper](https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf). 30 | The default output size is 256 bits. The current version of 31 | [Bao](https://github.com/oconnor663/bao) implements verified streaming 32 | with BLAKE3. 33 | 34 | This repository is the official implementation of BLAKE3. It includes: 35 | 36 | * The [`blake3`](https://crates.io/crates/blake3) Rust crate, which 37 | includes optimized implementations for SSE2, SSE4.1, AVX2, AVX-512, 38 | NEON, and WASM, with automatic runtime CPU feature detection on x86. 39 | The `rayon` feature provides multithreading. 40 | 41 | * The [`b3sum`](https://crates.io/crates/b3sum) Rust crate, which 42 | provides a command line interface. It uses multithreading by default, 43 | making it an order of magnitude faster than e.g. `sha256sum` on 44 | typical desktop hardware. 45 | 46 | * The [C implementation](c), which like the Rust implementation includes SIMD 47 | optimizations (all except WASM), CPU feature detection on x86, and optional 48 | multithreading. See [`c/README.md`](c/README.md). 49 | 50 | * The [Rust reference implementation](reference_impl/reference_impl.rs), 51 | which is discussed in Section 5.1 of the [BLAKE3 52 | paper](https://github.com/BLAKE3-team/BLAKE3-specs/blob/master/blake3.pdf). 53 | This implementation is much smaller and simpler than the optimized 54 | ones above. If you want to see how BLAKE3 works, or you're writing a 55 | port that doesn't need multithreading or SIMD optimizations, start 56 | here. Ports of the reference implementation to other languages are 57 | hosted in separate repositories 58 | ([C](https://github.com/oconnor663/blake3_reference_impl_c), 59 | [Python](https://github.com/oconnor663/pure_python_blake3)). 60 | 61 | * A [set of test 62 | vectors](https://github.com/BLAKE3-team/BLAKE3/blob/master/test_vectors/test_vectors.json) 63 | that covers extended outputs, all three modes, and a variety of input 64 | lengths. 65 | 66 | * [![Actions Status](https://github.com/BLAKE3-team/BLAKE3/workflows/tests/badge.svg)](https://github.com/BLAKE3-team/BLAKE3/actions) 67 | 68 | BLAKE3 was designed by: 69 | 70 | * [@oconnor663] (Jack O'Connor) 71 | * [@sneves] (Samuel Neves) 72 | * [@veorq] (Jean-Philippe Aumasson) 73 | * [@zookozcash] (Zooko) 74 | 75 | The development of BLAKE3 was sponsored by [Electric Coin Company](https://electriccoin.co). 76 | 77 | BLAKE3 is also [specified](https://c2sp.org/BLAKE3) in the [Community 78 | Cryptography Specification Project (C2SP)](https://c2sp.org). 79 | 80 | *NOTE: BLAKE3 is not a password hashing algorithm, because it's 81 | designed to be fast, whereas password hashing should not be fast. If you 82 | hash passwords to store the hashes or if you derive keys from passwords, 83 | we recommend [Argon2](https://github.com/P-H-C/phc-winner-argon2).* 84 | 85 | ## Usage 86 | 87 | ### The `b3sum` utility 88 | 89 | The `b3sum` command line utility prints the BLAKE3 hashes of files or of 90 | standard input. Prebuilt binaries are available for Linux, Windows, and 91 | macOS (requiring the [unidentified developer 92 | workaround](https://support.apple.com/guide/mac-help/open-a-mac-app-from-an-unidentified-developer-mh40616/mac)) 93 | on the [releases page](https://github.com/BLAKE3-team/BLAKE3/releases). 94 | If you've [installed Rust and 95 | Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html), 96 | you can also build `b3sum` yourself with: 97 | 98 | ```bash 99 | cargo install b3sum 100 | ``` 101 | 102 | If `rustup` didn't configure your `PATH` for you, you might need to go 103 | looking for the installed binary in e.g. `~/.cargo/bin`. You can test 104 | out how fast BLAKE3 is on your machine by creating a big file and 105 | hashing it, for example: 106 | 107 | ```bash 108 | # Create a 1 GB file. 109 | head -c 1000000000 /dev/zero > /tmp/bigfile 110 | # Hash it with SHA-256. 111 | time openssl sha256 /tmp/bigfile 112 | # Hash it with BLAKE3. 113 | time b3sum /tmp/bigfile 114 | ``` 115 | 116 | ### The `blake3` crate [![docs.rs](https://docs.rs/blake3/badge.svg)](https://docs.rs/blake3) 117 | 118 | To use BLAKE3 from Rust code, add a dependency on the `blake3` crate to 119 | your `Cargo.toml`. Here's an example of hashing some input bytes: 120 | 121 | ```rust 122 | // Hash an input all at once. 123 | let hash1 = blake3::hash(b"foobarbaz"); 124 | 125 | // Hash an input incrementally. 126 | let mut hasher = blake3::Hasher::new(); 127 | hasher.update(b"foo"); 128 | hasher.update(b"bar"); 129 | hasher.update(b"baz"); 130 | let hash2 = hasher.finalize(); 131 | assert_eq!(hash1, hash2); 132 | 133 | // Extended output. OutputReader also implements Read and Seek. 134 | let mut output = [0; 1000]; 135 | let mut output_reader = hasher.finalize_xof(); 136 | output_reader.fill(&mut output); 137 | assert_eq!(hash1, output[..32]); 138 | 139 | // Print a hash as hex. 140 | println!("{}", hash1); 141 | ``` 142 | 143 | Besides `hash`, BLAKE3 provides two other modes, `keyed_hash` and 144 | `derive_key`. The `keyed_hash` mode takes a 256-bit key: 145 | 146 | ```rust 147 | // MAC an input all at once. 148 | let example_key = [42u8; 32]; 149 | let mac1 = blake3::keyed_hash(&example_key, b"example input"); 150 | 151 | // MAC incrementally. 152 | let mut hasher = blake3::Hasher::new_keyed(&example_key); 153 | hasher.update(b"example input"); 154 | let mac2 = hasher.finalize(); 155 | assert_eq!(mac1, mac2); 156 | ``` 157 | 158 | The `derive_key` mode takes a context string and some key material (not a 159 | password). The context string should be hardcoded, globally unique, and 160 | application-specific. A good default format for the context string is 161 | `"[application] [commit timestamp] [purpose]"`: 162 | 163 | ```rust 164 | // Derive a couple of subkeys for different purposes. 165 | const EMAIL_CONTEXT: &str = "BLAKE3 example 2020-01-07 17:10:44 email key"; 166 | const API_CONTEXT: &str = "BLAKE3 example 2020-01-07 17:11:21 API key"; 167 | let input_key_material = b"usually at least 32 random bytes, not a password"; 168 | let email_key = blake3::derive_key(EMAIL_CONTEXT, input_key_material); 169 | let api_key = blake3::derive_key(API_CONTEXT, input_key_material); 170 | assert_ne!(email_key, api_key); 171 | ``` 172 | 173 | ### The C implementation 174 | 175 | See [`c/README.md`](c/README.md). 176 | 177 | ### Other implementations 178 | 179 | There are too many implementations out there for us to keep track of, 180 | but some highlights include [an optimized Go 181 | implementation](https://github.com/zeebo/blake3), [Wasm bindings for 182 | Node.js and browsers](https://github.com/connor4312/blake3), [binary 183 | wheels for Python](https://github.com/oconnor663/blake3-py), [.NET 184 | bindings](https://github.com/xoofx/Blake3.NET), and [a pure Java 185 | implementation](https://commons.apache.org/proper/commons-codec/apidocs/org/apache/commons/codec/digest/Blake3.html). 186 | 187 | ## Contributing 188 | 189 | Please see [CONTRIBUTING.md](CONTRIBUTING.md). 190 | 191 | ## Licenses 192 | 193 | This work is released into the public domain with [CC0 1.0](./LICENSE_CC0). 194 | Alternatively, it is licensed under any of the following: 195 | 196 | * [Apache 2.0](./LICENSE_A2) 197 | * [Apache 2.0 with LLVM exceptions](./LICENSE_A2LLVM) 198 | 199 | 200 | ## Adoption & deployment 201 | 202 | * [Bazel](https://github.com/bazelbuild/bazel/releases/tag/6.4.0) 203 | * [Cargo](https://github.com/rust-lang/cargo/pull/14137) 204 | * [Ccache](https://ccache.dev/releasenotes.html#_ccache_4_0) 205 | * [Chia](https://github.com/Chia-Network/chia-blockchain/blob/main/CHANGELOG.md#10beta8-aka-beta-18---2020-07-16) 206 | * [Clickhouse](https://github.com/ClickHouse/ClickHouse/blob/master/rust/chcache/Cargo.toml#L7) 207 | * [Farcaster](https://www.farcaster.xyz/) 208 | * [IPFS](https://github.com/ipfs/go-verifcid/issues/13) 209 | * [Iroh](https://www.iroh.computer/blog/blake3-hazmat-api) 210 | * [LLVM](https://reviews.llvm.org/D121510) 211 | * [Nix](https://github.com/NixOS/nix/pull/12379) 212 | * [Nym](https://github.com/nymtech/nym/blob/59056a22c5e6b01a38da2124662bd1fa3c8abef2/common/nymsphinx/params/src/lib.rs#L5) 213 | * [OpenZFS](https://github.com/openzfs/zfs/) 214 | * [Redox](https://www.redox-os.org/news/pkgar-introduction/) 215 | * [Solana](https://docs.rs/solana-program/1.9.5/solana_program/blake3/index.html) 216 | * [Tekken 8](https://x.com/rodarmor/status/1751567502050771189) 217 | * [Wasmer](https://github.com/wasmerio/wasmer/blob/4f935a8c162bf604df223003e434e4f7ca253688/lib/cache/src/hash.rs#L21) 218 | 219 | 220 | ## Miscellany 221 | 222 | - [@veorq] and [@oconnor663] did [an interview with Cryptography FM](https://cryptography.fireside.fm/3). 223 | - [@oconnor663] did [an interview with Saito](https://www.youtube.com/watch?v=cJkmIt7yN_E). 224 | 225 | [@oconnor663]: https://github.com/oconnor663 226 | [@sneves]: https://github.com/sneves 227 | [@veorq]: https://github.com/veorq 228 | [@zookozcash]: https://github.com/zookozcash 229 | -------------------------------------------------------------------------------- /c/blake3_dispatch.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "blake3_impl.h" 6 | 7 | #if defined(_MSC_VER) 8 | #include 9 | #endif 10 | 11 | #if defined(IS_X86) 12 | #if defined(_MSC_VER) 13 | #include 14 | #elif defined(__GNUC__) 15 | #include 16 | #else 17 | #undef IS_X86 /* Unimplemented! */ 18 | #endif 19 | #endif 20 | 21 | #if !defined(BLAKE3_ATOMICS) 22 | #if defined(__has_include) 23 | #if __has_include() && !defined(_MSC_VER) 24 | #define BLAKE3_ATOMICS 1 25 | #else 26 | #define BLAKE3_ATOMICS 0 27 | #endif /* __has_include() && !defined(_MSC_VER) */ 28 | #else 29 | #define BLAKE3_ATOMICS 0 30 | #endif /* defined(__has_include) */ 31 | #endif /* BLAKE3_ATOMICS */ 32 | 33 | #if BLAKE3_ATOMICS 34 | #define ATOMIC_INT _Atomic int 35 | #define ATOMIC_LOAD(x) x 36 | #define ATOMIC_STORE(x, y) x = y 37 | #elif defined(_MSC_VER) 38 | #define ATOMIC_INT LONG 39 | #define ATOMIC_LOAD(x) InterlockedOr(&x, 0) 40 | #define ATOMIC_STORE(x, y) InterlockedExchange(&x, y) 41 | #else 42 | #define ATOMIC_INT int 43 | #define ATOMIC_LOAD(x) x 44 | #define ATOMIC_STORE(x, y) x = y 45 | #endif 46 | 47 | #define MAYBE_UNUSED(x) (void)((x)) 48 | 49 | #if defined(IS_X86) 50 | static uint64_t xgetbv(void) { 51 | #if defined(_MSC_VER) 52 | return _xgetbv(0); 53 | #else 54 | uint32_t eax = 0, edx = 0; 55 | __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0)); 56 | return ((uint64_t)edx << 32) | eax; 57 | #endif 58 | } 59 | 60 | static void cpuid(uint32_t out[4], uint32_t id) { 61 | #if defined(_MSC_VER) 62 | __cpuid((int *)out, id); 63 | #elif defined(__i386__) || defined(_M_IX86) 64 | __asm__ __volatile__("movl %%ebx, %1\n" 65 | "cpuid\n" 66 | "xchgl %1, %%ebx\n" 67 | : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) 68 | : "a"(id)); 69 | #else 70 | __asm__ __volatile__("cpuid\n" 71 | : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) 72 | : "a"(id)); 73 | #endif 74 | } 75 | 76 | static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) { 77 | #if defined(_MSC_VER) 78 | __cpuidex((int *)out, id, sid); 79 | #elif defined(__i386__) || defined(_M_IX86) 80 | __asm__ __volatile__("movl %%ebx, %1\n" 81 | "cpuid\n" 82 | "xchgl %1, %%ebx\n" 83 | : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3]) 84 | : "a"(id), "c"(sid)); 85 | #else 86 | __asm__ __volatile__("cpuid\n" 87 | : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3]) 88 | : "a"(id), "c"(sid)); 89 | #endif 90 | } 91 | 92 | 93 | enum cpu_feature { 94 | SSE2 = 1 << 0, 95 | SSSE3 = 1 << 1, 96 | SSE41 = 1 << 2, 97 | AVX = 1 << 3, 98 | AVX2 = 1 << 4, 99 | AVX512F = 1 << 5, 100 | AVX512VL = 1 << 6, 101 | /* ... */ 102 | UNDEFINED = 1 << 30 103 | }; 104 | 105 | #if !defined(BLAKE3_TESTING) 106 | static /* Allow the variable to be controlled manually for testing */ 107 | #endif 108 | ATOMIC_INT g_cpu_features = UNDEFINED; 109 | 110 | #if !defined(BLAKE3_TESTING) 111 | static 112 | #endif 113 | enum cpu_feature 114 | get_cpu_features(void) { 115 | 116 | /* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */ 117 | enum cpu_feature features = ATOMIC_LOAD(g_cpu_features); 118 | if (features != UNDEFINED) { 119 | return features; 120 | } else { 121 | #if defined(IS_X86) 122 | uint32_t regs[4] = {0}; 123 | uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3]; 124 | (void)edx; 125 | features = 0; 126 | cpuid(regs, 0); 127 | const int max_id = *eax; 128 | cpuid(regs, 1); 129 | #if defined(__amd64__) || defined(_M_X64) 130 | features |= SSE2; 131 | #else 132 | if (*edx & (1UL << 26)) 133 | features |= SSE2; 134 | #endif 135 | if (*ecx & (1UL << 9)) 136 | features |= SSSE3; 137 | if (*ecx & (1UL << 19)) 138 | features |= SSE41; 139 | 140 | if (*ecx & (1UL << 27)) { // OSXSAVE 141 | const uint64_t mask = xgetbv(); 142 | if ((mask & 6) == 6) { // SSE and AVX states 143 | if (*ecx & (1UL << 28)) 144 | features |= AVX; 145 | if (max_id >= 7) { 146 | cpuidex(regs, 7, 0); 147 | if (*ebx & (1UL << 5)) 148 | features |= AVX2; 149 | if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm 150 | if (*ebx & (1UL << 31)) 151 | features |= AVX512VL; 152 | if (*ebx & (1UL << 16)) 153 | features |= AVX512F; 154 | } 155 | } 156 | } 157 | } 158 | ATOMIC_STORE(g_cpu_features, features); 159 | return features; 160 | #else 161 | /* How to detect NEON? */ 162 | return 0; 163 | #endif 164 | } 165 | } 166 | #endif 167 | 168 | void blake3_compress_in_place(uint32_t cv[8], 169 | const uint8_t block[BLAKE3_BLOCK_LEN], 170 | uint8_t block_len, uint64_t counter, 171 | uint8_t flags) { 172 | #if defined(IS_X86) 173 | const enum cpu_feature features = get_cpu_features(); 174 | MAYBE_UNUSED(features); 175 | #if !defined(BLAKE3_NO_AVX512) 176 | if (features & AVX512VL) { 177 | blake3_compress_in_place_avx512(cv, block, block_len, counter, flags); 178 | return; 179 | } 180 | #endif 181 | #if !defined(BLAKE3_NO_SSE41) 182 | if (features & SSE41) { 183 | blake3_compress_in_place_sse41(cv, block, block_len, counter, flags); 184 | return; 185 | } 186 | #endif 187 | #if !defined(BLAKE3_NO_SSE2) 188 | if (features & SSE2) { 189 | blake3_compress_in_place_sse2(cv, block, block_len, counter, flags); 190 | return; 191 | } 192 | #endif 193 | #endif 194 | blake3_compress_in_place_portable(cv, block, block_len, counter, flags); 195 | } 196 | 197 | void blake3_compress_xof(const uint32_t cv[8], 198 | const uint8_t block[BLAKE3_BLOCK_LEN], 199 | uint8_t block_len, uint64_t counter, uint8_t flags, 200 | uint8_t out[64]) { 201 | #if defined(IS_X86) 202 | const enum cpu_feature features = get_cpu_features(); 203 | MAYBE_UNUSED(features); 204 | #if !defined(BLAKE3_NO_AVX512) 205 | if (features & AVX512VL) { 206 | blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out); 207 | return; 208 | } 209 | #endif 210 | #if !defined(BLAKE3_NO_SSE41) 211 | if (features & SSE41) { 212 | blake3_compress_xof_sse41(cv, block, block_len, counter, flags, out); 213 | return; 214 | } 215 | #endif 216 | #if !defined(BLAKE3_NO_SSE2) 217 | if (features & SSE2) { 218 | blake3_compress_xof_sse2(cv, block, block_len, counter, flags, out); 219 | return; 220 | } 221 | #endif 222 | #endif 223 | blake3_compress_xof_portable(cv, block, block_len, counter, flags, out); 224 | } 225 | 226 | 227 | void blake3_xof_many(const uint32_t cv[8], 228 | const uint8_t block[BLAKE3_BLOCK_LEN], 229 | uint8_t block_len, uint64_t counter, uint8_t flags, 230 | uint8_t out[64], size_t outblocks) { 231 | if (outblocks == 0) { 232 | // The current assembly implementation always outputs at least 1 block. 233 | return; 234 | } 235 | #if defined(IS_X86) 236 | const enum cpu_feature features = get_cpu_features(); 237 | MAYBE_UNUSED(features); 238 | #if !defined(_WIN32) && !defined(__CYGWIN__) && !defined(BLAKE3_NO_AVX512) 239 | if (features & AVX512VL) { 240 | blake3_xof_many_avx512(cv, block, block_len, counter, flags, out, outblocks); 241 | return; 242 | } 243 | #endif 244 | #endif 245 | for(size_t i = 0; i < outblocks; ++i) { 246 | blake3_compress_xof(cv, block, block_len, counter + i, flags, out + 64*i); 247 | } 248 | } 249 | 250 | void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, 251 | size_t blocks, const uint32_t key[8], uint64_t counter, 252 | bool increment_counter, uint8_t flags, 253 | uint8_t flags_start, uint8_t flags_end, uint8_t *out) { 254 | #if defined(IS_X86) 255 | const enum cpu_feature features = get_cpu_features(); 256 | MAYBE_UNUSED(features); 257 | #if !defined(BLAKE3_NO_AVX512) 258 | if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { 259 | blake3_hash_many_avx512(inputs, num_inputs, blocks, key, counter, 260 | increment_counter, flags, flags_start, flags_end, 261 | out); 262 | return; 263 | } 264 | #endif 265 | #if !defined(BLAKE3_NO_AVX2) 266 | if (features & AVX2) { 267 | blake3_hash_many_avx2(inputs, num_inputs, blocks, key, counter, 268 | increment_counter, flags, flags_start, flags_end, 269 | out); 270 | return; 271 | } 272 | #endif 273 | #if !defined(BLAKE3_NO_SSE41) 274 | if (features & SSE41) { 275 | blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter, 276 | increment_counter, flags, flags_start, flags_end, 277 | out); 278 | return; 279 | } 280 | #endif 281 | #if !defined(BLAKE3_NO_SSE2) 282 | if (features & SSE2) { 283 | blake3_hash_many_sse2(inputs, num_inputs, blocks, key, counter, 284 | increment_counter, flags, flags_start, flags_end, 285 | out); 286 | return; 287 | } 288 | #endif 289 | #endif 290 | 291 | #if BLAKE3_USE_NEON == 1 292 | blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter, 293 | increment_counter, flags, flags_start, flags_end, out); 294 | return; 295 | #endif 296 | 297 | blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter, 298 | increment_counter, flags, flags_start, flags_end, 299 | out); 300 | } 301 | 302 | // The dynamically detected SIMD degree of the current platform. 303 | size_t blake3_simd_degree(void) { 304 | #if defined(IS_X86) 305 | const enum cpu_feature features = get_cpu_features(); 306 | MAYBE_UNUSED(features); 307 | #if !defined(BLAKE3_NO_AVX512) 308 | if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) { 309 | return 16; 310 | } 311 | #endif 312 | #if !defined(BLAKE3_NO_AVX2) 313 | if (features & AVX2) { 314 | return 8; 315 | } 316 | #endif 317 | #if !defined(BLAKE3_NO_SSE41) 318 | if (features & SSE41) { 319 | return 4; 320 | } 321 | #endif 322 | #if !defined(BLAKE3_NO_SSE2) 323 | if (features & SSE2) { 324 | return 4; 325 | } 326 | #endif 327 | #endif 328 | #if BLAKE3_USE_NEON == 1 329 | return 4; 330 | #endif 331 | return 1; 332 | } 333 | -------------------------------------------------------------------------------- /c/blake3_c_rust_bindings/build.rs: -------------------------------------------------------------------------------- 1 | use std::env; 2 | 3 | fn defined(var: &str) -> bool { 4 | env::var_os(var).is_some() 5 | } 6 | 7 | fn target_components() -> Vec { 8 | let target = env::var("TARGET").unwrap(); 9 | target.split("-").map(|s| s.to_string()).collect() 10 | } 11 | 12 | fn is_x86_64() -> bool { 13 | target_components()[0] == "x86_64" 14 | } 15 | 16 | fn is_windows_target() -> bool { 17 | env::var("CARGO_CFG_TARGET_OS").unwrap() == "windows" 18 | } 19 | 20 | fn use_msvc_asm() -> bool { 21 | const MSVC_NAMES: &[&str] = &["", "cl", "cl.exe"]; 22 | let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); 23 | let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); 24 | let target_windows_msvc = target_os == "windows" && target_env == "msvc"; 25 | let host_triple = env::var("HOST").unwrap_or_default(); 26 | let target_triple = env::var("TARGET").unwrap_or_default(); 27 | let cross_compiling = host_triple != target_triple; 28 | let cc = env::var("CC").unwrap_or_default().to_ascii_lowercase(); 29 | if !target_windows_msvc { 30 | // We are not building for Windows with the MSVC toolchain. 31 | false 32 | } else if !cross_compiling && MSVC_NAMES.contains(&&*cc) { 33 | // We are building on Windows with the MSVC toolchain (and not cross-compiling for another architecture or target). 34 | true 35 | } else { 36 | // We are cross-compiling to Windows with the MSVC toolchain. 37 | let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); 38 | let target_vendor = env::var("CARGO_CFG_TARGET_VENDOR").unwrap_or_default(); 39 | let cc = env::var(format!("CC_{target_arch}_{target_vendor}_windows_msvc")) 40 | .unwrap_or_default() 41 | .to_ascii_lowercase(); 42 | // Check if we are using the MSVC compiler. 43 | MSVC_NAMES.contains(&&*cc) 44 | } 45 | } 46 | 47 | fn is_x86_32() -> bool { 48 | let arch = &target_components()[0]; 49 | arch == "i386" || arch == "i586" || arch == "i686" 50 | } 51 | 52 | fn is_armv7() -> bool { 53 | target_components()[0] == "armv7" 54 | } 55 | 56 | fn is_aarch64() -> bool { 57 | target_components()[0] == "aarch64" 58 | } 59 | 60 | // Windows targets may be using the MSVC toolchain or the GNU toolchain. The 61 | // right compiler flags to use depend on the toolchain. (And we don't want to 62 | // use flag_if_supported, because we don't want features to be silently 63 | // disabled by old compilers.) 64 | fn is_windows_msvc() -> bool { 65 | // Some targets are only two components long, so check in steps. 66 | target_components()[1] == "pc" 67 | && target_components()[2] == "windows" 68 | && target_components()[3] == "msvc" 69 | } 70 | 71 | fn new_build() -> cc::Build { 72 | let mut build = cc::Build::new(); 73 | if !is_windows_msvc() { 74 | build.flag("-std=c11"); 75 | } 76 | build 77 | } 78 | 79 | fn new_cpp_build() -> cc::Build { 80 | let mut build = cc::Build::new(); 81 | build.cpp(true); 82 | if is_windows_msvc() { 83 | build.flag("/std:c++20"); 84 | build.flag("/EHs-c-"); 85 | build.flag("/GR-"); 86 | } else { 87 | build.flag("-std=c++20"); 88 | build.flag("-fno-exceptions"); 89 | build.flag("-fno-rtti"); 90 | } 91 | build 92 | } 93 | 94 | fn c_dir_path(filename: &str) -> String { 95 | // The `cross` tool doesn't support reading files in parent directories. As a hacky workaround 96 | // in `cross_test.sh`, we move the c/ directory around and set BLAKE3_C_DIR_OVERRIDE. Regular 97 | // building and testing doesn't require this. 98 | if let Ok(c_dir_override) = env::var("BLAKE3_C_DIR_OVERRIDE") { 99 | c_dir_override + "/" + filename 100 | } else { 101 | "../".to_string() + filename 102 | } 103 | } 104 | 105 | fn main() -> Result<(), Box> { 106 | let mut base_build = new_build(); 107 | base_build.file(c_dir_path("blake3.c")); 108 | base_build.file(c_dir_path("blake3_dispatch.c")); 109 | base_build.file(c_dir_path("blake3_portable.c")); 110 | if cfg!(feature = "tbb") { 111 | base_build.define("BLAKE3_USE_TBB", "1"); 112 | } 113 | base_build.compile("blake3_base"); 114 | 115 | if cfg!(feature = "tbb") { 116 | let mut tbb_build = new_cpp_build(); 117 | tbb_build.define("BLAKE3_USE_TBB", "1"); 118 | tbb_build.file(c_dir_path("blake3_tbb.cpp")); 119 | tbb_build.compile("blake3_tbb"); 120 | println!("cargo::rustc-link-lib=tbb"); 121 | } 122 | 123 | if is_x86_64() && !defined("CARGO_FEATURE_PREFER_INTRINSICS") { 124 | // On 64-bit, use the assembly implementations, unless the 125 | // "prefer_intrinsics" feature is enabled. 126 | if is_windows_target() { 127 | if use_msvc_asm() { 128 | let mut build = new_build(); 129 | build.file(c_dir_path("blake3_sse2_x86-64_windows_msvc.asm")); 130 | build.file(c_dir_path("blake3_sse41_x86-64_windows_msvc.asm")); 131 | build.file(c_dir_path("blake3_avx2_x86-64_windows_msvc.asm")); 132 | build.file(c_dir_path("blake3_avx512_x86-64_windows_msvc.asm")); 133 | build.compile("blake3_asm"); 134 | } else { 135 | let mut build = new_build(); 136 | build.file(c_dir_path("blake3_sse2_x86-64_windows_gnu.S")); 137 | build.file(c_dir_path("blake3_sse41_x86-64_windows_gnu.S")); 138 | build.file(c_dir_path("blake3_avx2_x86-64_windows_gnu.S")); 139 | build.file(c_dir_path("blake3_avx512_x86-64_windows_gnu.S")); 140 | build.compile("blake3_asm"); 141 | } 142 | } else { 143 | // All non-Windows implementations are assumed to support 144 | // Linux-style assembly. These files do contain a small 145 | // explicit workaround for macOS also. 146 | let mut build = new_build(); 147 | build.file(c_dir_path("blake3_sse2_x86-64_unix.S")); 148 | build.file(c_dir_path("blake3_sse41_x86-64_unix.S")); 149 | build.file(c_dir_path("blake3_avx2_x86-64_unix.S")); 150 | build.file(c_dir_path("blake3_avx512_x86-64_unix.S")); 151 | build.compile("blake3_asm"); 152 | } 153 | } else if is_x86_64() || is_x86_32() { 154 | // Assembly implementations are only for 64-bit. On 32-bit, or if 155 | // the "prefer_intrinsics" feature is enabled, use the 156 | // intrinsics-based C implementations. These each need to be 157 | // compiled separately, with the corresponding instruction set 158 | // extension explicitly enabled in the compiler. 159 | 160 | let mut sse2_build = new_build(); 161 | sse2_build.file(c_dir_path("blake3_sse2.c")); 162 | if is_windows_msvc() { 163 | // /arch:SSE2 is the default on x86 and undefined on x86_64: 164 | // https://docs.microsoft.com/en-us/cpp/build/reference/arch-x86 165 | // It also includes SSE4.1 intrinsics: 166 | // https://stackoverflow.com/a/32183222/823869 167 | } else { 168 | sse2_build.flag("-msse2"); 169 | } 170 | sse2_build.compile("blake3_sse2"); 171 | 172 | let mut sse41_build = new_build(); 173 | sse41_build.file(c_dir_path("blake3_sse41.c")); 174 | if is_windows_msvc() { 175 | // /arch:SSE2 is the default on x86 and undefined on x86_64: 176 | // https://docs.microsoft.com/en-us/cpp/build/reference/arch-x86 177 | // It also includes SSE4.1 intrinsics: 178 | // https://stackoverflow.com/a/32183222/823869 179 | } else { 180 | sse41_build.flag("-msse4.1"); 181 | } 182 | sse41_build.compile("blake3_sse41"); 183 | 184 | let mut avx2_build = new_build(); 185 | avx2_build.file(c_dir_path("blake3_avx2.c")); 186 | if is_windows_msvc() { 187 | avx2_build.flag("/arch:AVX2"); 188 | } else { 189 | avx2_build.flag("-mavx2"); 190 | } 191 | avx2_build.compile("blake3_avx2"); 192 | 193 | let mut avx512_build = new_build(); 194 | avx512_build.file(c_dir_path("blake3_avx512.c")); 195 | if is_windows_msvc() { 196 | // Note that a lot of versions of MSVC don't support /arch:AVX512, 197 | // and they'll discard it with a warning, hopefully leading to a 198 | // build error. 199 | avx512_build.flag("/arch:AVX512"); 200 | } else { 201 | avx512_build.flag("-mavx512f"); 202 | avx512_build.flag("-mavx512vl"); 203 | } 204 | avx512_build.compile("blake3_avx512"); 205 | } 206 | 207 | // We only build NEON code here if 208 | // 1) it's requested 209 | // and 2) the root crate is not already building it. 210 | // The only time this will really happen is if you build this 211 | // crate by hand with the "neon" feature for some reason. 212 | // 213 | // In addition, 3) if the target is aarch64, NEON is on by default. 214 | if defined("CARGO_FEATURE_NEON") || is_aarch64() { 215 | let mut neon_build = new_build(); 216 | neon_build.file(c_dir_path("blake3_neon.c")); 217 | // ARMv7 platforms that support NEON generally need the following 218 | // flags. AArch64 supports NEON by default and does not support -mpfu. 219 | if is_armv7() { 220 | neon_build.flag("-mfpu=neon-vfpv4"); 221 | neon_build.flag("-mfloat-abi=hard"); 222 | } 223 | neon_build.compile("blake3_neon"); 224 | } 225 | 226 | // The `cc` crate does not automatically emit rerun-if directives for the 227 | // environment variables it supports, in particular for $CC. We expect to 228 | // do a lot of benchmarking across different compilers, so we explicitly 229 | // add the variables that we're likely to need. 230 | println!("cargo:rerun-if-env-changed=CC"); 231 | println!("cargo:rerun-if-env-changed=CFLAGS"); 232 | 233 | // Ditto for source files, though these shouldn't change as often. `ignore::Walk` respects 234 | // .gitignore, so this doesn't traverse target/. 235 | for result in ignore::Walk::new("..") { 236 | let result = result?; 237 | let path = result.path(); 238 | if path.is_file() { 239 | println!("cargo:rerun-if-changed={}", path.to_str().unwrap()); 240 | } 241 | } 242 | 243 | // When compiling with clang-cl for windows, it adds .asm files to the root 244 | // which we need to delete so cargo doesn't get angry 245 | if is_windows_target() && !use_msvc_asm() { 246 | let _ = std::fs::remove_file("blake3_avx2_x86-64_windows_gnu.asm"); 247 | let _ = std::fs::remove_file("blake3_avx512_x86-64_windows_gnu.asm"); 248 | let _ = std::fs::remove_file("blake3_sse2_x86-64_windows_gnu.asm"); 249 | let _ = std::fs::remove_file("blake3_sse41_x86-64_windows_gnu.asm"); 250 | } 251 | 252 | Ok(()) 253 | } 254 | -------------------------------------------------------------------------------- /c/blake3_c_rust_bindings/src/lib.rs: -------------------------------------------------------------------------------- 1 | //! These are Rust bindings for the C implementation of BLAKE3. As there is a 2 | //! native (and faster) Rust implementation of BLAKE3 provided in this same 3 | //! repo, these bindings are not expected to be used in production. They're 4 | //! intended for testing and benchmarking. 5 | 6 | use std::ffi::{c_void, CString}; 7 | use std::mem::MaybeUninit; 8 | 9 | #[cfg(test)] 10 | mod test; 11 | 12 | pub const BLOCK_LEN: usize = 64; 13 | pub const CHUNK_LEN: usize = 1024; 14 | pub const OUT_LEN: usize = 32; 15 | 16 | // Feature detection functions for tests and benchmarks. Note that the C code 17 | // does its own feature detection in blake3_dispatch.c. 18 | #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] 19 | pub fn sse2_detected() -> bool { 20 | is_x86_feature_detected!("sse2") 21 | } 22 | 23 | #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] 24 | pub fn sse41_detected() -> bool { 25 | is_x86_feature_detected!("sse4.1") 26 | } 27 | 28 | #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] 29 | pub fn avx2_detected() -> bool { 30 | is_x86_feature_detected!("avx2") 31 | } 32 | 33 | #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] 34 | pub fn avx512_detected() -> bool { 35 | is_x86_feature_detected!("avx512f") && is_x86_feature_detected!("avx512vl") 36 | } 37 | 38 | #[derive(Clone)] 39 | pub struct Hasher(ffi::blake3_hasher); 40 | 41 | impl Hasher { 42 | pub fn new() -> Self { 43 | let mut c_state = MaybeUninit::uninit(); 44 | unsafe { 45 | ffi::blake3_hasher_init(c_state.as_mut_ptr()); 46 | Self(c_state.assume_init()) 47 | } 48 | } 49 | 50 | pub fn new_keyed(key: &[u8; 32]) -> Self { 51 | let mut c_state = MaybeUninit::uninit(); 52 | unsafe { 53 | ffi::blake3_hasher_init_keyed(c_state.as_mut_ptr(), key.as_ptr()); 54 | Self(c_state.assume_init()) 55 | } 56 | } 57 | 58 | pub fn new_derive_key(context: &str) -> Self { 59 | let mut c_state = MaybeUninit::uninit(); 60 | let context_c_string = CString::new(context).expect("valid C string, no null bytes"); 61 | unsafe { 62 | ffi::blake3_hasher_init_derive_key(c_state.as_mut_ptr(), context_c_string.as_ptr()); 63 | Self(c_state.assume_init()) 64 | } 65 | } 66 | 67 | pub fn new_derive_key_raw(context: &[u8]) -> Self { 68 | let mut c_state = MaybeUninit::uninit(); 69 | unsafe { 70 | ffi::blake3_hasher_init_derive_key_raw( 71 | c_state.as_mut_ptr(), 72 | context.as_ptr() as *const _, 73 | context.len(), 74 | ); 75 | Self(c_state.assume_init()) 76 | } 77 | } 78 | 79 | pub fn update(&mut self, input: &[u8]) { 80 | unsafe { 81 | ffi::blake3_hasher_update(&mut self.0, input.as_ptr() as *const c_void, input.len()); 82 | } 83 | } 84 | 85 | #[cfg(feature = "tbb")] 86 | pub fn update_tbb(&mut self, input: &[u8]) { 87 | unsafe { 88 | ffi::blake3_hasher_update_tbb( 89 | &mut self.0, 90 | input.as_ptr() as *const c_void, 91 | input.len(), 92 | ); 93 | } 94 | } 95 | 96 | pub fn finalize(&self, output: &mut [u8]) { 97 | unsafe { 98 | ffi::blake3_hasher_finalize(&self.0, output.as_mut_ptr(), output.len()); 99 | } 100 | } 101 | 102 | pub fn finalize_seek(&self, seek: u64, output: &mut [u8]) { 103 | unsafe { 104 | ffi::blake3_hasher_finalize_seek(&self.0, seek, output.as_mut_ptr(), output.len()); 105 | } 106 | } 107 | 108 | pub fn reset(&mut self) { 109 | unsafe { 110 | ffi::blake3_hasher_reset(&mut self.0); 111 | } 112 | } 113 | } 114 | 115 | pub mod ffi { 116 | #[repr(C)] 117 | #[derive(Copy, Clone)] 118 | pub struct blake3_chunk_state { 119 | pub cv: [u32; 8usize], 120 | pub chunk_counter: u64, 121 | pub buf: [u8; 64usize], 122 | pub buf_len: u8, 123 | pub blocks_compressed: u8, 124 | pub flags: u8, 125 | } 126 | 127 | #[repr(C)] 128 | #[derive(Copy, Clone)] 129 | pub struct blake3_hasher { 130 | pub key: [u32; 8usize], 131 | pub chunk: blake3_chunk_state, 132 | pub cv_stack_len: u8, 133 | pub cv_stack: [u8; 1728usize], 134 | } 135 | 136 | extern "C" { 137 | // public interface 138 | pub fn blake3_hasher_init(self_: *mut blake3_hasher); 139 | pub fn blake3_hasher_init_keyed(self_: *mut blake3_hasher, key: *const u8); 140 | pub fn blake3_hasher_init_derive_key( 141 | self_: *mut blake3_hasher, 142 | context: *const ::std::os::raw::c_char, 143 | ); 144 | pub fn blake3_hasher_init_derive_key_raw( 145 | self_: *mut blake3_hasher, 146 | context: *const ::std::os::raw::c_void, 147 | context_len: usize, 148 | ); 149 | pub fn blake3_hasher_update( 150 | self_: *mut blake3_hasher, 151 | input: *const ::std::os::raw::c_void, 152 | input_len: usize, 153 | ); 154 | #[cfg(feature = "tbb")] 155 | pub fn blake3_hasher_update_tbb( 156 | self_: *mut blake3_hasher, 157 | input: *const ::std::os::raw::c_void, 158 | input_len: usize, 159 | ); 160 | pub fn blake3_hasher_finalize(self_: *const blake3_hasher, out: *mut u8, out_len: usize); 161 | pub fn blake3_hasher_finalize_seek( 162 | self_: *const blake3_hasher, 163 | seek: u64, 164 | out: *mut u8, 165 | out_len: usize, 166 | ); 167 | pub fn blake3_hasher_reset(self_: *mut blake3_hasher); 168 | 169 | // portable low-level functions 170 | pub fn blake3_compress_in_place_portable( 171 | cv: *mut u32, 172 | block: *const u8, 173 | block_len: u8, 174 | counter: u64, 175 | flags: u8, 176 | ); 177 | pub fn blake3_compress_xof_portable( 178 | cv: *const u32, 179 | block: *const u8, 180 | block_len: u8, 181 | counter: u64, 182 | flags: u8, 183 | out: *mut u8, 184 | ); 185 | pub fn blake3_hash_many_portable( 186 | inputs: *const *const u8, 187 | num_inputs: usize, 188 | blocks: usize, 189 | key: *const u32, 190 | counter: u64, 191 | increment_counter: bool, 192 | flags: u8, 193 | flags_start: u8, 194 | flags_end: u8, 195 | out: *mut u8, 196 | ); 197 | } 198 | 199 | #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] 200 | pub mod x86 { 201 | extern "C" { 202 | // SSE2 low level functions 203 | pub fn blake3_compress_in_place_sse2( 204 | cv: *mut u32, 205 | block: *const u8, 206 | block_len: u8, 207 | counter: u64, 208 | flags: u8, 209 | ); 210 | pub fn blake3_compress_xof_sse2( 211 | cv: *const u32, 212 | block: *const u8, 213 | block_len: u8, 214 | counter: u64, 215 | flags: u8, 216 | out: *mut u8, 217 | ); 218 | pub fn blake3_hash_many_sse2( 219 | inputs: *const *const u8, 220 | num_inputs: usize, 221 | blocks: usize, 222 | key: *const u32, 223 | counter: u64, 224 | increment_counter: bool, 225 | flags: u8, 226 | flags_start: u8, 227 | flags_end: u8, 228 | out: *mut u8, 229 | ); 230 | 231 | // SSE4.1 low level functions 232 | pub fn blake3_compress_in_place_sse41( 233 | cv: *mut u32, 234 | block: *const u8, 235 | block_len: u8, 236 | counter: u64, 237 | flags: u8, 238 | ); 239 | pub fn blake3_compress_xof_sse41( 240 | cv: *const u32, 241 | block: *const u8, 242 | block_len: u8, 243 | counter: u64, 244 | flags: u8, 245 | out: *mut u8, 246 | ); 247 | pub fn blake3_hash_many_sse41( 248 | inputs: *const *const u8, 249 | num_inputs: usize, 250 | blocks: usize, 251 | key: *const u32, 252 | counter: u64, 253 | increment_counter: bool, 254 | flags: u8, 255 | flags_start: u8, 256 | flags_end: u8, 257 | out: *mut u8, 258 | ); 259 | 260 | // AVX2 low level functions 261 | pub fn blake3_hash_many_avx2( 262 | inputs: *const *const u8, 263 | num_inputs: usize, 264 | blocks: usize, 265 | key: *const u32, 266 | counter: u64, 267 | increment_counter: bool, 268 | flags: u8, 269 | flags_start: u8, 270 | flags_end: u8, 271 | out: *mut u8, 272 | ); 273 | 274 | // AVX-512 low level functions 275 | pub fn blake3_compress_xof_avx512( 276 | cv: *const u32, 277 | block: *const u8, 278 | block_len: u8, 279 | counter: u64, 280 | flags: u8, 281 | out: *mut u8, 282 | ); 283 | pub fn blake3_compress_in_place_avx512( 284 | cv: *mut u32, 285 | block: *const u8, 286 | block_len: u8, 287 | counter: u64, 288 | flags: u8, 289 | ); 290 | pub fn blake3_hash_many_avx512( 291 | inputs: *const *const u8, 292 | num_inputs: usize, 293 | blocks: usize, 294 | key: *const u32, 295 | counter: u64, 296 | increment_counter: bool, 297 | flags: u8, 298 | flags_start: u8, 299 | flags_end: u8, 300 | out: *mut u8, 301 | ); 302 | #[cfg(unix)] 303 | pub fn blake3_xof_many_avx512( 304 | cv: *const u32, 305 | block: *const u8, 306 | block_len: u8, 307 | counter: u64, 308 | flags: u8, 309 | out: *mut u8, 310 | outblocks: usize, 311 | ); 312 | } 313 | } 314 | 315 | #[cfg(feature = "neon")] 316 | pub mod neon { 317 | extern "C" { 318 | // NEON low level functions 319 | pub fn blake3_hash_many_neon( 320 | inputs: *const *const u8, 321 | num_inputs: usize, 322 | blocks: usize, 323 | key: *const u32, 324 | counter: u64, 325 | increment_counter: bool, 326 | flags: u8, 327 | flags_start: u8, 328 | flags_end: u8, 329 | out: *mut u8, 330 | ); 331 | } 332 | } 333 | } 334 | -------------------------------------------------------------------------------- /LICENSE_A2: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2019 Jack O'Connor and Samuel Neves 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /test_vectors/src/lib.rs: -------------------------------------------------------------------------------- 1 | use blake3::{BLOCK_LEN, CHUNK_LEN}; 2 | use serde::{Deserialize, Serialize}; 3 | 4 | // Reading files at runtime requires special configuration under WASM/WASI, so including this at 5 | // compile time is simpler. 6 | const TEST_VECTORS_JSON: &str = include_str!("../test_vectors.json"); 7 | 8 | // A non-multiple of 4 is important, since one possible bug is to fail to emit 9 | // partial words. 10 | pub const OUTPUT_LEN: usize = 2 * BLOCK_LEN + 3; 11 | 12 | pub const TEST_CASES: &[usize] = &[ 13 | 0, 14 | 1, 15 | 2, 16 | 3, 17 | 4, 18 | 5, 19 | 6, 20 | 7, 21 | 8, 22 | BLOCK_LEN - 1, 23 | BLOCK_LEN, 24 | BLOCK_LEN + 1, 25 | 2 * BLOCK_LEN - 1, 26 | 2 * BLOCK_LEN, 27 | 2 * BLOCK_LEN + 1, 28 | CHUNK_LEN - 1, 29 | CHUNK_LEN, 30 | CHUNK_LEN + 1, 31 | 2 * CHUNK_LEN, 32 | 2 * CHUNK_LEN + 1, 33 | 3 * CHUNK_LEN, 34 | 3 * CHUNK_LEN + 1, 35 | 4 * CHUNK_LEN, 36 | 4 * CHUNK_LEN + 1, 37 | 5 * CHUNK_LEN, 38 | 5 * CHUNK_LEN + 1, 39 | 6 * CHUNK_LEN, 40 | 6 * CHUNK_LEN + 1, 41 | 7 * CHUNK_LEN, 42 | 7 * CHUNK_LEN + 1, 43 | 8 * CHUNK_LEN, 44 | 8 * CHUNK_LEN + 1, 45 | 16 * CHUNK_LEN, // AVX512's bandwidth 46 | 31 * CHUNK_LEN, // 16 + 8 + 4 + 2 + 1 47 | 100 * CHUNK_LEN, // subtrees larger than MAX_SIMD_DEGREE chunks 48 | ]; 49 | 50 | pub const TEST_KEY: &[u8; blake3::KEY_LEN] = b"whats the Elvish word for friend"; 51 | pub const TEST_CONTEXT: &str = "BLAKE3 2019-12-27 16:29:52 test vectors context"; 52 | 53 | const COMMENT: &str = r#" 54 | Each test is an input length and three outputs, one for each of the hash, 55 | keyed_hash, and derive_key modes. The input in each case is filled with a 56 | repeating sequence of 251 bytes: 0, 1, 2, ..., 249, 250, 0, 1, ..., and so on. 57 | The key used with keyed_hash is the 32-byte ASCII string "whats the Elvish word 58 | for friend", also given in the `key` field below. The context string used with 59 | derive_key is the ASCII string "BLAKE3 2019-12-27 16:29:52 test vectors 60 | context", also given in the `context_string` field below. Outputs are encoded 61 | as hexadecimal. Each case is an extended output, and implementations should 62 | also check that the first 32 bytes match their default-length output. 63 | "#; 64 | 65 | // Paint the input with a repeating byte pattern. We use a cycle length of 251, 66 | // because that's the largest prime number less than 256. This makes it 67 | // unlikely to swapping any two adjacent input blocks or chunks will give the 68 | // same answer. 69 | pub fn paint_test_input(buf: &mut [u8]) { 70 | for (i, b) in buf.iter_mut().enumerate() { 71 | *b = (i % 251) as u8; 72 | } 73 | } 74 | 75 | #[derive(Debug, Serialize, Deserialize)] 76 | pub struct Cases { 77 | pub _comment: String, 78 | pub key: String, 79 | pub context_string: String, 80 | pub cases: Vec, 81 | } 82 | 83 | #[derive(Debug, Serialize, Deserialize)] 84 | pub struct Case { 85 | pub input_len: usize, 86 | pub hash: String, 87 | pub keyed_hash: String, 88 | pub derive_key: String, 89 | } 90 | 91 | pub fn generate_json() -> String { 92 | let mut cases = Vec::new(); 93 | for &input_len in TEST_CASES { 94 | let mut input = vec![0; input_len]; 95 | paint_test_input(&mut input); 96 | 97 | let mut hash_out = [0; OUTPUT_LEN]; 98 | blake3::Hasher::new() 99 | .update(&input) 100 | .finalize_xof() 101 | .fill(&mut hash_out); 102 | 103 | let mut keyed_hash_out = [0; OUTPUT_LEN]; 104 | blake3::Hasher::new_keyed(TEST_KEY) 105 | .update(&input) 106 | .finalize_xof() 107 | .fill(&mut keyed_hash_out); 108 | 109 | let mut derive_key_out = [0; OUTPUT_LEN]; 110 | blake3::Hasher::new_derive_key(TEST_CONTEXT) 111 | .update(&input) 112 | .finalize_xof() 113 | .fill(&mut derive_key_out); 114 | 115 | cases.push(Case { 116 | input_len, 117 | hash: hex::encode(&hash_out[..]), 118 | keyed_hash: hex::encode(&keyed_hash_out[..]), 119 | derive_key: hex::encode(&derive_key_out[..]), 120 | }); 121 | } 122 | 123 | let mut json = serde_json::to_string_pretty(&Cases { 124 | _comment: COMMENT.trim().replace("\n", " "), 125 | key: std::str::from_utf8(TEST_KEY).unwrap().to_string(), 126 | context_string: TEST_CONTEXT.to_string(), 127 | cases, 128 | }) 129 | .unwrap(); 130 | 131 | // Add a trailing newline. 132 | json.push('\n'); 133 | json 134 | } 135 | 136 | pub fn parse_test_cases() -> Cases { 137 | serde_json::from_str(TEST_VECTORS_JSON).expect("failed to parse test_vectors.json") 138 | } 139 | 140 | #[cfg(test)] 141 | mod tests { 142 | use super::*; 143 | 144 | fn test_reference_impl_all_at_once( 145 | key: &[u8; blake3::KEY_LEN], 146 | input: &[u8], 147 | expected_hash: &[u8], 148 | expected_keyed_hash: &[u8], 149 | expected_derive_key: &[u8], 150 | ) { 151 | let mut out = vec![0; expected_hash.len()]; 152 | let mut hasher = reference_impl::Hasher::new(); 153 | hasher.update(input); 154 | hasher.finalize(&mut out); 155 | assert_eq!(expected_hash, &out[..]); 156 | 157 | let mut out = vec![0; expected_keyed_hash.len()]; 158 | let mut hasher = reference_impl::Hasher::new_keyed(key); 159 | hasher.update(input); 160 | hasher.finalize(&mut out); 161 | assert_eq!(expected_keyed_hash, &out[..]); 162 | 163 | let mut out = vec![0; expected_derive_key.len()]; 164 | let mut hasher = reference_impl::Hasher::new_derive_key(TEST_CONTEXT); 165 | hasher.update(input); 166 | hasher.finalize(&mut out); 167 | assert_eq!(expected_derive_key, &out[..]); 168 | } 169 | 170 | fn test_reference_impl_one_at_a_time( 171 | key: &[u8; blake3::KEY_LEN], 172 | input: &[u8], 173 | expected_hash: &[u8], 174 | expected_keyed_hash: &[u8], 175 | expected_derive_key: &[u8], 176 | ) { 177 | let mut out = vec![0; expected_hash.len()]; 178 | let mut hasher = reference_impl::Hasher::new(); 179 | for &b in input { 180 | hasher.update(&[b]); 181 | } 182 | hasher.finalize(&mut out); 183 | assert_eq!(expected_hash, &out[..]); 184 | 185 | let mut out = vec![0; expected_keyed_hash.len()]; 186 | let mut hasher = reference_impl::Hasher::new_keyed(key); 187 | for &b in input { 188 | hasher.update(&[b]); 189 | } 190 | hasher.finalize(&mut out); 191 | assert_eq!(expected_keyed_hash, &out[..]); 192 | 193 | let mut out = vec![0; expected_derive_key.len()]; 194 | let mut hasher = reference_impl::Hasher::new_derive_key(TEST_CONTEXT); 195 | for &b in input { 196 | hasher.update(&[b]); 197 | } 198 | hasher.finalize(&mut out); 199 | assert_eq!(expected_derive_key, &out[..]); 200 | } 201 | 202 | fn test_incremental_all_at_once( 203 | key: &[u8; blake3::KEY_LEN], 204 | input: &[u8], 205 | expected_hash: &[u8], 206 | expected_keyed_hash: &[u8], 207 | expected_derive_key: &[u8], 208 | ) { 209 | let mut out = vec![0; expected_hash.len()]; 210 | let mut hasher = blake3::Hasher::new(); 211 | hasher.update(input); 212 | hasher.finalize_xof().fill(&mut out); 213 | assert_eq!(expected_hash, &out[..]); 214 | assert_eq!(&expected_hash[..32], hasher.finalize().as_bytes()); 215 | 216 | let mut out = vec![0; expected_keyed_hash.len()]; 217 | let mut hasher = blake3::Hasher::new_keyed(key); 218 | hasher.update(input); 219 | hasher.finalize_xof().fill(&mut out); 220 | assert_eq!(expected_keyed_hash, &out[..]); 221 | assert_eq!(&expected_keyed_hash[..32], hasher.finalize().as_bytes()); 222 | 223 | let mut out = vec![0; expected_derive_key.len()]; 224 | let mut hasher = blake3::Hasher::new_derive_key(TEST_CONTEXT); 225 | hasher.update(input); 226 | hasher.finalize_xof().fill(&mut out); 227 | assert_eq!(expected_derive_key, &out[..]); 228 | assert_eq!(&expected_derive_key[..32], hasher.finalize().as_bytes()); 229 | } 230 | 231 | fn test_incremental_one_at_a_time( 232 | key: &[u8; blake3::KEY_LEN], 233 | input: &[u8], 234 | expected_hash: &[u8], 235 | expected_keyed_hash: &[u8], 236 | expected_derive_key: &[u8], 237 | ) { 238 | let mut out = vec![0; expected_hash.len()]; 239 | let mut hasher = blake3::Hasher::new(); 240 | for i in 0..input.len() { 241 | hasher.update(&[input[i]]); 242 | assert_eq!(i as u64 + 1, hasher.count()); 243 | } 244 | hasher.finalize_xof().fill(&mut out); 245 | assert_eq!(expected_hash, &out[..]); 246 | assert_eq!(&expected_hash[..32], hasher.finalize().as_bytes()); 247 | 248 | let mut out = vec![0; expected_keyed_hash.len()]; 249 | let mut hasher = blake3::Hasher::new_keyed(key); 250 | for i in 0..input.len() { 251 | hasher.update(&[input[i]]); 252 | assert_eq!(i as u64 + 1, hasher.count()); 253 | } 254 | hasher.finalize_xof().fill(&mut out); 255 | assert_eq!(expected_keyed_hash, &out[..]); 256 | assert_eq!(&expected_keyed_hash[..32], hasher.finalize().as_bytes()); 257 | 258 | let mut out = vec![0; expected_derive_key.len()]; 259 | let mut hasher = blake3::Hasher::new_derive_key(TEST_CONTEXT); 260 | for i in 0..input.len() { 261 | hasher.update(&[input[i]]); 262 | assert_eq!(i as u64 + 1, hasher.count()); 263 | } 264 | hasher.finalize_xof().fill(&mut out); 265 | assert_eq!(expected_derive_key, &out[..]); 266 | assert_eq!(&expected_derive_key[..32], hasher.finalize().as_bytes()); 267 | } 268 | 269 | fn test_recursive( 270 | key: &[u8; blake3::KEY_LEN], 271 | input: &[u8], 272 | expected_hash: &[u8], 273 | expected_keyed_hash: &[u8], 274 | expected_derive_key: &[u8], 275 | ) { 276 | assert_eq!(&expected_hash[..32], blake3::hash(input).as_bytes()); 277 | assert_eq!( 278 | &expected_keyed_hash[..32], 279 | blake3::keyed_hash(key, input).as_bytes(), 280 | ); 281 | assert_eq!( 282 | expected_derive_key[..32], 283 | blake3::derive_key(TEST_CONTEXT, input) 284 | ); 285 | } 286 | 287 | #[test] 288 | fn run_test_vectors() { 289 | let cases = parse_test_cases(); 290 | let key: &[u8; blake3::KEY_LEN] = cases.key.as_bytes().try_into().unwrap(); 291 | for case in &cases.cases { 292 | dbg!(case.input_len); 293 | let mut input = vec![0; case.input_len]; 294 | paint_test_input(&mut input); 295 | let expected_hash = hex::decode(&case.hash).unwrap(); 296 | let expected_keyed_hash = hex::decode(&case.keyed_hash).unwrap(); 297 | let expected_derive_key = hex::decode(&case.derive_key).unwrap(); 298 | 299 | test_reference_impl_all_at_once( 300 | key, 301 | &input, 302 | &expected_hash, 303 | &expected_keyed_hash, 304 | &expected_derive_key, 305 | ); 306 | 307 | test_reference_impl_one_at_a_time( 308 | key, 309 | &input, 310 | &expected_hash, 311 | &expected_keyed_hash, 312 | &expected_derive_key, 313 | ); 314 | 315 | test_incremental_all_at_once( 316 | key, 317 | &input, 318 | &expected_hash, 319 | &expected_keyed_hash, 320 | &expected_derive_key, 321 | ); 322 | 323 | test_incremental_one_at_a_time( 324 | key, 325 | &input, 326 | &expected_hash, 327 | &expected_keyed_hash, 328 | &expected_derive_key, 329 | ); 330 | 331 | test_recursive( 332 | key, 333 | &input, 334 | &expected_hash, 335 | &expected_keyed_hash, 336 | &expected_derive_key, 337 | ); 338 | } 339 | } 340 | 341 | #[test] 342 | fn test_checked_in_vectors_up_to_date() { 343 | // Replace Windows newlines, in case Git is configured to alter 344 | // newlines when files are checked out. 345 | let json = TEST_VECTORS_JSON.replace("\r\n", "\n"); 346 | if generate_json() != json { 347 | panic!("Checked-in test_vectors.json is not up to date. Regenerate with `cargo run --bin generate > ./test_vectors.json`."); 348 | } 349 | } 350 | } 351 | -------------------------------------------------------------------------------- /LICENSE_A2LLVM: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Jack O'Connor and Samuel Neves 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | 204 | ---- LLVM Exceptions to the Apache 2.0 License ---- 205 | 206 | As an exception, if, as a result of your compiling your source code, portions 207 | of this Software are embedded into an Object form of such source code, you 208 | may redistribute such embedded portions in such Object form without complying 209 | with the conditions of Sections 4(a), 4(b) and 4(d) of the License. 210 | 211 | In addition, if you combine or link compiled forms of this Software with 212 | software that is licensed under the GPLv2 ("Combined Software") and if a 213 | court of competent jurisdiction determines that the patent provision (Section 214 | 3), the indemnity provision (Section 9) or other Section of the License 215 | conflicts with the conditions of the GPLv2, you may retroactively and 216 | prospectively choose to deem waived or otherwise exclude such Section(s) of 217 | the License, but only in their entirety and only with respect to the Combined 218 | Software. 219 | 220 | --------------------------------------------------------------------------------