├── .appveyor.yml ├── .cirrus.yml ├── .github └── FUNDING.yml ├── .gitignore ├── .gitmodules ├── .travis.yml ├── .whitesource ├── AvalancheTest.cpp ├── AvalancheTest.h ├── Bitslice.cpp ├── Bitvec.cpp ├── Bitvec.h ├── CMakeLists.txt ├── CONTRIBUTING.md ├── City.cpp ├── City.h ├── CityCrc.h ├── CityTest.cpp ├── DifferentialTest.cpp ├── DifferentialTest.h ├── FarmTest.cc ├── HashMapTest.cpp ├── HashMapTest.h ├── Hashes.cpp ├── Hashes.h ├── HighwayHash.cpp ├── KeysetTest.cpp ├── KeysetTest.h ├── MurmurHash1.cpp ├── MurmurHash1.h ├── MurmurHash2.cpp ├── MurmurHash2.h ├── MurmurHash3.cpp ├── MurmurHash3.h ├── PMP_Multilinear.cpp ├── PMP_Multilinear.h ├── PMP_Multilinear_64.cpp ├── PMP_Multilinear_64.h ├── PMP_Multilinear_64_out_32.h ├── PMP_Multilinear_common.h ├── PMP_Multilinear_common_naive.h ├── PMP_Multilinear_naive.h ├── PMP_Multilinear_naive_64.h ├── PMP_Multilinear_test.cpp ├── PMP_Multilinear_test.h ├── PMP_Multilinear_test_naive.cpp ├── PMP_Multilinear_test_naive.h ├── PMP_Multilinear_test_speed_alt.h ├── PMurHash.c ├── PMurHash.h ├── Platform.cpp ├── Platform.h ├── README.md ├── Random.cpp ├── Random.h ├── SpeedTest.cpp ├── SpeedTest.h ├── Spooky.cpp ├── Spooky.h ├── SpookyTest.cpp ├── Stats.cpp ├── Stats.h ├── SuperFastHash.cpp ├── Types.cpp ├── Types.h ├── beamsplitter.cpp ├── beamsplitter.h ├── blake2b.c ├── blake2s.c ├── blake3 ├── LICENSE ├── README.md ├── blake3.c ├── blake3.h ├── blake3_avx2.c ├── blake3_avx512.c ├── blake3_dispatch.c ├── blake3_impl.h ├── blake3_neon.c ├── blake3_portable.c └── blake3_sse41.c ├── build-msys2.bat ├── build.sh ├── build32.sh ├── chaskey.c ├── checkdoctests.pl ├── clhash.c ├── clhash.h ├── cmetrohash.h ├── cmetrohash64.c ├── crc.cpp ├── crc32-generated-constants.cpp ├── crc32-pclmul_asm.S ├── crc32_hw.c ├── crc32_hw1.c ├── crc32c.cpp ├── crc32c.h ├── discohash.cpp ├── discohash.h ├── doc ├── BEBB4185 ├── BadHash ├── City128 ├── City32 ├── City64 ├── City64low ├── City64noSeed ├── CityCrc128 ├── Crap8 ├── FNV1A_Pippip_Yurii ├── FNV1A_Totenschiff ├── FNV1a ├── FNV1a_YT ├── FNV2 ├── FNV64 ├── FarmHash128 ├── FarmHash32 ├── FarmHash64 ├── GoodOAAT ├── HalfSipHash ├── HighwayHash64 ├── JenkinsOOAT ├── JenkinsOOAT_perl ├── MUM ├── MUMlow ├── MeowHash ├── MeowHash32low ├── MicroOAAT ├── Murmur2 ├── Murmur2A ├── Murmur2B ├── Murmur2C ├── Murmur3A ├── Murmur3C ├── Murmur3F ├── MurmurOAAT ├── NOP_OAAT_read64 ├── PMPML_32 ├── PMPML_64 ├── PMurHash32 ├── SipHash ├── SipHash13 ├── Spooky128 ├── Spooky32 ├── Spooky64 ├── TSip ├── VHASH_32 ├── VHASH_64 ├── aesnihash ├── beamsplitter ├── bernstein ├── blake2b-160 ├── blake2b-224 ├── blake2b-256 ├── blake2b-256_64 ├── blake2s-128 ├── blake2s-160 ├── blake2s-224 ├── blake2s-256 ├── blake2s-256_64 ├── blake3_c ├── chaskey ├── clhash ├── cmetrohash64_1 ├── cmetrohash64_1o ├── cmetrohash64_2 ├── crc32 ├── crc32_hw ├── crc32_hw1 ├── crc32_pclmul ├── crc64_hw ├── crcutil ├── donothing128 ├── donothing32 ├── donothing64 ├── falkhash ├── farmhash128_c ├── farmhash32_c ├── farmhash64_c ├── farsh32 ├── farsh64 ├── fasthash32 ├── fasthash64 ├── fibonacci ├── fletcher2 ├── fletcher4 ├── floppsyhash_64 ├── hasshe2 ├── jodyhash32 ├── jodyhash64 ├── lookup3 ├── md5-128 ├── md5_32a ├── metrohash128 ├── metrohash128_1 ├── metrohash128_2 ├── metrohash128crc_1 ├── metrohash128crc_2 ├── metrohash64 ├── metrohash64_1 ├── metrohash64_2 ├── metrohash64crc_1 ├── metrohash64crc_2 ├── mirhash ├── mirhash32low ├── mirhashstrict ├── mirhashstrict32low ├── multiply_shift ├── pair_multiply_shift ├── rmd128 ├── rmd160 ├── rmd256 ├── sdbm ├── seahash ├── seahash32low ├── sha1-160 ├── sha1_32a ├── sha1ni ├── sha1ni_32 ├── sha2-224 ├── sha2-224_64 ├── sha2-256 ├── sha2-256_64 ├── sha2ni-256 ├── sha2ni-256_64 ├── sha3-256 ├── sha3-256_64 ├── sumhash ├── sumhash32 ├── superfast ├── t1ha0_32be ├── t1ha0_32le ├── t1ha0_aes_avx1 ├── t1ha0_aes_avx2 ├── t1ha0_aes_noavx ├── t1ha1_64be ├── t1ha1_64le ├── t1ha2_atonce ├── t1ha2_atonce128 ├── t1ha2_stream ├── t1ha2_stream128 ├── table.html ├── tifuhash_64 ├── wyhash ├── wyhash32low ├── x17 ├── xxHash32 ├── xxHash64 ├── xxh128 ├── xxh128low ├── xxh3 └── xxh3low ├── extra.lst ├── falkhash-elf64.o ├── falkhash-macho64.o ├── falkhash.asm ├── farmhash-c-test.cc ├── farmhash-c.c ├── farmhash-c.h ├── farmhash.cc ├── farmhash.h ├── farsh.c ├── farsh.h ├── fasthash.cpp ├── fasthash.h ├── fhtw-elf64.o ├── fhtw-macho64.o ├── fhtw.asm ├── fhtw.h ├── fixupdocspeeds.pl ├── fixupdoctests.pl ├── floppsyhash.cpp ├── floppsyhash.h ├── halfsiphash.c ├── hasshe2.c ├── jody_hash32.c ├── jody_hash32.h ├── jody_hash64.c ├── jody_hash64.h ├── log.hashes ├── log.hashmap ├── log.speed ├── lookup3.cpp ├── main.cpp ├── md5.cpp ├── md5.h ├── meow_hash_x64_aesni.h ├── metrohash ├── metrohash.h ├── metrohash128.cpp ├── metrohash128.h ├── metrohash128crc.cpp ├── metrohash128crc.h ├── metrohash64.cpp ├── metrohash64.h ├── metrohash64crc.cpp ├── metrohash64crc.h └── platform.h ├── mir-hash.h ├── mum.cc ├── mum.h ├── opt_cmetrohash.h ├── opt_cmetrohash64_1.c ├── os_dependent_stuff.asm ├── parallel_hashmap ├── LICENSE ├── README.md ├── meminfo.h ├── phmap.h ├── phmap.natvis ├── phmap_base.h ├── phmap_bits.h ├── phmap_config.h ├── phmap_dump.h ├── phmap_fwd_decl.h └── phmap_utils.h ├── rijndael-alg-fst.c ├── rijndael-alg-fst.h ├── rmd128.c ├── rmd160.c ├── rmd256.c ├── sanity.sh ├── seahash.c ├── sha1.cpp ├── sha1.h ├── sha2 ├── sha224.c ├── sha256.c ├── sha512_224.c └── sha512_256.c ├── sha3.c ├── siphash.c ├── siphash.h ├── siphash_impl.h ├── siphash_sse2.c ├── siphash_ssse3.c ├── speed.pl ├── speed.sh ├── speedall.sh ├── split.pl ├── t1ha.h ├── t1ha ├── t1ha0.c ├── t1ha0_ia32aes_a.h ├── t1ha0_ia32aes_avx.c ├── t1ha0_ia32aes_avx2.c ├── t1ha0_ia32aes_b.h ├── t1ha0_ia32aes_noavx.c ├── t1ha1.c ├── t1ha2.c └── t1ha_bits.h ├── testall.sh ├── testextra.sh ├── testgood-but-poor.sh ├── testhashmaps.sh ├── testpar-good.sh ├── testpar.sh ├── testpar1.sh ├── testpar2.sh ├── testpoor-but-good.sh ├── testspeed.sh ├── tifuhash.cpp ├── tifuhash.h ├── tomcrypt.h ├── tomcrypt_argchk.h ├── tomcrypt_cfg.h ├── tomcrypt_cipher.h ├── tomcrypt_custom.h ├── tomcrypt_hash.h ├── tomcrypt_mac.h ├── tomcrypt_macros.h ├── tomcrypt_math.h ├── tomcrypt_misc.h ├── tomcrypt_pk.h ├── tomcrypt_pkcs.h ├── tomcrypt_prng.h ├── tsip.c ├── vmac.cpp ├── vmac.h ├── wyhash.h ├── xxh3.h ├── xxhash.c └── xxhash.h /.appveyor.yml: -------------------------------------------------------------------------------- 1 | version: 0.2.{build} 2 | max_jobs: 4 3 | os: Visual Studio 2017 4 | environment: 5 | matrix: 6 | - MSVC: 1 7 | PLATFORM: x64 8 | - MSVC: 1 9 | PLATFORM: x86 10 | #- MINGW: 1 11 | # MSYS2_ARCH: x86_64 12 | # MSYSTEM: MINGW64 13 | # ARCH: win64 14 | # PLATFORM: x64 15 | #- MINGW: 1 16 | # MSYS2_ARCH: i686 17 | # MSYSTEM: MINGW32 18 | # ARCH: win32 19 | # PLATFORM: x86 20 | clone_depth: 1 21 | skip_tags: true 22 | build: 23 | verbosity: detailed 24 | 25 | init: 26 | - git config --global core.autocrlf input 27 | # Disable popups as they hang the build as there is nobody to click on the OK button... 28 | # Hanging the build is a lot less user friendly than reporting a build failure. 29 | # 30 | # Disable of system hard error popup 31 | # See: https://msdn.microsoft.com/en-us/library/bb513638%28VS.85%29.aspx 32 | - reg add "HKLM\SYSTEM\CurrentControlSet\Control\Windows" /f /v ErrorMode /d 2 33 | # Disable the following popup on program failure: 34 | # | ** has stopped working ** | 35 | # | Windows can check online for a solution to the problem| 36 | # | - Check online for a solution and close the program | 37 | # | - Close the program | 38 | # See: https://msdn.microsoft.com/en-us/library/bb513638%28VS.85%29.aspx 39 | - reg add "HKLM\Software\Microsoft\Windows\Windows Error Reporting" /f /v DontShowUI /d 1 40 | - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) 41 | # git bash conflicts with MinGW makefiles, esp. the space in 'Program Files' 42 | #- if "%MINGW%"=="1" set "PATH=%PATH:C:\Program Files\Git\usr\bin;=%" 43 | - set PROCESSOR_ARCHITECTURE=%PLATFORM% 44 | - if "%PLATFORM%"=="x64" set PROCESSOR_ARCHITECTURE=AMD64 45 | 46 | #- if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" (call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" amd64) 47 | #- if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2015" (call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64) 48 | #- if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2013" (call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" amd64) 49 | # call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" 50 | 51 | build_script: 52 | #- if "%MINGW%"=="1" call build-msys2.bat 53 | #- if "%MINGW%"=="1" bash -c "cmake ." 54 | #- if "%MINGW%"=="1" make VERBOSE=1 55 | #- if "%MSVC%"=="1" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" %PLATFORM% 56 | - if "%APPVEYOR_BUILD_WORKER_IMAGE%" == "Visual Studio 2017" (call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" %PLATFORM%) 57 | - if "%PLATFORM%"=="x64" cmake . -DCMAKE_BUILD_TYPE=Release -A %PLATFORM% 58 | - if "%PLATFORM%"=="x86" cmake . -DCMAKE_BUILD_TYPE=Release 59 | - if "%MSVC%"=="1" cmake --build . --config Release 60 | 61 | test_script: 62 | - Release\SMHasher --test=VerifyAll,Sanity,Speed,Cyclic,Zeroes,Seed 63 | -------------------------------------------------------------------------------- /.cirrus.yml: -------------------------------------------------------------------------------- 1 | # Build configuration for https://cirrus-ci.com/github/rurban/smhasher 2 | 3 | freebsd_instance: 4 | image_family: freebsd-12-1 5 | 6 | task: 7 | install_script: 8 | - pkg install -y cmake git 9 | - git submodule update --init 10 | build_script: 11 | - cmake . 12 | - make VERBOSE=1 13 | test_script: 14 | - make test ARGS=-V 15 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: rurban 4 | patreon: rurban 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | /build 3 | /.svn 4 | /build32 5 | libSMHasherSupport.a 6 | /TAGS 7 | 8 | SMHasher 9 | 10 | README.speed 11 | YoshimitsuTRIAD/ 12 | Yoshimura/ 13 | crcutil-1.0/ 14 | crcutil/ 15 | doc/FarmHash128.fast 16 | falkhash/ 17 | fast-hash/ 18 | fsbench-0.14/ 19 | fsbench-density/ 20 | *.bak 21 | prof.h 22 | /zlib/ 23 | /clhash/ 24 | 25 | # Apple artefact 26 | .DS_Store 27 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "SHA-Intrinsics"] 2 | path = SHA-Intrinsics 3 | url = https://github.com/noloader/SHA-Intrinsics 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: 3 | - c++ 4 | sudo: false 5 | matrix: 6 | fast_finish: true 7 | include: 8 | - compiler: gcc 9 | env: XCC=gcc XCXX=g++ 10 | - compiler: clang 11 | env: XCC=clang XCXX=clang++ 12 | - env: XCC="gcc -m32" XCXX="g++ -m32" 13 | addons: 14 | apt: 15 | packages: 16 | - g++-multilib 17 | - gcc-multilib 18 | 19 | script: 20 | - CC="$XCC" CXX="$XCXX" cmake . 21 | - make -j4 VERBOSE=1 22 | - make -j4 test ARGS=-V 23 | -------------------------------------------------------------------------------- /.whitesource: -------------------------------------------------------------------------------- 1 | { 2 | "generalSettings": { 3 | "shouldScanRepo": true 4 | }, 5 | "checkRunSettings": { 6 | "vulnerableCheckRunConclusionLevel": "failure" 7 | } 8 | } -------------------------------------------------------------------------------- /AvalancheTest.cpp: -------------------------------------------------------------------------------- 1 | #include "AvalancheTest.h" 2 | 3 | //----------------------------------------------------------------------------- 4 | 5 | void PrintAvalancheDiagram ( int x, int y, int reps, double scale, int * bins ) 6 | { 7 | const char * symbols = ".123456789X"; 8 | 9 | for(int i = 0; i < y; i++) 10 | { 11 | printf("["); 12 | for(int j = 0; j < x; j++) 13 | { 14 | int k = (y - i) -1; 15 | 16 | int bin = bins[k + (j*y)]; 17 | 18 | double b = double(bin) / double(reps); 19 | b = fabs(b*2 - 1); 20 | 21 | b *= scale; 22 | 23 | int s = (int)floor(b*10); 24 | 25 | if(s > 10) s = 10; 26 | if(s < 0) s = 0; 27 | 28 | printf("%c",symbols[s]); 29 | } 30 | 31 | printf("]\n"); 32 | fflush(NULL); 33 | } 34 | } 35 | 36 | //---------------------------------------------------------------------------- 37 | 38 | double maxBias ( std::vector & counts, int reps ) 39 | { 40 | double worst = 0; 41 | 42 | for(int i = 0; i < (int)counts.size(); i++) 43 | { 44 | double c = double(counts[i]) / double(reps); 45 | 46 | double d = fabs(c * 2 - 1); 47 | 48 | if(d > worst) 49 | { 50 | worst = d; 51 | } 52 | } 53 | 54 | return worst; 55 | } 56 | 57 | //----------------------------------------------------------------------------- 58 | -------------------------------------------------------------------------------- /Bitslice.cpp: -------------------------------------------------------------------------------- 1 | #include "Bitvec.h" 2 | #include 3 | #include 4 | 5 | // handle xnor 6 | 7 | typedef std::vector slice; 8 | typedef std::vector slice_vec; 9 | 10 | int countbits ( slice & v ) 11 | { 12 | int c = 0; 13 | 14 | for(size_t i = 0; i < v.size(); i++) 15 | { 16 | int d = countbits(v[i]); 17 | 18 | c += d; 19 | } 20 | 21 | return c; 22 | } 23 | 24 | int countxor ( slice & a, slice & b ) 25 | { 26 | assert(a.size() == b.size()); 27 | 28 | int c = 0; 29 | 30 | for(size_t i = 0; i < a.size(); i++) 31 | { 32 | int d = countbits(a[i] ^ b[i]); 33 | 34 | c += d; 35 | } 36 | 37 | return c; 38 | } 39 | 40 | void xoreq ( slice & a, slice & b ) 41 | { 42 | assert(a.size() == b.size()); 43 | 44 | for(size_t i = 0; i < a.size(); i++) 45 | { 46 | a[i] ^= b[i]; 47 | } 48 | } 49 | 50 | //----------------------------------------------------------------------------- 51 | // Bitslice a hash set 52 | 53 | template< typename hashtype > 54 | void Bitslice ( std::vector & hashes, slice_vec & slices ) 55 | { 56 | const int hashbytes = sizeof(hashtype); 57 | const int hashbits = hashbytes * 8; 58 | const int slicelen = ((int)hashes.size() + 31) / 32; 59 | 60 | slices.clear(); 61 | slices.resize(hashbits); 62 | 63 | for(int i = 0; i < (int)slices.size(); i++) 64 | { 65 | slices[i].resize(slicelen,0); 66 | } 67 | 68 | for(int j = 0; j < hashbits; j++) 69 | { 70 | void * sliceblob = &(slices[j][0]); 71 | 72 | for(int i = 0; i < (int)hashes.size(); i++) 73 | { 74 | int b = getbit(hashes[i],j); 75 | 76 | setbit(sliceblob,slicelen*4,i,b); 77 | } 78 | } 79 | } 80 | 81 | void FactorSlices ( slice_vec & slices ) 82 | { 83 | std::vector counts(slices.size(),0); 84 | 85 | for(size_t i = 0; i < slices.size(); i++) 86 | { 87 | counts[i] = countbits(slices[i]); 88 | } 89 | 90 | bool changed = true; 91 | 92 | while(changed) 93 | { 94 | int bestA = -1; 95 | int bestB = -1; 96 | 97 | for(int j = 0; j < (int)slices.size()-1; j++) 98 | { 99 | for(int i = j+1; i < (int)slices.size(); i++) 100 | { 101 | int d = countxor(slices[i],slices[j]); 102 | 103 | if((d < counts[i]) && (d < counts[j])) 104 | { 105 | if(counts[i] < counts[j]) 106 | { 107 | bestA = j; 108 | bestB = i; 109 | } 110 | } 111 | else if(d < counts[i]) 112 | { 113 | //bestA = 114 | } 115 | } 116 | } 117 | } 118 | } 119 | 120 | 121 | void foo ( void ) 122 | { 123 | slice a; 124 | slice_vec b; 125 | 126 | Bitslice(a,b); 127 | } -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to smhasher 2 | 3 | First off, thanks for taking the time to contribute! 4 | 5 | ## No Code of Conduct 6 | 7 | This project adheres to [No Code of Conduct](https://github.com/domgetter/NCoC#what-is-ncoc). 8 | We are all adults. 9 | We accept anyone's contributions. 10 | If you are feeling harassed speak up by yourself. 11 | We do not censor, we don't stop any discussions. 12 | Nothing else matters. 13 | 14 | ## New hash functions 15 | 16 | We take functions written in C++, C and sometimes even in ASM. 17 | The smokers only run on travis and appveyor, so only Ubuntu 18 | (g++ and clang++), darwin and Windows MSVC is tested, no mingw, no FreeBSD (yet). 19 | Note that Windows MSVC is only C89 or C++, but not C99, and a long is not long. 20 | Use stdint.h 21 | 22 | `git add submodule` is fine. 23 | 24 | ## New tests 25 | 26 | Please check that a change should not exceed the test function >5min, 27 | esp. for >=64bit hashes. A 2hr test (e.g. for 128bit) is unacceptable. 28 | We don't want to wait 4 days to run through all test for all hashes. 29 | Adjust the window length or repetition count on `info->hashbits` then. 30 | My testmachine is a MacAir 2013 i7-4650U with 8GB RAM on darwin, which has a 31 | faster CPU and libc than my big linux machine. 32 | 33 | Hide longer tests behind `--extra` (`g_testExtra`). 34 | 35 | 64bit hashes don't have collisions. 36 | A practically useful test tests only the 8 lowest bits, as they are the 37 | most common bits uses in a hash table. The highest bits are usually better, thus 38 | rather test the lower bits. 39 | 40 | Every test which breaks a hash function is appreciated. 41 | Look at the random crusher tests also (TestU01, Dieharder). 42 | -------------------------------------------------------------------------------- /City.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | // 21 | // CityHash, by Geoff Pike and Jyrki Alakuijala 22 | // 23 | // This file provides a few functions for hashing strings. On x86-64 24 | // hardware in 2011, CityHash64() is faster than other high-quality 25 | // hash functions, such as Murmur. This is largely due to higher 26 | // instruction-level parallelism. CityHash64() and CityHash128() also perform 27 | // well on hash-quality tests. 28 | // 29 | // CityHash128() is optimized for relatively long strings and returns 30 | // a 128-bit hash. For strings more than about 2000 bytes it can be 31 | // faster than CityHash64(). 32 | // 33 | // Functions in the CityHash family are not suitable for cryptography. 34 | // 35 | // WARNING: This code has not been tested on big-endian platforms! 36 | // It is known to work well on little-endian platforms that have a small penalty 37 | // for unaligned reads, such as current Intel and AMD moderate-to-high-end CPUs. 38 | // 39 | // By the way, for some hash functions, given strings a and b, the hash 40 | // of a+b is easily derived from the hashes of a and b. This property 41 | // doesn't hold for any hash functions in this file. 42 | 43 | #ifndef CITY_HASH_H_ 44 | #define CITY_HASH_H_ 45 | 46 | #include "Platform.h" 47 | #include // for size_t. 48 | //#include 49 | #include 50 | 51 | typedef uint8_t uint8; 52 | typedef uint32_t uint32; 53 | typedef uint64_t uint64; 54 | typedef std::pair uint128; 55 | 56 | inline uint64 Uint128Low64(const uint128& x) { return x.first; } 57 | inline uint64 Uint128High64(const uint128& x) { return x.second; } 58 | 59 | // Hash functions for a byte array. 60 | uint32 CityHash32WithSeed(const char *buf, size_t len, uint32 seed); 61 | uint64 CityHash64(const char *buf, size_t len); 62 | 63 | // Hash function for a byte array. For convenience, a 64-bit seed is also 64 | // hashed into the result. 65 | uint64 CityHash64WithSeed(const char *buf, size_t len, uint64 seed); 66 | 67 | // Hash function for a byte array. For convenience, two seeds are also 68 | // hashed into the result. 69 | uint64 CityHash64WithSeeds(const char *buf, size_t len, 70 | uint64 seed0, uint64 seed1); 71 | 72 | // Hash function for a byte array. 73 | uint128 CityHash128(const char *s, size_t len); 74 | 75 | // Hash function for a byte array. For convenience, a 128-bit seed is also 76 | // hashed into the result. 77 | uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed); 78 | 79 | // Hash 128 input bits down to 64 bits of output. 80 | // This is intended to be a reasonably good hash function. 81 | inline uint64 Hash128to64(const uint128& x) { 82 | // Murmur-inspired hashing. 83 | const uint64 kMul = 0x9ddfea08eb382d69ULL; 84 | uint64 a = (Uint128Low64(x) ^ Uint128High64(x)) * kMul; 85 | a ^= (a >> 47); 86 | uint64 b = (Uint128High64(x) ^ a) * kMul; 87 | b ^= (b >> 47); 88 | b *= kMul; 89 | return b; 90 | } 91 | 92 | // Conditionally include declarations for versions of City that require SSE4.2 93 | // instructions to be available. 94 | #if defined(__SSE4_2__) && defined(__x86_64__) 95 | 96 | // Hash function for a byte array. 97 | uint128 CityHashCrc128(const char *s, size_t len); 98 | 99 | // Hash function for a byte array. For convenience, a 128-bit seed is also 100 | // hashed into the result. 101 | uint128 CityHashCrc128WithSeed(const char *s, size_t len, uint128 seed); 102 | 103 | // Hash function for a byte array. Sets result[0] ... result[3]. 104 | void CityHashCrc256(const char *s, size_t len, uint64 *result); 105 | 106 | #endif // __SSE4_2__ 107 | 108 | #endif // CITY_HASH_H_ 109 | -------------------------------------------------------------------------------- /CityCrc.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | // 21 | // CityHash, by Geoff Pike and Jyrki Alakuijala 22 | // 23 | // This file declares the subset of the CityHash functions that require 24 | // _mm_crc32_u64(). See the CityHash README for details. 25 | // 26 | // Functions in the CityHash family are not suitable for cryptography. 27 | 28 | #ifndef CITY_HASH_CRC_H_ 29 | #define CITY_HASH_CRC_H_ 30 | 31 | #include "City.h" 32 | 33 | // Hash function for a byte array. 34 | uint128 CityHashCrc128(const char *s, size_t len); 35 | 36 | // Hash function for a byte array. For convenience, a 128-bit seed is also 37 | // hashed into the result. 38 | uint128 CityHashCrc128WithSeed(const char *s, size_t len, uint128 seed); 39 | 40 | // Hash function for a byte array. Sets result[0] ... result[3]. 41 | void CityHashCrc256(const char *s, size_t len, uint64 *result); 42 | 43 | #endif // CITY_HASH_CRC_H_ 44 | -------------------------------------------------------------------------------- /CityTest.cpp: -------------------------------------------------------------------------------- 1 | #include "City.h" 2 | #include "CityCrc.h" 3 | 4 | void CityHash32_test ( const void * key, int len, uint32_t seed, void * out ) 5 | { 6 | // objsize 0-527: 1319 7 | *(uint32*)out = CityHash32WithSeed((const char *)key,len,seed); 8 | } 9 | 10 | void CityHash64_test ( const void * key, int len, uint32_t seed, void * out ) 11 | { 12 | // objsize 1038 + a20-a72: 82 = 1120 13 | *(uint64*)out = CityHash64WithSeed((const char *)key,len,seed); 14 | } 15 | 16 | void CityHash64noSeed_test ( const void * key, int len, uint32_t seed, void * out ) 17 | { 18 | // objsize 530-93e: 1038 19 | *(uint64*)out = CityHash64((const char *)key,len); (void)seed; 20 | } 21 | 22 | #if defined(__SSE4_2__) && defined(__x86_64__) 23 | void CityHash128_test ( const void * key, int len, uint32_t seed, void * out ) 24 | { 25 | uint128 s(0,0); 26 | s.first = seed; 27 | 28 | // objsize ad0-1201: 1841 29 | *(uint128*)out = CityHash128WithSeed((const char*)key,len,s); 30 | } 31 | 32 | void CityHashCrc128_test ( const void * key, int len, uint32_t seed, void * out ) 33 | { 34 | uint128 s(0,0); 35 | s.first = seed; 36 | 37 | // objsize 1940-1a67: 295 38 | *(uint128*)out = CityHashCrc128WithSeed((const char*)key,len,s); 39 | } 40 | #endif 41 | -------------------------------------------------------------------------------- /DifferentialTest.cpp: -------------------------------------------------------------------------------- 1 | #include "DifferentialTest.h" 2 | 3 | //---------------------------------------------------------------------------- 4 | -------------------------------------------------------------------------------- /FarmTest.cc: -------------------------------------------------------------------------------- 1 | /* We have conflicting uint128_t types in Types.h (Blob<128>) and farmhash.h */ 2 | 3 | #undef FARMHASH_UINT128_T_DEFINED 4 | #ifdef HAVE_SSE42 5 | # define FARMHASH_ASSUME_SSE42 6 | #endif 7 | #ifdef HAVE_AESNI 8 | # define FARMHASH_ASSUME_AESNI 9 | #endif 10 | #include "farmhash.h" 11 | 12 | void FarmHash32_test ( const void * key, int len, uint32_t seed, void * out ) { 13 | using namespace NAMESPACE_FOR_HASH_FUNCTIONS; 14 | *(uint32_t*)out = Hash32WithSeed((const char *)key,(size_t)len,seed); 15 | } 16 | void FarmHash64_test ( const void * key, int len, uint32_t seed, void * out ) { 17 | using namespace NAMESPACE_FOR_HASH_FUNCTIONS; 18 | *(uint64_t*)out = Hash64WithSeed((const char *)key,(size_t)len,(uint64_t)seed); 19 | } 20 | void FarmHash64noSeed_test ( const void * key, int len, uint32_t seed, void * out ) { 21 | using namespace NAMESPACE_FOR_HASH_FUNCTIONS; 22 | *(uint64_t*)out = Hash64((const char *)key,(size_t)len); (void)seed; 23 | } 24 | void FarmHash128_test ( const void * key, int len, uint32_t seed, void * out ) { 25 | using namespace NAMESPACE_FOR_HASH_FUNCTIONS; 26 | uint128_t s((uint64_t)seed, (uint64_t)0UL); 27 | uint128_t result = Hash128WithSeed((const char *)key, (size_t)len, s); 28 | memcpy(out, &result, 128/8); 29 | } 30 | -------------------------------------------------------------------------------- /HashMapTest.cpp: -------------------------------------------------------------------------------- 1 | #include "Platform.h" 2 | #include "HashMapTest.h" 3 | #include "SpeedTest.h" 4 | #include "Random.h" 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | using namespace std; 13 | 14 | //----------------------------------------------------------------------------- 15 | // This should be a realistic I-Cache test, when our hash is used inlined 16 | // in a hash table. There the size matters more than the bulk speed. 17 | 18 | std::vector HashMapInit(bool verbose) { 19 | std::vector words; 20 | std::string line; 21 | std::string filename = "/usr/share/dict/words"; 22 | int lines = 0, sum = 0; 23 | std::ifstream wordfile(filename.c_str()); 24 | if (!wordfile.is_open()) { 25 | std::cout << "Unable to open words dict file " << filename << "\n"; 26 | return words; 27 | } 28 | while (getline(wordfile, line)) { 29 | int len = line.length(); 30 | lines++; 31 | sum += len; 32 | words.push_back(line); 33 | } 34 | wordfile.close(); 35 | if (verbose) { 36 | printf ("Read %d words from '%s'\n", lines, filename.c_str()); 37 | printf ("Avg len: %0.3f\n", (sum+0.0)/lines); 38 | } 39 | return words; 40 | } 41 | 42 | bool HashMapTest ( pfHash pfhash, 43 | const int hashbits, std::vector words, 44 | const int trials, bool verbose ) 45 | { 46 | double mean = 0.0; 47 | try { 48 | mean = HashMapSpeedTest( pfhash, hashbits, words, trials, verbose); 49 | } 50 | catch (...) { 51 | printf(" aborted !!!!\n"); 52 | } 53 | // if faster than ~sha1 54 | if (mean > 5. && mean < 1500.) 55 | printf(" ....... PASS\n"); 56 | else 57 | printf(" ....... FAIL\n"); 58 | return true; 59 | } 60 | -------------------------------------------------------------------------------- /HashMapTest.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Platform.h" 4 | #include "Types.h" 5 | 6 | std::vector HashMapInit(bool verbose); 7 | bool HashMapTest ( pfHash pfhash, 8 | const int hashbits, std::vector words, 9 | const int trials, bool verbose ); 10 | -------------------------------------------------------------------------------- /HighwayHash.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "highwayhash/c_bindings.h" 4 | 5 | static uint64_t HighwayHash_key[4]; 6 | 7 | void HighwayHash_init() 8 | { 9 | for(int i=0; i<4; i++){ 10 | uint64_t r = random(); 11 | memcpy(&HighwayHash_key[i], &r, 8); 12 | } 13 | } 14 | 15 | void HighwayHash64_test(const void *bytes, int len, uint32_t seed, void *out) 16 | { 17 | memcpy(&HighwayHash_key, &seed, 4); 18 | *(uint64_t*)out = HighwayHash64(HighwayHash_key, (const char*)bytes, (uint64_t)len); 19 | } 20 | -------------------------------------------------------------------------------- /MurmurHash1.cpp: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | // Note - This code makes a few assumptions about how your machine behaves - 6 | 7 | // 1. We can read a 4-byte value from any address without crashing 8 | // 2. sizeof(int) == 4 9 | 10 | // And it has a few limitations - 11 | 12 | // 1. It will not work incrementally. 13 | // 2. It will not produce the same results on little-endian and big-endian 14 | // machines. 15 | 16 | #include "MurmurHash1.h" 17 | 18 | //----------------------------------------------------------------------------- 19 | 20 | // objsize: 0-0x157: 343 21 | uint32_t MurmurHash1 ( const void * key, int len, uint32_t seed ) 22 | { 23 | const unsigned int m = 0xc6a4a793; 24 | 25 | const int r = 16; 26 | 27 | unsigned int h = seed ^ (len * m); 28 | 29 | //---------- 30 | 31 | const unsigned char * data = (const unsigned char *)key; 32 | 33 | while(len >= 4) 34 | { 35 | unsigned int k = *(unsigned int *)data; 36 | 37 | h += k; 38 | h *= m; 39 | h ^= h >> 16; 40 | 41 | data += 4; 42 | len -= 4; 43 | } 44 | 45 | //---------- 46 | 47 | switch(len) 48 | { 49 | case 3: 50 | h += data[2] << 16; 51 | case 2: 52 | h += data[1] << 8; 53 | case 1: 54 | h += data[0]; 55 | h *= m; 56 | h ^= h >> r; 57 | }; 58 | 59 | //---------- 60 | 61 | h *= m; 62 | h ^= h >> 10; 63 | h *= m; 64 | h ^= h >> 17; 65 | 66 | return h; 67 | } 68 | 69 | //----------------------------------------------------------------------------- 70 | // MurmurHash1Aligned, by Austin Appleby 71 | 72 | // Same algorithm as MurmurHash1, but only does aligned reads - should be safer 73 | // on certain platforms. 74 | 75 | // Performance should be equal to or better than the simple version. 76 | // objsize: 0x160-0x4e3: 899 77 | 78 | unsigned int MurmurHash1Aligned ( const void * key, int len, unsigned int seed ) 79 | { 80 | const unsigned int m = 0xc6a4a793; 81 | const int r = 16; 82 | 83 | const unsigned char * data = (const unsigned char *)key; 84 | 85 | unsigned int h = seed ^ (len * m); 86 | 87 | int align = (uint64_t)data & 3; 88 | 89 | if(align && (len >= 4)) 90 | { 91 | // Pre-load the temp registers 92 | 93 | unsigned int t = 0, d = 0; 94 | 95 | switch(align) 96 | { 97 | case 1: t |= data[2] << 16; 98 | case 2: t |= data[1] << 8; 99 | case 3: t |= data[0]; 100 | } 101 | 102 | t <<= (8 * align); 103 | 104 | data += 4-align; 105 | len -= 4-align; 106 | 107 | int sl = 8 * (4-align); 108 | int sr = 8 * align; 109 | 110 | // Mix 111 | 112 | while(len >= 4) 113 | { 114 | d = *(unsigned int *)data; 115 | t = (t >> sr) | (d << sl); 116 | h += t; 117 | h *= m; 118 | h ^= h >> r; 119 | t = d; 120 | 121 | data += 4; 122 | len -= 4; 123 | } 124 | 125 | // Handle leftover data in temp registers 126 | 127 | int pack = len < align ? len : align; 128 | 129 | d = 0; 130 | 131 | switch(pack) 132 | { 133 | case 3: d |= data[2] << 16; 134 | case 2: d |= data[1] << 8; 135 | case 1: d |= data[0]; 136 | case 0: h += (t >> sr) | (d << sl); 137 | h *= m; 138 | h ^= h >> r; 139 | } 140 | 141 | data += pack; 142 | len -= pack; 143 | } 144 | else 145 | { 146 | while(len >= 4) 147 | { 148 | h += *(unsigned int *)data; 149 | h *= m; 150 | h ^= h >> r; 151 | 152 | data += 4; 153 | len -= 4; 154 | } 155 | } 156 | 157 | //---------- 158 | // Handle tail bytes 159 | 160 | switch(len) 161 | { 162 | case 3: h += data[2] << 16; 163 | case 2: h += data[1] << 8; 164 | case 1: h += data[0]; 165 | h *= m; 166 | h ^= h >> r; 167 | }; 168 | 169 | h *= m; 170 | h ^= h >> 10; 171 | h *= m; 172 | h ^= h >> 17; 173 | 174 | return h; 175 | } 176 | 177 | -------------------------------------------------------------------------------- /MurmurHash1.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash1 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | #ifndef _MURMURHASH1_H_ 6 | #define _MURMURHASH1_H_ 7 | 8 | //----------------------------------------------------------------------------- 9 | // Platform-specific functions and macros 10 | 11 | // Microsoft Visual Studio 12 | 13 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 14 | 15 | typedef unsigned char uint8_t; 16 | typedef unsigned int uint32_t; 17 | typedef unsigned __int64 uint64_t; 18 | 19 | // Other compilers 20 | 21 | #else // defined(_MSC_VER) 22 | 23 | #include 24 | 25 | #endif // !defined(_MSC_VER) 26 | 27 | //----------------------------------------------------------------------------- 28 | 29 | uint32_t MurmurHash1 ( const void * key, int len, uint32_t seed ); 30 | uint32_t MurmurHash1Aligned ( const void * key, int len, uint32_t seed ); 31 | 32 | //----------------------------------------------------------------------------- 33 | 34 | #endif // _MURMURHASH1_H_ 35 | -------------------------------------------------------------------------------- /MurmurHash2.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash2 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | #ifndef _MURMURHASH2_H_ 6 | #define _MURMURHASH2_H_ 7 | 8 | //----------------------------------------------------------------------------- 9 | // Platform-specific functions and macros 10 | 11 | // Microsoft Visual Studio 12 | 13 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 14 | 15 | typedef unsigned char uint8_t; 16 | typedef unsigned int uint32_t; 17 | typedef unsigned __int64 uint64_t; 18 | 19 | // Other compilers 20 | 21 | #else // defined(_MSC_VER) 22 | 23 | #include 24 | 25 | #endif // !defined(_MSC_VER) 26 | 27 | //----------------------------------------------------------------------------- 28 | 29 | uint32_t MurmurHash2 ( const void * key, int len, uint32_t seed ); 30 | uint64_t MurmurHash64A ( const void * key, int len, uint64_t seed ); 31 | uint64_t MurmurHash64B ( const void * key, int len, uint64_t seed ); 32 | uint32_t MurmurHash2A ( const void * key, int len, uint32_t seed ); 33 | uint32_t MurmurHashNeutral2 ( const void * key, int len, uint32_t seed ); 34 | uint32_t MurmurHashAligned2 ( const void * key, int len, uint32_t seed ); 35 | 36 | //----------------------------------------------------------------------------- 37 | 38 | #endif // _MURMURHASH2_H_ 39 | 40 | -------------------------------------------------------------------------------- /MurmurHash3.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash3 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | #ifndef _MURMURHASH3_H_ 6 | #define _MURMURHASH3_H_ 7 | 8 | //----------------------------------------------------------------------------- 9 | // Platform-specific functions and macros 10 | 11 | // Microsoft Visual Studio 12 | 13 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 14 | 15 | typedef unsigned char uint8_t; 16 | typedef unsigned int uint32_t; 17 | typedef unsigned __int64 uint64_t; 18 | 19 | // Other compilers 20 | 21 | #else // defined(_MSC_VER) 22 | 23 | #include 24 | 25 | #endif // !defined(_MSC_VER) 26 | 27 | //----------------------------------------------------------------------------- 28 | 29 | void MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed, void * out ); 30 | 31 | void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out ); 32 | 33 | void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out ); 34 | 35 | //----------------------------------------------------------------------------- 36 | 37 | #endif // _MURMURHASH3_H_ 38 | -------------------------------------------------------------------------------- /PMP_Multilinear_test.cpp: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------------------- 2 | * Copyright (c) 2014, Dmytro Ivanchykhin, Sergey Ignatchenko, Daniel Lemire 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, this 9 | * list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright notice, 11 | * this list of conditions and the following disclaimer in the documentation 12 | * and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 | * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | * ------------------------------------------------------------------------------- 25 | * 26 | * PMP+-Multilinear hash family implementation 27 | * 28 | * v.1.00 Apr-14-2014 Initial release 29 | * 30 | * -------------------------------------------------------------------------------*/ 31 | 32 | // PMP_Multilinear_test.cpp: interface of PMP+-Multilinear hash family with SMHasher 33 | 34 | #include "Platform.h" 35 | 36 | #include "PMP_Multilinear_test.h" 37 | #include "PMP_Multilinear_common.h" 38 | 39 | #include 40 | class UniformRandomNumberGenerator_ : public UniformRandomNumberGenerator 41 | { 42 | public: 43 | uint32_t rand() { return (::rand() << 20) | (::rand() << 12) | ::rand(); } 44 | }; 45 | 46 | static UniformRandomNumberGenerator_ rng; 47 | 48 | 49 | 50 | #if !defined __arm__ 51 | #if (defined _WIN64) || (defined __x86_64__) 52 | 53 | #include "PMP_Multilinear_64_out_32.h" 54 | 55 | 56 | static PMP_Multilinear_Hasher_64_out_32 pmpml_hasher_64_out_32; 57 | 58 | void PMPML_64_CPP_out_32( const void * key, int len, uint32_t seed, void * res ) 59 | { 60 | //pmpml_hasher_64_out_32.seed (seed); 61 | *(uint64_t*)res = pmpml_hasher_64_out_32.hash( (const unsigned char*)key, len ); 62 | } 63 | 64 | void PMPML_64_CPP_out_32_randomize() 65 | { 66 | pmpml_hasher_64_out_32.randomize( rng ); 67 | } 68 | 69 | int PMPML_TestSpeedAlt_64_out_32( const void * key, int len, int iter ) 70 | { 71 | int dummy = 0; 72 | for ( int i=0; i 42 | 43 | void PMPML_32_CPP( const void * key, int len, uint32_t seed, void * res ); 44 | void PMPML_32_CPP_randomize(); 45 | 46 | #if defined(_WIN64) || defined(__x86_64__) 47 | void PMPML_64_CPP( const void * key, int len, uint32_t seed, void * res ); 48 | void PMPML_64_CPP_randomize(); 49 | 50 | void PMPML_64_CPP_out_32( const void * key, int len, uint32_t seed, void * res ); 51 | void PMPML_64_CPP_out_32_randomize(); 52 | int PMPML_TestSpeedAlt_64_out_32( const void * key, int len, int iter ); 53 | #endif 54 | 55 | #endif // __MULTILINEARPRIMESTRINGHASHFUNCTOR_TEST_H__ 56 | -------------------------------------------------------------------------------- /PMP_Multilinear_test_naive.cpp: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------------------- 2 | * Copyright (c) 2014, Dmytro Ivanchykhin, Sergey Ignatchenko, Daniel Lemire 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, this 9 | * list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright notice, 11 | * this list of conditions and the following disclaimer in the documentation 12 | * and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 | * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | * ------------------------------------------------------------------------------- 25 | * 26 | * PMP+-Multilinear hash family implementation 27 | * 28 | * v.1.00 Apr-14-2014 Initial release 29 | * 30 | * -------------------------------------------------------------------------------*/ 31 | 32 | // PMP_Multilinear_test_naive.cpp: interface of PMP+-Multilinear hash family with SMHasher, naive implementation 33 | 34 | 35 | //#include "PMP_Multilinear_test_naive.h" 36 | #include "PMP_Multilinear_naive.h" 37 | static PMP_Multilinear_Hasher_Naive pmpml_hasher_naive; 38 | void PMPML_32_CPP_naive( const void * key, int len, unsigned long seed, void * res ) 39 | { 40 | (void) seed; //unused 41 | *(uint32_t*)res = pmpml_hasher_naive.hash( (unsigned char*)key, len ); 42 | } 43 | 44 | #if !defined __arm__ 45 | 46 | #include "PMP_Multilinear_naive_64.h" 47 | 48 | static PMP_Multilinear_Hasher_64_naive pmpml_hasher_64_naive; 49 | 50 | void PMPML_64_CPP_naive( const void * key, int len, uint64_t seed, void * res ) 51 | { 52 | (void) seed; //unused 53 | *(uint64_t*)res = pmpml_hasher_64_naive.hash( (const unsigned char*)key, len ); 54 | } 55 | 56 | #endif // __arm__ 57 | 58 | -------------------------------------------------------------------------------- /PMP_Multilinear_test_naive.h: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------------------- 2 | * Copyright (c) 2014, Dmytro Ivanchykhin, Sergey Ignatchenko, Daniel Lemire 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, this 9 | * list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright notice, 11 | * this list of conditions and the following disclaimer in the documentation 12 | * and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 | * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | * ------------------------------------------------------------------------------- 25 | * 26 | * PMP+-Multilinear hash family implementation 27 | * 28 | * v.1.00 Apr-14-2014 Initial release 29 | * 30 | * -------------------------------------------------------------------------------*/ 31 | 32 | // PMP_Multilinear_test.h: interface of PMP+-Multilinear hash family with SMHasher 33 | 34 | #if ! defined __MULTILINEARPRIMESTRINGHASHFUNCTOR_TEST_NAIVE_H__ 35 | #define __MULTILINEARPRIMESTRINGHASHFUNCTOR_TEST_NAIVE_H__ 36 | 37 | 38 | //----------------------------------------------------------------------------- 39 | // Platform-specific functions and macros 40 | 41 | // Microsoft Visual Studio 42 | 43 | #if defined(_MSC_VER) 44 | 45 | typedef unsigned char uint8_t; 46 | typedef unsigned long uint32_t; 47 | typedef unsigned __int64 uint64_t; 48 | 49 | // Other compilers 50 | 51 | #else // defined(_MSC_VER) 52 | 53 | #include 54 | 55 | #endif // !defined(_MSC_VER) 56 | 57 | void PMPML_32_CPP_naive( const void * key, int len, uint32_t seed, void * res ); 58 | void PMPML_64_CPP_naive( const void * key, int len, uint64_t seed, void * res ); 59 | 60 | #endif // __MULTILINEARPRIMESTRINGHASHFUNCTOR_TEST_NAIVE_H__ 61 | -------------------------------------------------------------------------------- /PMP_Multilinear_test_speed_alt.h: -------------------------------------------------------------------------------- 1 | /* ------------------------------------------------------------------------------- 2 | * Copyright (c) 2014, Dmytro Ivanchykhin, Sergey Ignatchenko, Daniel Lemire 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, this 9 | * list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright notice, 11 | * this list of conditions and the following disclaimer in the documentation 12 | * and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 15 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 18 | * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 20 | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 21 | * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 23 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | * ------------------------------------------------------------------------------- 25 | * 26 | * PMP+-Multilinear hash family implementation 27 | * 28 | * v.1.00 Apr-14-2014 Initial release 29 | * 30 | * -------------------------------------------------------------------------------*/ 31 | 32 | // PMP_Multilinear_test.h: interface of PMP+-Multilinear hash family with SMHasher 33 | 34 | #if ! defined __MULTILINEARPRIMESTRINGHASHFUNCTOR_TEST_SPEED_ALT_H__ 35 | #define __MULTILINEARPRIMESTRINGHASHFUNCTOR_TEST_SPEED_ALT_H__ 36 | 37 | int PMPML_TestSpeedAlt_32( const void * key, int len, int iter ); 38 | int PMPML_TestSpeedAlt_64( const void * key, int len, int iter ); 39 | int PMPML_TestSpeedAlt_64_out_32( const void * key, int len, int iter ); 40 | 41 | #endif // __MULTILINEARPRIMESTRINGHASHFUNCTOR_TEST_SPEED_ALT_H__ 42 | -------------------------------------------------------------------------------- /PMurHash.h: -------------------------------------------------------------------------------- 1 | /*----------------------------------------------------------------------------- 2 | * MurmurHash3 was written by Austin Appleby, and is placed in the public 3 | * domain. 4 | * 5 | * This implementation was written by Shane Day, and is also public domain. 6 | * 7 | * This is a portable ANSI C implementation of MurmurHash3_x86_32 (Murmur3A) 8 | * with support for progressive processing. 9 | */ 10 | 11 | /* ------------------------------------------------------------------------- */ 12 | /* Determine what native type to use for uint32_t */ 13 | 14 | /* We can't use the name 'uint32_t' here because it will conflict with 15 | * any version provided by the system headers or application. */ 16 | 17 | /* If the compiler says it's C99 then take its word for it */ 18 | #if !defined(MH_UINT32) && ( \ 19 | defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L ) 20 | #include 21 | #define MH_UINT32 uint32_t 22 | #endif 23 | 24 | /* Otherwise try testing against max value macros from limit.h */ 25 | #if !defined(MH_UINT32) 26 | #include 27 | #if (USHRT_MAX == 0xffffffffUL) 28 | #define MH_UINT32 unsigned short 29 | #elif (UINT_MAX == 0xffffffffUL) 30 | #define MH_UINT32 unsigned int 31 | #elif (ULONG_MAX == 0xffffffffUL) 32 | #define MH_UINT32 unsigned long 33 | #endif 34 | #endif 35 | 36 | #if !defined(MH_UINT32) 37 | #error Unable to determine type name for unsigned 32-bit int 38 | #endif 39 | 40 | /* I'm yet to work on a platform where 'unsigned char' is not 8 bits */ 41 | #define MH_UINT8 unsigned char 42 | 43 | 44 | /* ------------------------------------------------------------------------- */ 45 | /* Prototypes */ 46 | 47 | #ifdef __cplusplus 48 | extern "C" { 49 | #endif 50 | 51 | void PMurHash32_Process(MH_UINT32 *ph1, MH_UINT32 *pcarry, const void *key, int len); 52 | MH_UINT32 PMurHash32_Result(MH_UINT32 h1, MH_UINT32 carry, MH_UINT32 total_length); 53 | MH_UINT32 PMurHash32(MH_UINT32 seed, const void *key, int len); 54 | 55 | void PMurHash32_test(const void *key, int len, MH_UINT32 seed, void *out); 56 | 57 | #ifdef __cplusplus 58 | } 59 | #endif 60 | -------------------------------------------------------------------------------- /Platform.cpp: -------------------------------------------------------------------------------- 1 | #include "Platform.h" 2 | 3 | #include 4 | 5 | void testRDTSC ( void ) 6 | { 7 | int64_t temp = rdtsc(); 8 | printf("%ld",(long)temp); 9 | } 10 | 11 | #if defined(_MSC_VER) 12 | 13 | #include 14 | 15 | void SetAffinity ( int cpu ) 16 | { 17 | SetProcessAffinityMask(GetCurrentProcess(),cpu); 18 | SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST); 19 | } 20 | 21 | #else 22 | 23 | #include 24 | 25 | void SetAffinity ( int /*cpu*/ ) 26 | { 27 | #if !defined(__CYGWIN__) && !defined(__APPLE__) && !defined(__FreeBSD__) 28 | cpu_set_t mask; 29 | 30 | CPU_ZERO(&mask); 31 | 32 | CPU_SET(2,&mask); 33 | 34 | if( sched_setaffinity(0,sizeof(mask),&mask) == -1) 35 | { 36 | printf("WARNING: Could not set CPU affinity\n"); 37 | } 38 | #endif 39 | } 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /Platform.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // Platform-specific functions and macros 3 | 4 | #pragma once 5 | 6 | void SetAffinity ( int cpu ); 7 | 8 | #ifndef __x86_64__ 9 | #if defined(__x86_64) || defined(_M_AMD64) || defined(_M_X64) 10 | #define __x86_64__ 11 | #endif 12 | #endif 13 | 14 | #ifndef HAVE_INT64 15 | #if (__WORDSIZE >= 64) || defined(HAVE_SSE42) 16 | #define HAVE_INT64 17 | #endif 18 | #endif 19 | 20 | //----------------------------------------------------------------------------- 21 | // Microsoft Visual Studio 22 | 23 | #if defined(_MSC_VER) 24 | 25 | #define FORCE_INLINE __forceinline 26 | #define NEVER_INLINE __declspec(noinline) 27 | 28 | #include 29 | #include // Has to be included before intrin.h or VC complains about 'ceil' 30 | #include // for __rdtsc 31 | #include 32 | 33 | #define ROTL32(x,y) _rotl(x,y) 34 | #define ROTL64(x,y) _rotl64(x,y) 35 | #define ROTR32(x,y) _rotr(x,y) 36 | #define ROTR64(x,y) _rotr64(x,y) 37 | 38 | #pragma warning(disable : 4127) // "conditional expression is constant" in the if()s for avalanchetest 39 | #pragma warning(disable : 4100) 40 | #pragma warning(disable : 4702) 41 | 42 | #pragma intrinsic(__rdtsc) 43 | // Read Time Stamp Counter 44 | #define rdtsc() __rdtsc() 45 | #define timer_start() __rdtsc() 46 | #define timer_end() __rdtsc() 47 | 48 | #define popcount8(x) __popcnt(x) 49 | 50 | //----------------------------------------------------------------------------- 51 | // Other compilers 52 | 53 | #else // !defined(_MSC_VER) 54 | 55 | #include 56 | #include 57 | 58 | #define FORCE_INLINE inline __attribute__((always_inline)) 59 | #define NEVER_INLINE __attribute__((noinline)) 60 | 61 | #ifdef HAVE_BIT32 62 | #define popcount8(x) __builtin_popcountll(x) 63 | #else 64 | #define popcount8(x) __builtin_popcountl(x) 65 | #endif 66 | 67 | inline uint32_t rotl32 ( uint32_t x, int8_t r ) 68 | { 69 | return (x << r) | (x >> (32 - r)); 70 | } 71 | 72 | inline uint64_t rotl64 ( uint64_t x, int8_t r ) 73 | { 74 | return (x << r) | (x >> (64 - r)); 75 | } 76 | 77 | inline uint32_t rotr32 ( uint32_t x, int8_t r ) 78 | { 79 | return (x >> r) | (x << (32 - r)); 80 | } 81 | 82 | inline uint64_t rotr64 ( uint64_t x, int8_t r ) 83 | { 84 | return (x >> r) | (x << (64 - r)); 85 | } 86 | 87 | #define ROTL32(x,y) rotl32(x,y) 88 | #define ROTL64(x,y) rotl64(x,y) 89 | #define ROTR32(x,y) rotr32(x,y) 90 | #define ROTR64(x,y) rotr64(x,y) 91 | 92 | __inline__ uint64_t rdtsc() 93 | { 94 | #ifdef _MSC_VER 95 | return __rdtsc(); 96 | #elif defined (__i386__) || defined (__x86_64__) 97 | return __builtin_ia32_rdtsc(); 98 | #else 99 | #define NO_CYCLE_COUNTER 100 | return 0; 101 | #endif 102 | } 103 | 104 | // see https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/ia-32-ia-64-benchmark-code-execution-paper.pdf 3.2.1 The Improved Benchmarking Method 105 | __inline__ uint64_t timer_start() 106 | { 107 | #if defined(__i386__) && !defined(HAVE_INT64) 108 | uint32_t cycles_high, cycles_low; 109 | __asm__ volatile 110 | ("cpuid\n\t" 111 | "rdtsc\n\t" 112 | "mov %%edx, %0\n\t" 113 | "mov %%eax, %1\n\t": "=r" (cycles_high), "=r" (cycles_low):: 114 | "%eax", "%ebx", "%ecx", "%edx"); 115 | return ((uint64_t)cycles_high << 32) | cycles_low; 116 | #elif defined __x86_64__ 117 | uint32_t cycles_high, cycles_low; 118 | __asm__ volatile 119 | ("cpuid\n\t" 120 | "rdtsc\n\t" 121 | "mov %%edx, %0\n\t" 122 | "mov %%eax, %1\n\t": "=r" (cycles_high), "=r" (cycles_low):: 123 | "%rax", "%rbx", "%rcx", "%rdx"); 124 | return ((uint64_t)cycles_high << 32) | cycles_low; 125 | #else 126 | #define NO_CYCLE_COUNTER 127 | return 0; 128 | #endif 129 | } 130 | 131 | __inline__ uint64_t timer_end() 132 | { 133 | #if defined(__i386__) && !defined(HAVE_INT64) 134 | uint32_t cycles_high, cycles_low; 135 | __asm__ volatile 136 | ("rdtscp\n\t" 137 | "mov %%edx, %0\n\t" 138 | "mov %%eax, %1\n\t" 139 | "cpuid\n\t": "=r" (cycles_high), "=r" (cycles_low):: 140 | "%eax", "%ebx", "%ecx", "%edx"); 141 | return ((uint64_t)cycles_high << 32) | cycles_low; 142 | #elif defined __x86_64__ 143 | uint32_t cycles_high, cycles_low; 144 | __asm__ volatile 145 | ("rdtscp\n\t" 146 | "mov %%edx, %0\n\t" 147 | "mov %%eax, %1\n\t" 148 | "cpuid\n\t": "=r" (cycles_high), "=r" (cycles_low):: 149 | "%rax", "%rbx", "%rcx", "%rdx"); 150 | return ((uint64_t)cycles_high << 32) | cycles_low; 151 | #else 152 | #define NO_CYCLE_COUNTER 153 | return 0; 154 | #endif 155 | } 156 | 157 | 158 | #include 159 | #define _stricmp strcasecmp 160 | 161 | #endif // !defined(_MSC_VER) 162 | 163 | //----------------------------------------------------------------------------- 164 | 165 | #ifndef __WORDSIZE 166 | # ifdef HAVE_BIT32 167 | # define __WORDSIZE 32 168 | # else 169 | # define __WORDSIZE 64 170 | # endif 171 | #endif 172 | -------------------------------------------------------------------------------- /Random.cpp: -------------------------------------------------------------------------------- 1 | #include "Random.h" 2 | 3 | Rand g_rand1(1); 4 | Rand g_rand2(2); 5 | Rand g_rand3(3); 6 | Rand g_rand4(4); 7 | 8 | //----------------------------------------------------------------------------- 9 | -------------------------------------------------------------------------------- /Random.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Types.h" 4 | 5 | //----------------------------------------------------------------------------- 6 | // Xorshift RNG based on code by George Marsaglia 7 | // http://en.wikipedia.org/wiki/Xorshift 8 | 9 | struct Rand 10 | { 11 | uint32_t x; 12 | uint32_t y; 13 | uint32_t z; 14 | uint32_t w; 15 | 16 | Rand() 17 | { 18 | reseed(uint32_t(0)); 19 | } 20 | 21 | Rand( uint32_t seed ) 22 | { 23 | reseed(seed); 24 | } 25 | 26 | void reseed ( uint32_t seed ) 27 | { 28 | x = 0x498b3bc5 ^ seed; 29 | y = 0; 30 | z = 0; 31 | w = 0; 32 | 33 | for(int i = 0; i < 10; i++) mix(); 34 | } 35 | 36 | void reseed ( uint64_t seed ) 37 | { 38 | x = 0x498b3bc5 ^ (uint32_t)(seed >> 0); 39 | y = 0x5a05089a ^ (uint32_t)(seed >> 32); 40 | z = 0; 41 | w = 0; 42 | 43 | for(int i = 0; i < 10; i++) mix(); 44 | } 45 | 46 | //----------------------------------------------------------------------------- 47 | 48 | void mix ( void ) 49 | { 50 | uint32_t t = x ^ (x << 11); 51 | x = y; y = z; z = w; 52 | w = w ^ (w >> 19) ^ t ^ (t >> 8); 53 | } 54 | 55 | uint32_t rand_u32 ( void ) 56 | { 57 | mix(); 58 | 59 | return x; 60 | } 61 | 62 | uint64_t rand_u64 ( void ) 63 | { 64 | mix(); 65 | 66 | uint64_t a = x; 67 | uint64_t b = y; 68 | 69 | return (a << 32) | b; 70 | } 71 | 72 | void rand_p ( void * blob, int bytes ) 73 | { 74 | uint32_t * blocks = reinterpret_cast(blob); 75 | 76 | while(bytes >= 4) 77 | { 78 | blocks[0] = rand_u32(); 79 | blocks++; 80 | bytes -= 4; 81 | } 82 | 83 | uint8_t * tail = reinterpret_cast(blocks); 84 | 85 | for(int i = 0; i < bytes; i++) 86 | { 87 | tail[i] = (uint8_t)rand_u32(); 88 | } 89 | } 90 | }; 91 | 92 | //----------------------------------------------------------------------------- 93 | 94 | extern Rand g_rand1; 95 | 96 | inline uint32_t rand_u32 ( void ) { return g_rand1.rand_u32(); } 97 | inline uint64_t rand_u64 ( void ) { return g_rand1.rand_u64(); } 98 | 99 | inline void rand_p ( void * blob, int bytes ) 100 | { 101 | uint32_t * blocks = (uint32_t*)blob; 102 | 103 | while(bytes >= 4) 104 | { 105 | *blocks++ = rand_u32(); 106 | bytes -= 4; 107 | } 108 | 109 | uint8_t * tail = (uint8_t*)blocks; 110 | 111 | for(int i = 0; i < bytes; i++) 112 | { 113 | tail[i] = (uint8_t)rand_u32(); 114 | } 115 | } 116 | 117 | //----------------------------------------------------------------------------- 118 | -------------------------------------------------------------------------------- /SpeedTest.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Types.h" 4 | 5 | void BulkSpeedTest ( pfHash hash, uint32_t seed ); 6 | double TinySpeedTest ( pfHash hash, int hashsize, int keysize, uint32_t seed, bool verbose ); 7 | double HashMapSpeedTest ( pfHash pfhash, int hashbits, std::vector words, 8 | const int trials, bool verbose ); 9 | //----------------------------------------------------------------------------- 10 | -------------------------------------------------------------------------------- /SpookyTest.cpp: -------------------------------------------------------------------------------- 1 | #include "Spooky.h" 2 | 3 | void SpookyHash32_test(const void *key, int len, uint32_t seed, void *out) { 4 | *(uint32_t*)out = SpookyHash::Hash32(key, len, seed); 5 | } 6 | 7 | void SpookyHash64_test(const void *key, int len, uint32_t seed, void *out) { 8 | *(uint64_t*)out = SpookyHash::Hash64(key, len, seed); 9 | } 10 | 11 | void SpookyHash128_test(const void *key, int len, uint32_t seed, void *out) { 12 | uint64_t h1 = seed, h2 = seed; 13 | SpookyHash::Hash128(key, len, &h1, &h2); 14 | ((uint64_t*)out)[0] = h1; 15 | ((uint64_t*)out)[1] = h2; 16 | } 17 | -------------------------------------------------------------------------------- /Stats.cpp: -------------------------------------------------------------------------------- 1 | #include "Stats.h" 2 | 3 | //----------------------------------------------------------------------------- 4 | 5 | double chooseK ( int n, int k ) 6 | { 7 | if(k > (n - k)) k = n - k; 8 | 9 | double c = 1; 10 | 11 | for(int i = 0; i < k; i++) 12 | { 13 | double t = double(n-i) / double(i+1); 14 | 15 | c *= t; 16 | } 17 | 18 | return c; 19 | } 20 | 21 | double chooseUpToK ( int n, int k ) 22 | { 23 | double c = 0; 24 | 25 | for(int i = 1; i <= k; i++) 26 | { 27 | c += chooseK(n,i); 28 | } 29 | 30 | return c; 31 | } 32 | 33 | //----------------------------------------------------------------------------- 34 | // Distribution "score" 35 | // TODO - big writeup of what this score means 36 | 37 | // Basically, we're computing a constant that says "The test distribution is as 38 | // uniform, RMS-wise, as a random distribution restricted to (1-X)*100 percent of 39 | // the bins. This makes for a nice uniform way to rate a distribution that isn't 40 | // dependent on the number of bins or the number of keys 41 | 42 | // (as long as # keys > # bins * 3 or so, otherwise random fluctuations show up 43 | // as distribution weaknesses) 44 | 45 | double calcScore ( const int * bins, const int bincount, const int keycount ) 46 | { 47 | double n = bincount; 48 | double k = keycount; 49 | 50 | // compute rms value 51 | 52 | double r = 0; 53 | 54 | for(int i = 0; i < bincount; i++) 55 | { 56 | double b = bins[i]; 57 | 58 | r += b*b; 59 | } 60 | 61 | r = sqrt(r / n); 62 | 63 | // compute fill factor 64 | 65 | double f = (k*k - 1) / (n*r*r - k); 66 | 67 | // rescale to (0,1) with 0 = good, 1 = bad 68 | 69 | return 1 - (f / n); 70 | } 71 | 72 | 73 | //---------------------------------------------------------------------------- 74 | 75 | void plot ( double n ) 76 | { 77 | double n2 = n * 1; 78 | 79 | if(n2 < 0) n2 = 0; 80 | 81 | n2 *= 100; 82 | 83 | if(n2 > 64) n2 = 64; 84 | 85 | int n3 = (int)n2; 86 | 87 | if(n3 == 0) 88 | printf("."); 89 | else 90 | { 91 | char x = '0' + char(n3); 92 | 93 | if(x > '9') x = 'X'; 94 | 95 | printf("%c",x); 96 | } 97 | } 98 | 99 | //----------------------------------------------------------------------------- 100 | -------------------------------------------------------------------------------- /SuperFastHash.cpp: -------------------------------------------------------------------------------- 1 | #include "Platform.h" 2 | 3 | /* By Paul Hsieh (C) 2004, 2005. Covered under the Paul Hsieh derivative 4 | license. See: 5 | http://www.azillionmonkeys.com/qed/weblicense.html for license details. 6 | 7 | http://www.azillionmonkeys.com/qed/hash.html */ 8 | 9 | /* 10 | #undef get16bits 11 | #if (defined(__GNUC__) && defined(__i386__)) || defined(__WATCOMC__) \ 12 | || defined(_MSC_VER) || defined (__BORLANDC__) || defined (__TURBOC__) 13 | #define get16bits(d) (*((const uint16_t *) (d))) 14 | #endif 15 | 16 | #if !defined (get16bits) 17 | #define get16bits(d) ((((uint32_t)(((const uint8_t *)(d))[1])) << 8)\ 18 | +(uint32_t)(((const uint8_t *)(d))[0]) ) 19 | #endif 20 | */ 21 | 22 | FORCE_INLINE uint16_t get16bits ( const void * p ) 23 | { 24 | return *(const uint16_t*)p; 25 | } 26 | 27 | // objsize: 0-d2: 210 28 | uint32_t SuperFastHash (const char * data, int len, int32_t hash) { 29 | uint32_t tmp; 30 | int rem; 31 | 32 | if (len <= 0 || !data) return 0; 33 | 34 | rem = len & 3; 35 | len >>= 2; 36 | 37 | /* Main loop */ 38 | for (;len > 0; len--) { 39 | hash += get16bits (data); 40 | tmp = (get16bits (data+2) << 11) ^ hash; 41 | hash = (hash << 16) ^ tmp; 42 | data += 2*sizeof (uint16_t); 43 | hash += hash >> 11; 44 | } 45 | 46 | /* Handle end cases */ 47 | switch (rem) { 48 | case 3: hash += get16bits (data); 49 | hash ^= hash << 16; 50 | hash ^= data[sizeof (uint16_t)] << 18; 51 | hash += hash >> 11; 52 | break; 53 | case 2: hash += get16bits (data); 54 | hash ^= hash << 11; 55 | hash += hash >> 17; 56 | break; 57 | case 1: hash += *data; 58 | hash ^= hash << 10; 59 | hash += hash >> 1; 60 | } 61 | 62 | /* Force "avalanching" of final 127 bits */ 63 | hash ^= hash << 3; 64 | hash += hash >> 5; 65 | hash ^= hash << 4; 66 | hash += hash >> 17; 67 | hash ^= hash << 25; 68 | hash += hash >> 6; 69 | 70 | return hash; 71 | } 72 | -------------------------------------------------------------------------------- /Types.cpp: -------------------------------------------------------------------------------- 1 | #include "Types.h" 2 | #include "Random.h" 3 | 4 | #include 5 | 6 | uint32_t MurmurOAAT ( const char * blob, int len, uint32_t seed ); 7 | 8 | //----------------------------------------------------------------------------- 9 | 10 | #if defined(_MSC_VER) 11 | #pragma optimize( "", off ) 12 | #endif 13 | 14 | void blackhole ( uint32_t ) 15 | { 16 | } 17 | 18 | uint32_t whitehole ( void ) 19 | { 20 | return 0; 21 | } 22 | 23 | #if defined(_MSC_VER) 24 | #pragma optimize( "", on ) 25 | #endif 26 | 27 | uint32_t g_verify = 1; 28 | 29 | void MixVCode ( const void * blob, int len ) 30 | { 31 | g_verify = MurmurOAAT((const char*)blob,len,g_verify); 32 | } 33 | 34 | //----------------------------------------------------------------------------- 35 | // unused 36 | bool isprime ( uint32_t x ) 37 | { 38 | uint32_t p[] = 39 | { 40 | 2,3,5,7,11,13,17,19,23,29,31,37,41,43,47,53,59,61,67,71,73,79,83,89,97,101, 41 | 103,107,109,113,127,131,137,139,149,151,157,163,167,173,179,181,191,193,197, 42 | 199,211,223,227,229,233,239,241,251 43 | }; 44 | 45 | for(size_t i=0; i < sizeof(p)/sizeof(uint32_t); i++) 46 | { 47 | if((x % p[i]) == 0) 48 | { 49 | return false; 50 | } 51 | } 52 | 53 | for(int i = 257; i < 65536; i += 2) 54 | { 55 | if((x % i) == 0) 56 | { 57 | return false; 58 | } 59 | } 60 | 61 | return true; 62 | } 63 | 64 | // unused 65 | void GenerateMixingConstants ( void ) 66 | { 67 | Rand r(8350147); 68 | 69 | int count = 0; 70 | 71 | int trials = 0; 72 | int bitfail = 0; 73 | int popfail = 0; 74 | int matchfail = 0; 75 | int primefail = 0; 76 | 77 | //for(uint32_t x = 1; x; x++) 78 | while(count < 100) 79 | { 80 | //if(x % 100000000 == 0) printf("."); 81 | 82 | trials++; 83 | uint32_t b = r.rand_u32(); 84 | //uint32_t b = x; 85 | 86 | //---------- 87 | // must have between 14 and 18 set bits 88 | 89 | if(popcount(b) < 16) { b = 0; popfail++; } 90 | if(popcount(b) > 16) { b = 0; popfail++; } 91 | 92 | if(b == 0) continue; 93 | 94 | //---------- 95 | // must have 3-5 bits set per 8-bit window 96 | 97 | for(int i = 0; i < 32; i++) 98 | { 99 | uint32_t c = ROTL32(b,i) & 0xFF; 100 | 101 | if(popcount(c) < 3) { b = 0; bitfail++; break; } 102 | if(popcount(c) > 5) { b = 0; bitfail++; break; } 103 | } 104 | 105 | if(b == 0) continue; 106 | 107 | //---------- 108 | // all 8-bit windows must be different 109 | 110 | uint8_t match[256]; 111 | 112 | memset(match,0,256); 113 | 114 | for(int i = 0; i < 32; i++) 115 | { 116 | uint32_t c = ROTL32(b,i) & 0xFF; 117 | 118 | if(match[c]) { b = 0; matchfail++; break; } 119 | 120 | match[c] = 1; 121 | } 122 | 123 | if(b == 0) continue; 124 | 125 | //---------- 126 | // must be prime 127 | 128 | if(!isprime(b)) 129 | { 130 | b = 0; 131 | primefail++; 132 | } 133 | 134 | if(b == 0) continue; 135 | 136 | //---------- 137 | 138 | if(b) 139 | { 140 | printf("0x%08x : 0x%08x\n",b,~b); 141 | count++; 142 | } 143 | } 144 | 145 | printf("%d %d %d %d %d %d\n",trials,popfail,bitfail,matchfail,primefail,count); 146 | } 147 | 148 | //----------------------------------------------------------------------------- 149 | -------------------------------------------------------------------------------- /beamsplitter.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Cris Stringfellow 2 | // Licensed under GPL-3.0 3 | // https://github.com/cris691/beamsplitter 4 | #include 5 | #include 6 | #include "beamsplitter.h" 7 | 8 | #if defined(_MSC_VER) 9 | 10 | #define FORCE_INLINE __forceinline 11 | 12 | // Other compilers 13 | 14 | #else // defined(_MSC_VER) 15 | 16 | #define FORCE_INLINE inline __attribute__((always_inline)) 17 | 18 | #endif // !defined(_MSC_VER) 19 | 20 | const int STATE = 32; 21 | uint8_t buf[STATE] = {0}; 22 | uint64_t MASK = 0xffffffffffffff; 23 | uint8_t *state8 = (uint8_t *)buf; 24 | uint64_t *state = (uint64_t *)buf; 25 | 26 | //-------- 27 | // State mix function 28 | 29 | FORCE_INLINE uint64_t rot( uint64_t v, int n) 30 | { 31 | n = n & 63U; 32 | if (n) 33 | v = (v >> n) | (v << (64-n)); 34 | return v; 35 | } 36 | 37 | FORCE_INLINE uint8_t rot8( uint8_t v, int n) 38 | { 39 | n = n & 7U; 40 | if (n) 41 | v = (v >> n) | (v << (8-n)); 42 | return v; 43 | } 44 | 45 | FORCE_INLINE void mix(const int A) 46 | { 47 | const int B = A+1; 48 | const int iv = state[A] & 1023; 49 | const uint64_t M = T[iv]; 50 | state[B] += M + state[A]; 51 | 52 | state[A] ^= state[B]; 53 | state[B] ^= state[A]; 54 | state[A] ^= state[B]; 55 | 56 | state[B] = rot(state[B], state[A]); 57 | } 58 | 59 | //--------- 60 | // Hash round function 61 | 62 | FORCE_INLINE void round( const uint64_t * m64, const uint8_t * m8, int len ) 63 | { 64 | int index = 0; 65 | int sindex = 0; 66 | 67 | for( int Len = len >> 3; index < Len; index++) { 68 | state[sindex] += rot(m64[index] + index + 1, state[sindex] +index +1); 69 | if ( sindex == 1 ) { 70 | mix(0); 71 | } else if ( sindex == 3 ) { 72 | mix(2); 73 | sindex = -1; 74 | } 75 | sindex++; 76 | } 77 | 78 | mix(0); 79 | 80 | index <<= 3; 81 | sindex = index&31; 82 | for( ; index < len; index++) { 83 | state8[sindex] += rot8(m8[index] + index + 1, state8[sindex] + index+1); 84 | // state+[0,1,2] 85 | mix(index%3); 86 | if ( sindex >= 31 ) { 87 | sindex = -1; 88 | } 89 | sindex++; 90 | } 91 | 92 | mix(0); 93 | mix(1); 94 | mix(2); 95 | } 96 | 97 | //--------- 98 | // main hash function 99 | 100 | void beamsplitter_64 ( const void * key, int len, unsigned seed, void * out ) 101 | { 102 | const uint8_t *key8Arr = (uint8_t *)key; 103 | const uint64_t *key64Arr = (uint64_t *)key; 104 | 105 | uint8_t seedbuf[8] = {0}; 106 | uint8_t *seed8Arr = (uint8_t *)seedbuf; 107 | uint64_t *seed64Arr = (uint64_t *)seedbuf; 108 | uint32_t *seed32Arr = (uint32_t *)seedbuf; 109 | 110 | // the cali number from the Matrix (1999) 111 | seed32Arr[0] = 0xc5550690; 112 | seed32Arr[0] -= seed; 113 | seed32Arr[1] = ~(1 - seed); 114 | 115 | // nothing up my sleeve 116 | state[0] = 0x123456789abcdef0; 117 | state[1] = 0x0fedcba987654321; 118 | state[2] = 0xaccadacca80081e5; 119 | state[3] = 0xf00baaf00f00baaa; 120 | 121 | round( key64Arr, key8Arr, len ); 122 | round( key64Arr, key8Arr, len ); 123 | round( key64Arr, key8Arr, len ); 124 | round( seed64Arr, seed8Arr, 8 ); 125 | //round( state, state8, STATE ); 126 | round( seed64Arr, seed8Arr, 8 ); 127 | round( key64Arr, key8Arr, len ); 128 | round( key64Arr, key8Arr, len ); 129 | round( key64Arr, key8Arr, len ); 130 | 131 | /* 132 | //printf("state = %#018" PRIx64 " %#018" PRIx64 " %#018" PRIx64 " %#018" PRIx64 "\n", 133 | // state[0], state[1], state[2], state[3] ); 134 | */ 135 | 136 | //printf("state = %#018" PRIx64 " %#018" PRIx64 "\n", 137 | // state[0], state[1] ); 138 | 139 | uint8_t output[STATE] = {0}; 140 | uint64_t *h = (uint64_t *)output; 141 | 142 | // The new combination step 143 | h[0] = state[2]; 144 | h[1] = state[3]; 145 | 146 | h[0] += h[1]; 147 | 148 | ((uint64_t *)out)[0] = h[0]; 149 | } 150 | -------------------------------------------------------------------------------- /blake3/README.md: -------------------------------------------------------------------------------- 1 | # BLAKE3-c [![Actions Status](https://github.com/veorq/BLAKE3-c/workflows/tests/badge.svg)](https://github.com/veorq/BLAKE3-c/actions) 2 | 3 | A very rough initial implementation of BLAKE3 in C. SSE4.1, AVX2, 4 | AVX-512, and NEON are supported, using compile-time feature selection in 5 | the Makefile. 6 | 7 | This implementation is simpler than the [Rust 8 | implementation](https://github.com/veorq/BLAKE3). It doesn't support 9 | multithreading, and it doesn't parallelize parent hashes, so throughput 10 | is lower. 11 | 12 | TODO: 13 | - CI testing for AVX-512 and NEON. 14 | - Cross-platform build, e.g. Windows. 15 | - Dynamic CPU feature detection, at least for x86. 16 | 17 | Example usage: 18 | 19 | ```bash 20 | $ make avx2 21 | $ head -c 1000000 /dev/urandom | ./blake3 22 | 43f2cae3cfd7678bc3a3ebdbf170608d19d5ebaad23e9d06291dba3269853608 23 | $ head -c 1000000 /dev/urandom | ./blake3 --length 50 24 | 4fc0ee74a60aa77fb699821997498fd93f1a98bd03eaf2a7969c4b35fb742c233a7a161fd2a431605f6e92dcf4cd7d052102 25 | $ head -c 1000000 /dev/urandom | ./blake3 --keyed 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef 26 | 8aee87b232fe90b042bf9119591e24409763a268139ff157d20021003e314064 27 | 28 | ``` 29 | -------------------------------------------------------------------------------- /blake3/blake3.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #define BLAKE3_KEY_LEN 32 6 | #define BLAKE3_OUT_LEN 32 7 | #define BLAKE3_BLOCK_LEN 64 8 | #define BLAKE3_CHUNK_LEN 1024 9 | #define BLAKE3_MAX_DEPTH 54 10 | #define BLAKE3_MAX_SIMD_DEGREE 16 11 | 12 | typedef struct { 13 | uint32_t cv[8]; 14 | uint64_t chunk_counter; 15 | uint8_t buf[BLAKE3_BLOCK_LEN]; 16 | uint8_t buf_len; 17 | uint8_t blocks_compressed; 18 | uint8_t flags; 19 | } blake3_chunk_state; 20 | 21 | typedef struct { 22 | uint32_t key[8]; 23 | blake3_chunk_state chunk; 24 | uint8_t cv_stack_len; 25 | uint8_t cv_stack[BLAKE3_MAX_DEPTH * BLAKE3_OUT_LEN]; 26 | } blake3_hasher; 27 | 28 | void blake3_hasher_init(blake3_hasher *self); 29 | void blake3_hasher_init_keyed(blake3_hasher *self, 30 | const uint8_t key[BLAKE3_KEY_LEN]); 31 | void blake3_hasher_init_derive_key(blake3_hasher *self, const char *context); 32 | void blake3_hasher_update(blake3_hasher *self, const void *input, 33 | size_t input_len); 34 | void blake3_hasher_finalize(const blake3_hasher *self, uint8_t *out, 35 | size_t out_len); 36 | -------------------------------------------------------------------------------- /blake3/blake3_impl.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #if __POPCNT__ 10 | #include 11 | #endif 12 | 13 | #include "blake3.h" 14 | 15 | // internal flags 16 | #define CHUNK_START 1 17 | #define CHUNK_END 2 18 | #define PARENT 4 19 | #define ROOT 8 20 | #define KEYED_HASH 16 21 | #define DERIVE_KEY_CONTEXT 32 22 | #define DERIVE_KEY_MATERIAL 64 23 | 24 | // This C implementation tries to support recent versions of GCC, Clang, and 25 | // MSVC. 26 | #if defined(_MSC_VER) 27 | #define INLINE __forceinline static 28 | #else 29 | #define INLINE __attribute__((always_inline)) static inline 30 | #endif 31 | 32 | static const uint32_t IV[8] = {0x6A09E667, 0xBB67AE85, 0x3C6EF372, 33 | 0xA54FF53A, 0x510E527F, 0x9B05688C, 34 | 0x1F83D9AB, 0x5BE0CD19}; 35 | 36 | static const uint8_t MSG_SCHEDULE[7][16] = { 37 | {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, 38 | {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8}, 39 | {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1}, 40 | {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6}, 41 | {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4}, 42 | {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7}, 43 | {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13}, 44 | }; 45 | 46 | // Count the number of 1 bits. 47 | INLINE uint8_t popcnt(uint64_t x) { 48 | #if __POPCNT__ 49 | # ifdef __x86_64__ 50 | return (uint8_t)_mm_popcnt_u64(x); 51 | #else 52 | return (uint8_t)(_mm_popcnt_u32(x >> 32) + _mm_popcnt_u32(x && 0xFFFFFFFF)); 53 | #endif 54 | #else 55 | uint8_t count = 0; 56 | while (x > 0) { 57 | count += ((uint8_t)x) & 1; 58 | x >>= 1; 59 | } 60 | return count; 61 | #endif 62 | } 63 | 64 | INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; } 65 | 66 | INLINE uint32_t counter_high(uint64_t counter) { 67 | return (uint32_t)(counter >> 32); 68 | } 69 | 70 | INLINE uint32_t load32(const void *src) { 71 | const uint8_t *p = (const uint8_t *)src; 72 | return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) | 73 | ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24); 74 | } 75 | 76 | INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], 77 | uint32_t key_words[8]) { 78 | key_words[0] = load32(&key[0 * 4]); 79 | key_words[1] = load32(&key[1 * 4]); 80 | key_words[2] = load32(&key[2 * 4]); 81 | key_words[3] = load32(&key[3 * 4]); 82 | key_words[4] = load32(&key[4 * 4]); 83 | key_words[5] = load32(&key[5 * 4]); 84 | key_words[6] = load32(&key[6 * 4]); 85 | key_words[7] = load32(&key[7 * 4]); 86 | } 87 | 88 | void blake3_compress_in_place(uint32_t cv[8], 89 | const uint8_t block[BLAKE3_BLOCK_LEN], 90 | uint8_t block_len, uint64_t counter, 91 | uint8_t flags); 92 | 93 | void blake3_compress_xof(const uint32_t cv[8], 94 | const uint8_t block[BLAKE3_BLOCK_LEN], 95 | uint8_t block_len, uint64_t counter, uint8_t flags, 96 | uint8_t out[64]); 97 | 98 | void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs, 99 | size_t blocks, const uint32_t key[8], uint64_t counter, 100 | bool increment_counter, uint8_t flags, 101 | uint8_t flags_start, uint8_t flags_end, uint8_t *out); 102 | -------------------------------------------------------------------------------- /build-msys2.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | cd %APPVEYOR_BUILD_FOLDER% 4 | set MSYS2_DIR=msys64 5 | 6 | echo ARCH: %ARCH% 7 | echo Compiler: %COMPILER% 8 | echo MSYS2_ARCH: %MSYS2_ARCH% 9 | echo Platform: %PLATFORM% 10 | echo MSYS2 directory: %MSYS2_DIR% 11 | echo MSYS2 system: %MSYSTEM% 12 | 13 | echo Installation MSYS2 build preconditions... 14 | 15 | echo Extending path to MSYS... 16 | SET "PATH=C:\%MSYS2_DIR%\%MSYSTEM%\bin;C:\%MSYS2_DIR%\usr\bin;%PATH%" 17 | 18 | echo Updating pacman... 19 | bash -lc "pacman -S --needed --noconfirm pacman-mirrors" 20 | bash -lc "pacman -Syyu --noconfirm" 21 | 22 | echo Installing dependencies... 23 | rem see https://github.com/Alexpux/MSYS2-packages 24 | bash -lc "pacman -S --needed --noconfirm mingw-w64-%MSYS2_ARCH%-toolchain" 25 | 26 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | rm -rf build 3 | mkdir build 4 | cd build 5 | # Note that gcc-mp-5 on darwin is still ~2000 MiB/sec faster than the system clang 6 | # clang-mp-dev (4.0) also 7 | case `uname -s` in 8 | #Darwin) CXX=clang++-mp-devel CC=clang-mp-devel cmake .. ;; 9 | #Darwin) CXX=g++-mp-6 CC=gcc-mp-6 cmake .. ;; 10 | Darwin) cmake .. ;; 11 | FreeBSD) CXX=clang++ CC=clang cmake .. ;; 12 | Linux) CXX=g++ CC=gcc cmake .. ;; 13 | *) CXX=clang++-4.0 CC=clang-4.0 cmake .. ;; 14 | esac 15 | make -j4 $@ 16 | cp SMHasher .. 17 | cd .. 18 | -------------------------------------------------------------------------------- /build32.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | rm -rf build32 3 | mkdir build32 4 | cd build32 5 | case `uname -s` in 6 | Darwin) CXX="/usr/bin/c++ -m32" CC="/usr/bin/cc -m32" cmake .. $@ ;; 7 | *) CXX="g++ -m32" CC="gcc -m32" cmake .. $@ ;; 8 | esac 9 | make 10 | cd .. 11 | -------------------------------------------------------------------------------- /checkdoctests.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use File::Copy 'mv'; 4 | my $endtest = qr(^(?:---|\[\[\[ |Input vcode 0x)); 5 | # mkdir partests; build/SMHasher --list|perl -alne'print $F[0] | parallel -j4 --bar 'build/SMHasher --test=Sparse,Permutation,Cyclic,TwoBytes,DiffDist,Text,Zeroes,Seed,Sanity,Avalanche,BIC,LongNeighbors,Diff,MomentChi2 {} >partests/{}' 6 | my @keysettests = qw(Sparse Permutation Cyclic TwoBytes Window Text Zeroes Seed); 7 | my @othertests = qw(Sanity Speed Hashmap Avalanche Diff DiffDist BIC LongNeighbors MomentChi2); 8 | my %tests = map {$_ => 1} @keysettests, @othertests; 9 | my $testrx = '(' . join('|',@othertests) . ')'; 10 | $testrx = qr($testrx); 11 | delete $tests{LongNeighbors}; # invalid license 12 | 13 | if (@ARGV) { 14 | readf($_) for @ARGV; 15 | } else { 16 | readf($_) for ; 17 | } 18 | 19 | sub readf { 20 | my $fn = shift; 21 | my ($n,%r); 22 | open(my $l, "<", $fn) or die "open $fn $!"; 23 | while (<$l>) { 24 | if (/^--- Testing ([\w_-]+) \"/) { 25 | #if ($n) { # fixup previous name 26 | # check($n,\%r,$fn) if %r; 27 | #} 28 | $n = $1; 29 | } 30 | NEXT: 31 | if (/^\[\[\[ Keyset '(.+?)' Tests/) { 32 | my $t = $1; 33 | while (<$l>) { 34 | goto NEXT if $_ =~ $endtest; 35 | $r{$t} .= $_; 36 | } 37 | } 38 | if (/^\[\[\[ '?$testrx'? /) { 39 | my $t = $1; 40 | while (<$l>) { 41 | goto NEXT if $_ =~ $endtest; 42 | $r{$t} .= $_; 43 | } 44 | } 45 | } 46 | check($n,\%r,$fn) if $n; # fixup last name 47 | } 48 | 49 | sub check { 50 | my ($n,$r,$fn) = @_; 51 | return unless $n; 52 | my @keys = sort keys %$r; 53 | my @tests = sort keys %tests; 54 | my @t; 55 | if (@keys != @tests) { 56 | print "# missing tests in $fn for $n:\n"; 57 | print "SMHasher --test="; 58 | for (@tests) { 59 | push @t, $_ unless $r->{$_}; 60 | } 61 | print join(',',@t)," $n >>doc/$n\n"; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /clhash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * CLHash is a very fast hashing function that uses the 3 | * carry-less multiplication and SSE instructions. 4 | * 5 | * Daniel Lemire, Owen Kaser, Faster 64-bit universal hashing 6 | * using carry-less multiplications, Journal of Cryptographic Engineering (to appear) 7 | * 8 | * Best used on recent x64 processors (Haswell or better). 9 | * 10 | * Compile option: if you define BITMIX during compilation, extra work is done to 11 | * pass smhasher's avalanche test succesfully. Disabled by default. 12 | **/ 13 | 14 | #ifndef INCLUDE_CLHASH_H_ 15 | #define INCLUDE_CLHASH_H_ 16 | 17 | 18 | #include 19 | #include // life is short, please use a C99-compliant compiler 20 | #include 21 | 22 | #ifdef __cplusplus 23 | extern "C" { 24 | #endif 25 | 26 | // BITMIX necessary to pass the Avalanche Tests 27 | #define BITMIX 28 | 29 | enum {RANDOM_64BITWORDS_NEEDED_FOR_CLHASH=133,RANDOM_BYTES_NEEDED_FOR_CLHASH=133*8}; 30 | 31 | /** 32 | * random : the random data source (should contain at least 33 | * RANDOM_BYTES_NEEDED_FOR_CLHASH random bytes), it should 34 | * also be aligned on 16-byte boundaries so that (((uintptr_t) random & 15) == 0) 35 | * for performance reasons. This is usually generated once and reused with many 36 | * inputs. 37 | * 38 | * 39 | * stringbyte : the input data source, could be anything you want to has 40 | * 41 | * 42 | * length : number of bytes in the string 43 | */ 44 | uint64_t clhash(const void* random, const char * stringbyte, 45 | const size_t lengthbyte); 46 | 47 | 48 | 49 | /** 50 | * Convenience method. Will generate a random key from two 64-bit seeds. 51 | * Caller is responsible to call "free" on the result. 52 | */ 53 | void * get_random_key_for_clhash(uint64_t seed1, uint64_t seed2); 54 | 55 | #ifdef __cplusplus 56 | } // extern "C" 57 | #endif 58 | 59 | #ifdef __cplusplus 60 | #include 61 | #include 62 | #include // For std::strlen 63 | 64 | struct clhasher { 65 | const void *random_data_; 66 | clhasher(uint64_t seed1=137, uint64_t seed2=777): random_data_(get_random_key_for_clhash(seed1, seed2)) {} 67 | template 68 | uint64_t operator()(const T *data, const size_t len) const { 69 | return clhash(random_data_, (const char *)data, len * sizeof(T)); 70 | } 71 | uint64_t operator()(const char *str) const {return operator()(str, std::strlen(str));} 72 | template 73 | uint64_t operator()(const T &input) const { 74 | return operator()((const char *)&input, sizeof(T)); 75 | } 76 | template 77 | uint64_t operator()(const std::vector &input) const { 78 | return operator()((const char *)input.data(), sizeof(T) * input.size()); 79 | } 80 | uint64_t operator()(const std::string &str) const { 81 | return operator()(str.data(), str.size()); 82 | } 83 | ~clhasher() { 84 | free((void *)random_data_); 85 | } 86 | }; 87 | #endif // #ifdef __cplusplus 88 | 89 | #endif /* INCLUDE_CLHASH_H_ */ 90 | -------------------------------------------------------------------------------- /cmetrohash.h: -------------------------------------------------------------------------------- 1 | // metrohash.h 2 | // 3 | // The MIT License (MIT) 4 | // 5 | // Copyright (c) 2015 J. Andrew Rogers 6 | // 7 | // Permission is hereby granted, free of charge, to any person obtaining a copy 8 | // of this software and associated documentation files (the "Software"), to deal 9 | // in the Software without restriction, including without limitation the rights 10 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | // copies of the Software, and to permit persons to whom the Software is 12 | // furnished to do so, subject to the following conditions: 13 | // 14 | // The above copyright notice and this permission notice shall be included in all 15 | // copies or substantial portions of the Software. 16 | // 17 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | // SOFTWARE. 24 | // 25 | 26 | #ifndef CMETROHASH_METROHASH_H 27 | #define CMETROHASH_METROHASH_H 28 | 29 | #pragma once 30 | 31 | #if defined (__cplusplus) 32 | extern "C" { 33 | #endif 34 | 35 | #include 36 | #include 37 | 38 | // MetroHash 64-bit hash functions 39 | void cmetrohash64_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out); 40 | void cmetrohash64_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out); 41 | 42 | 43 | /* rotate right idiom recognized by compiler*/ 44 | inline static uint64_t crotate_right(uint64_t v, unsigned k) 45 | { 46 | return (v >> k) | (v << (64 - k)); 47 | } 48 | 49 | // unaligned reads, fast and safe on Nehalem and later microarchitectures 50 | inline static uint64_t cread_u64(const void * const ptr) 51 | { 52 | return * (uint64_t *) ptr; 53 | } 54 | 55 | inline static uint64_t cread_u32(const void * const ptr) 56 | { 57 | return * (uint32_t *) ptr; 58 | } 59 | 60 | inline static uint64_t cread_u16(const void * const ptr) 61 | { 62 | return * (uint16_t *) ptr; 63 | } 64 | 65 | inline static uint64_t cread_u8 (const void * const ptr) 66 | { 67 | return * (uint8_t *) ptr; 68 | } 69 | 70 | 71 | #if defined (__cplusplus) 72 | } 73 | #endif 74 | #endif // #ifndef CMETROHASH_METROHASH_H 75 | -------------------------------------------------------------------------------- /crc.cpp: -------------------------------------------------------------------------------- 1 | #include "Platform.h" 2 | 3 | /* 4 | * This file is derived from crc32.c from the zlib-1.1.3 distribution 5 | * by Jean-loup Gailly and Mark Adler. 6 | */ 7 | 8 | /* crc32.c -- compute the CRC-32 of a data stream 9 | * Copyright (C) 1995-1998 Mark Adler 10 | * For conditions of distribution and use, see copyright notice in zlib.h 11 | */ 12 | 13 | 14 | /* ======================================================================== 15 | * Table of CRC-32's of all single-byte values (made by make_crc_table) 16 | */ 17 | static const uint32_t crc_table[256] = { 18 | 0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L, 19 | 0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L, 20 | 0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L, 21 | 0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL, 22 | 0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L, 23 | 0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L, 24 | 0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L, 25 | 0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL, 26 | 0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L, 27 | 0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL, 28 | 0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L, 29 | 0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L, 30 | 0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L, 31 | 0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL, 32 | 0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL, 33 | 0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L, 34 | 0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL, 35 | 0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L, 36 | 0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L, 37 | 0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L, 38 | 0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL, 39 | 0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L, 40 | 0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L, 41 | 0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL, 42 | 0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L, 43 | 0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L, 44 | 0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L, 45 | 0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L, 46 | 0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L, 47 | 0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL, 48 | 0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL, 49 | 0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L, 50 | 0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L, 51 | 0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL, 52 | 0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL, 53 | 0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L, 54 | 0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL, 55 | 0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L, 56 | 0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL, 57 | 0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L, 58 | 0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL, 59 | 0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L, 60 | 0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L, 61 | 0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL, 62 | 0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L, 63 | 0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L, 64 | 0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L, 65 | 0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L, 66 | 0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L, 67 | 0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L, 68 | 0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL, 69 | 0x2d02ef8dL 70 | }; 71 | 72 | /* ========================================================================= */ 73 | 74 | #define DO1(buf) crc = crc_table[((int)crc ^ (*buf++)) & 0xff] ^ (crc >> 8); 75 | #define DO2(buf) DO1(buf); DO1(buf); 76 | #define DO4(buf) DO2(buf); DO2(buf); 77 | #define DO8(buf) DO4(buf); DO4(buf); 78 | 79 | /* ========================================================================= */ 80 | 81 | void crc32 ( const void * key, int len, uint32_t seed, void * out ) 82 | { 83 | uint8_t * buf = (uint8_t*)key; 84 | uint32_t crc = seed ^ 0xffffffffL; 85 | 86 | while (len >= 8) 87 | { 88 | DO8(buf); 89 | len -= 8; 90 | } 91 | 92 | while(len--) 93 | { 94 | DO1(buf); 95 | } 96 | 97 | crc ^= 0xffffffffL; 98 | 99 | *(uint32_t*)out = crc; 100 | } 101 | -------------------------------------------------------------------------------- /crc32_hw.c: -------------------------------------------------------------------------------- 1 | /* Compile with gcc -O3 -msse4.2 ... */ 2 | 3 | #include 4 | #ifdef __SSE4_2__ 5 | #include 6 | 7 | // Byte-boundary alignment issues 8 | #define ALIGN_SIZE 0x08UL 9 | #define ALIGN_MASK (ALIGN_SIZE - 1) 10 | #define CALC_CRC(op, crc, type, buf, len) \ 11 | do { \ 12 | for (; (len) >= sizeof (type); (len) -= sizeof(type), buf += sizeof (type)) { \ 13 | (crc) = op((crc), *(type *) (buf)); \ 14 | } \ 15 | } while(0) 16 | 17 | 18 | /* Compute CRC-32C using the Intel hardware instruction. */ 19 | /* for better parallelization with bigger buffers see 20 | http://www.drdobbs.com/parallel/fast-parallelized-crc-computation-using/229401411 */ 21 | uint32_t crc32c_hw(const void *input, int len, uint32_t crc) 22 | { 23 | const char* buf = (const char*)input; 24 | 25 | // XOR the initial CRC with INT_MAX 26 | crc ^= 0xFFFFFFFF; 27 | 28 | // Align the input to the word boundary 29 | for (; (len > 0) && ((size_t)buf & ALIGN_MASK); len--, buf++) { 30 | crc = _mm_crc32_u8(crc, *buf); 31 | } 32 | 33 | // Blast off the CRC32 calculation 34 | #ifdef __x86_64__ 35 | CALC_CRC(_mm_crc32_u64, crc, uint64_t, buf, len); 36 | #endif 37 | CALC_CRC(_mm_crc32_u32, crc, uint32_t, buf, len); 38 | CALC_CRC(_mm_crc32_u16, crc, uint16_t, buf, len); 39 | CALC_CRC(_mm_crc32_u8, crc, uint8_t, buf, len); 40 | 41 | // Post-process the crc 42 | return (crc ^ 0xFFFFFFFF); 43 | } 44 | 45 | uint64_t crc64c_hw(const void *input, int len, uint32_t seed) 46 | { 47 | const char* buf = (const char*)input; 48 | uint64_t crc = (uint64_t)seed; 49 | 50 | // Align the input to the word boundary 51 | for (; (len > 0) && ((size_t)buf & ALIGN_MASK); len--, buf++) { 52 | crc = _mm_crc32_u8(crc, *buf); 53 | } 54 | 55 | // Blast off the CRC32 calculation 56 | #ifdef __x86_64__ 57 | CALC_CRC(_mm_crc32_u64, crc, uint64_t, buf, len); 58 | #endif 59 | CALC_CRC(_mm_crc32_u32, crc, uint32_t, buf, len); 60 | CALC_CRC(_mm_crc32_u16, crc, uint16_t, buf, len); 61 | CALC_CRC(_mm_crc32_u8, crc, uint8_t, buf, len); 62 | 63 | // Post-process the crc 64 | return crc; 65 | } 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /crc32c.h: -------------------------------------------------------------------------------- 1 | #ifndef CRC32C_H 2 | #define CRC32C_H 3 | 4 | #ifndef CRC32C_STATIC 5 | #ifdef CRC32C_EXPORTS 6 | #define CRC32C_API __declspec(dllexport) 7 | #else 8 | #define CRC32C_API __declspec(dllimport) 9 | #endif 10 | #else 11 | #define CRC32C_API 12 | #endif 13 | 14 | #include 15 | 16 | /* 17 | Computes CRC-32C (Castagnoli) checksum. Uses Intel's CRC32 instruction if it is available. 18 | Otherwise it uses a very fast software fallback. 19 | */ 20 | extern "C" CRC32C_API uint32_t crc32c_append( 21 | uint32_t crc, // Initial CRC value. Typically it's 0. 22 | // You can supply non-trivial initial value here. 23 | // Initial value can be used to chain CRC from multiple buffers. 24 | const uint8_t *input, // Data to be put through the CRC algorithm. 25 | size_t length); // Length of the data in the input buffer. 26 | 27 | extern "C" CRC32C_API void crc32c_unittest(); 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /discohash.cpp: -------------------------------------------------------------------------------- 1 | // also known as BEBB4185 2 | // Copyright 2020 Cris Stringfellow 3 | // Licensed under GPL-3.0 4 | // https://github.com/cris691/discohash 5 | #include 6 | #include 7 | #include "discohash.h" 8 | 9 | #if defined(_MSC_VER) 10 | 11 | #define FORCE_INLINE __forceinline 12 | 13 | // Other compilers 14 | 15 | #else // defined(_MSC_VER) 16 | 17 | #define FORCE_INLINE inline __attribute__((always_inline)) 18 | 19 | #endif // !defined(_MSC_VER) 20 | 21 | const int STATE = 32; 22 | const int STATE64 = STATE >> 3; 23 | const int STATEM = STATE-1; 24 | const int HSTATE64M = (STATE64 >> 1)-1; 25 | const int STATE64M = STATE64-1; 26 | uint8_t disco_buf[STATE] = {0}; 27 | uint64_t P = 0xFFFFFFFFFFFFFFFF - 58; 28 | uint64_t Q = 13166748625691186689U; 29 | uint8_t *ds8 = (uint8_t *)disco_buf; 30 | uint32_t *ds32 = (uint32_t *)disco_buf; 31 | uint64_t *ds = (uint64_t *)disco_buf; 32 | 33 | //-------- 34 | // State mix function 35 | 36 | FORCE_INLINE uint64_t rot( uint64_t v, int n) 37 | { 38 | n = n & 63U; 39 | if (n) 40 | v = (v >> n) | (v << (64-n)); 41 | return v; 42 | } 43 | 44 | FORCE_INLINE uint64_t rot32( uint64_t v, int n) 45 | { 46 | n = n & 31U; 47 | if (n) 48 | v = (v >> n) | (v << (64-n)); 49 | return v; 50 | } 51 | 52 | FORCE_INLINE uint8_t rot8( uint8_t v, int n) 53 | { 54 | n = n & 7U; 55 | if (n) 56 | v = (v >> n) | (v << (8-n)); 57 | return v; 58 | } 59 | 60 | FORCE_INLINE void mixA() 61 | { 62 | int i = ds32[0] & 1; 63 | int j = ds32[3] & 3; 64 | 65 | ds[0] = rot(ds[0], ds[i]); 66 | ds[i] *= (P % (ds32[j] + 1) + 1); 67 | ds[1] += ds32[j]; 68 | } 69 | 70 | FORCE_INLINE void mix(const int A) 71 | { 72 | const int B = A+1; 73 | ds[A] *= P; 74 | ds[A] = rot(ds[A], 23); 75 | ds[A] *= Q; 76 | //ds[A] = rot(ds[A], 23); 77 | 78 | ds[B] ^= ds[A]; 79 | 80 | ds[B] *= P; 81 | ds[B] = rot(ds[B], 23); 82 | ds[B] *= Q; 83 | //ds[B] = rot(ds[B], 23); 84 | } 85 | 86 | //--------- 87 | // Hash round function 88 | 89 | FORCE_INLINE void round( const uint64_t * m64, const uint8_t * m8, int len ) 90 | { 91 | int index = 0; 92 | int sindex = 0; 93 | uint64_t counter = 0xfaccadaccad09997; 94 | uint8_t counter8 = 137; 95 | 96 | for( int Len = len >> 3; index < Len; index++) { 97 | ds[sindex] += rot(m64[index] + index + counter + 1, 23); 98 | counter += ~m64[index] + 1; 99 | if ( sindex == HSTATE64M ) { 100 | mix(0); 101 | } else if ( sindex == STATE64M ) { 102 | mix(2); 103 | sindex = -1; 104 | } 105 | sindex++; 106 | } 107 | 108 | mix(1); 109 | 110 | index <<= 3; 111 | sindex = index&(STATEM); 112 | for( ; index < len; index++) { 113 | ds8[sindex] += rot8(m8[index] + index + counter8 + 1, 23); 114 | counter8 += ~m8[sindex] + 1; 115 | mix(index%STATE64M); 116 | if ( sindex >= STATEM ) { 117 | sindex = -1; 118 | } 119 | sindex++; 120 | } 121 | 122 | mix(0); 123 | mix(1); 124 | mix(2); 125 | } 126 | 127 | //--------- 128 | // main hash function 129 | 130 | void BEBB4185_64 ( const void * key, int len, unsigned seed, void * out ) 131 | { 132 | const uint8_t *key8Arr = (uint8_t *)key; 133 | const uint64_t *key64Arr = (uint64_t *)key; 134 | 135 | const uint8_t seedbuf[16] = {0}; 136 | const uint8_t *seed8Arr = (uint8_t *)seedbuf; 137 | const uint64_t *seed64Arr = (uint64_t *)seedbuf; 138 | uint32_t *seed32Arr = (uint32_t *)seedbuf; 139 | 140 | // the cali number from the Matrix (1999) 141 | seed32Arr[0] = 0xc5550690; 142 | seed32Arr[0] -= seed; 143 | // if seed mod doesn't work let's try reverse order of seed/key round calls 144 | seed32Arr[1] = 1 + seed; 145 | seed32Arr[2] = ~(1 - seed); 146 | seed32Arr[3] = (1+seed) * 0xf00dacca; 147 | 148 | // nothing up my sleeve 149 | ds[0] = 0x123456789abcdef0; 150 | ds[1] = 0x0fedcba987654321; 151 | ds[2] = 0xaccadacca80081e5; 152 | ds[3] = 0xf00baaf00f00baaa; 153 | 154 | round( key64Arr, key8Arr, len ); 155 | round( seed64Arr, seed8Arr, 16 ); 156 | round( ds, ds8, STATE ); 157 | 158 | /** 159 | printf("ds = %#018" PRIx64 " %#018" PRIx64 " %#018" PRIx64 " %#018" PRIx64 "\n", 160 | ds[0], ds[1], ds[2], ds[3] ); 161 | **/ 162 | 163 | const uint8_t output[STATE] = {0}; 164 | uint64_t *h = (uint64_t *)output; 165 | 166 | //h[0] = ds[1]; 167 | h[0] = ds[2]; 168 | h[1] = ds[3]; 169 | 170 | h[0] += h[1]; 171 | 172 | ((uint64_t *)out)[0] = h[0]; 173 | } 174 | -------------------------------------------------------------------------------- /discohash.h: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #ifndef _DISCOHASH_H_ 4 | #define _DISCOHASH_H_ 5 | 6 | // gotten from random.org 7 | // as hex bytes that I formatted into 64-bit values 8 | 9 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 10 | 11 | typedef unsigned char uint8_t; 12 | typedef unsigned int uint32_t; 13 | typedef unsigned __int64 uint64_t; 14 | 15 | // Other compilers 16 | 17 | #else // defined(_MSC_VER) 18 | 19 | #include 20 | 21 | #endif // !defined(_MSC_VER) 22 | 23 | void BEBB4185_64 ( const void * key, int len, uint32_t seed, void * out ); 24 | 25 | #endif // _DISCOHASH_H 26 | 27 | -------------------------------------------------------------------------------- /extra.lst: -------------------------------------------------------------------------------- 1 | SipHash 2 | GoodOAAT 3 | Murmur3A 4 | fasthash32 5 | MUM 6 | farmhash32_c 7 | xxh3 8 | Spooky32 9 | cmetrohash64_1o 10 | falkhash 11 | t1ha2_atonce 12 | wyhash -------------------------------------------------------------------------------- /falkhash-elf64.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cyan4973/smhasher/e1adb07114f5f1f156c10b70ad9255f24f42a7d1/falkhash-elf64.o -------------------------------------------------------------------------------- /falkhash-macho64.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cyan4973/smhasher/e1adb07114f5f1f156c10b70ad9255f24f42a7d1/falkhash-macho64.o -------------------------------------------------------------------------------- /falkhash.asm: -------------------------------------------------------------------------------- 1 | ;;; https://github.com/gamozolabs/falkhash 2 | ;;; nasm -f elf64 -o falkhash-elf64.o falkhash.asm 3 | ;;; nasm -f macho64 -o falkhash-macho64.o falkhash.asm 4 | [bits 64] 5 | 6 | ;%ifdef ELF 7 | ;section .code 8 | ;%endif 9 | ;%ifdef MACHO64 10 | ;section .text 11 | ;%endif 12 | 13 | %macro XMMPUSH 1 14 | sub rsp, 16 15 | movdqu [rsp], %1 16 | %endmacro 17 | 18 | %macro XMMPOP 1 19 | movdqu %1, [rsp] 20 | add rsp, 16 21 | %endmacro 22 | 23 | ; A chunk_size of 0x50 is ideal for AMD fam 15h platforms, which is what this 24 | ; was optimized and designed for. If you change this value, you have to 25 | ; manually add/remove movdqus and aesencs from the core loop. This must be 26 | ; divisible by 16. 27 | %define CHUNK_SIZE 0x50 28 | 29 | ; rdi -> data 30 | ; rsi -> len 31 | ; edx -> seed 32 | ; xmm5 <- 128-bit hash 33 | ; 34 | ; All non-output GP registers are preserved, conforming to the falkos ABI. 35 | ; All non-output XMM registers are also preserved. 36 | falkhash: 37 | push rax 38 | push rcx 39 | push rdi 40 | push rsi 41 | push rbp 42 | 43 | XMMPUSH xmm0 44 | XMMPUSH xmm1 45 | XMMPUSH xmm2 46 | XMMPUSH xmm3 47 | XMMPUSH xmm4 48 | 49 | sub rsp, CHUNK_SIZE 50 | 51 | ; Add the seed to the length 52 | mov rbp, rsi 53 | add rbp, rdx 54 | 55 | ; Place the length+seed for both the low and high 64-bits into xmm5, 56 | ; our hash output. 57 | pinsrq xmm5, rbp, 0 58 | pinsrq xmm5, rbp, 1 59 | 60 | .lewp: 61 | ; If we have less than a chunk, copy the partial chunk to the stack. 62 | cmp rsi, CHUNK_SIZE 63 | jb short .pad_last_chunk 64 | 65 | .continue: 66 | ; Read 5 pieces from memory into xmms 67 | movdqu xmm0, [rdi + 0x00] 68 | movdqu xmm1, [rdi + 0x10] 69 | movdqu xmm2, [rdi + 0x20] 70 | movdqu xmm3, [rdi + 0x30] 71 | movdqu xmm4, [rdi + 0x40] 72 | 73 | ; Mix all pieces into xmm0 74 | aesenc xmm0, xmm1 75 | aesenc xmm0, xmm2 76 | aesenc xmm0, xmm3 77 | aesenc xmm0, xmm4 78 | 79 | ; Finalize xmm0 by mixing with itself 80 | aesenc xmm0, xmm0 81 | 82 | ; Mix in xmm0 to the hash 83 | aesenc xmm5, xmm0 84 | 85 | ; Go to the next chunk, fall through if we're done. 86 | add rdi, CHUNK_SIZE 87 | sub rsi, CHUNK_SIZE 88 | jnz short .lewp 89 | jmp short .done 90 | 91 | .pad_last_chunk: 92 | ; Fill the stack with 0xff's, this is our padding 93 | push rdi 94 | lea rdi, [rsp + 8] 95 | mov rax, -1 96 | mov ecx, (CHUNK_SIZE / 8) 97 | rep stosq 98 | pop rdi 99 | 100 | ; Copy the remainder of data to the stack 101 | mov rcx, rsi 102 | mov rsi, rdi 103 | mov rdi, rsp 104 | rep movsb 105 | 106 | ; Make our data now come from the stack, and set the size to one chunk. 107 | mov rdi, rsp 108 | mov rsi, CHUNK_SIZE 109 | 110 | jmp short .continue 111 | 112 | .done: 113 | ; Finalize the hash. This is required at least once to pass 114 | ; Combination 0x8000000 and Combination 0x0000001. Need more than 1 to 115 | ; pass the Seed tests. We do 4 because they're pretty much free. 116 | ; Maybe we should actually use the seed better? Nah, more finalizing! 117 | aesenc xmm5, xmm5 118 | aesenc xmm5, xmm5 119 | aesenc xmm5, xmm5 120 | aesenc xmm5, xmm5 121 | 122 | add rsp, CHUNK_SIZE 123 | 124 | XMMPOP xmm4 125 | XMMPOP xmm3 126 | XMMPOP xmm2 127 | XMMPOP xmm1 128 | XMMPOP xmm0 129 | 130 | pop rbp 131 | pop rsi 132 | pop rdi 133 | pop rcx 134 | pop rax 135 | ret 136 | 137 | ; rdi -> pointer to data 138 | ; rsi -> len 139 | ; edx -> 32-bit seed 140 | ; rcx <> pointer to caller allocated 128-bit hash destination 141 | ; 142 | ; All non-output GP registers are preserved, conforming to the falkos ABI. 143 | ; All XMM registers are preserved. 144 | global _falkhash_test 145 | _falkhash_test: 146 | push rcx 147 | push rdx 148 | push rsi 149 | push rdi 150 | 151 | XMMPUSH xmm5 152 | 153 | %ifdef WIN 154 | ; Translate from windows to linux calling convention 155 | mov rdi, rcx 156 | mov rsi, rdx 157 | mov rdx, r8 158 | mov rcx, r9 159 | %endif 160 | 161 | call falkhash 162 | 163 | ; Store the hash into the hash destination 164 | movdqu [rcx], xmm5 165 | 166 | XMMPOP xmm5 167 | 168 | pop rdi 169 | pop rsi 170 | pop rdx 171 | pop rcx 172 | ret 173 | 174 | ; rax <- 64-bit rdtsc value 175 | global rdtsc64 176 | rdtsc64: 177 | push rdx 178 | 179 | rdtsc 180 | shl rdx, 32 181 | or rax, rdx 182 | 183 | pop rdx 184 | ret 185 | 186 | -------------------------------------------------------------------------------- /farmhash-c-test.cc: -------------------------------------------------------------------------------- 1 | #include "farmhash-c.h" 2 | 3 | #include 4 | #include 5 | 6 | void farmhash32_c_test ( const void * key, int len, uint32_t seed, void * out ) { 7 | *(uint32_t*)out = farmhash32_with_seed((const char *)key,(size_t)len,seed); 8 | } 9 | void farmhash64_c_test ( const void * key, int len, uint32_t seed, void * out ) { 10 | *(uint64_t*)out = farmhash64_with_seed((const char *)key,(size_t)len,(uint64_t)seed); 11 | } 12 | void farmhash128_c_test ( const void * key, int len, uint32_t seed, void * out ) { 13 | uint128_c_t s = make_uint128_c_t(seed, 0UL); 14 | uint128_c_t result = farmhash128_with_seed((const char *)key, (size_t)len, s); 15 | memcpy(out, &result, 128/8); 16 | } 17 | -------------------------------------------------------------------------------- /farsh.h: -------------------------------------------------------------------------------- 1 | #ifndef _FARSH_H 2 | #define _FARSH_H 3 | #include /* for size_t */ 4 | #include /* for uint32_t & uint64_t */ 5 | 6 | /* Return 32-bit hash of the buffer */ 7 | uint32_t farsh (const void *data, size_t bytes, uint64_t seed); 8 | 9 | /* Compute `n` 32-bit hashes starting with the hash number `k`, storing results to the `hash` buffer. 10 | It's `n` times slower than computation of single 32-bit hash. 11 | Hash computed by the `farsh` function has number 0. The function aborts if `k+n > 32`. */ 12 | void farsh_n (const void *data, size_t bytes, int k, int n, uint64_t seed, void *hash); 13 | 14 | /* Compute 32-bit hash using `key`, that should be 1024-byte long and aligned to 16-byte boundary. */ 15 | uint32_t farsh_keyed (const void *data, size_t bytes, const void *key, uint64_t seed); 16 | 17 | /* Compute `n` 32-bit hashes using `key`, storing results to the `hash` buffer. 18 | `key` should be `1024+16*(n-1)` bytes long and aligned to 16-byte boundary. */ 19 | void farsh_keyed_n (const void *data, size_t bytes, const void *key, int n, uint64_t seed, void *hash); 20 | 21 | /* Hash functions accept 64-bit `seed` that can be used to "personalize" the hash value. Use seed==0 if you don't need that feature. 22 | Seeding may have lower quality than in xxHash&co since the seed value mixed with block hashes rather than raw data. */ 23 | 24 | /* Symbolic names for the above-mentioned constants */ 25 | #define FARSH_MAX_HASHES 32 /* number of 32-bit hashes supported by the built-in key */ 26 | #define FARSH_BASE_KEY_SIZE 1024 /* size of user-supplied key required to compute 32-bit hash with index 0 */ 27 | #define FARSH_EXTRA_KEY_SIZE 16 /* extra bytes required to compute 32-bit hash with every next index */ 28 | #define FARSH_BASE_KEY_ALIGNMENT 16 /* user-supplied key should be aligned to this size, otherwise SSE2 code may fail. For maximum speed, it's recommended to align key to 64 bytes. */ 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /fasthash.cpp: -------------------------------------------------------------------------------- 1 | /* The MIT License 2 | 3 | Copyright (C) 2012 Zilong Tan (eric.zltan@gmail.com) 4 | 5 | Permission is hereby granted, free of charge, to any person 6 | obtaining a copy of this software and associated documentation 7 | files (the "Software"), to deal in the Software without 8 | restriction, including without limitation the rights to use, copy, 9 | modify, merge, publish, distribute, sublicense, and/or sell copies 10 | of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 20 | BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 21 | ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 22 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | */ 25 | 26 | #include "fasthash.h" 27 | 28 | // Compression function for Merkle-Damgard construction. 29 | // This function is generated using the framework provided. 30 | static inline uint64_t mix(uint64_t h) { 31 | h ^= h >> 23; 32 | h *= 0x2127599bf4325c37ULL; 33 | h ^= h >> 47; 34 | return h; 35 | } 36 | 37 | // objsize: 0-1fd: 509 38 | uint64_t fasthash64(const void *buf, size_t len, uint64_t seed) 39 | { 40 | const uint64_t m = 0x880355f21e6d1965ULL; 41 | const uint64_t *pos = (const uint64_t *)buf; 42 | const uint64_t *end = pos + (len / 8); 43 | const unsigned char *pos2; 44 | uint64_t h = seed ^ (len * m); 45 | uint64_t v; 46 | 47 | while (pos != end) { 48 | v = *pos++; 49 | h ^= mix(v); 50 | h *= m; 51 | } 52 | 53 | pos2 = (const unsigned char*)pos; 54 | v = 0; 55 | 56 | switch (len & 7) { 57 | case 7: v ^= (uint64_t)pos2[6] << 48; 58 | case 6: v ^= (uint64_t)pos2[5] << 40; 59 | case 5: v ^= (uint64_t)pos2[4] << 32; 60 | case 4: v ^= (uint64_t)pos2[3] << 24; 61 | case 3: v ^= (uint64_t)pos2[2] << 16; 62 | case 2: v ^= (uint64_t)pos2[1] << 8; 63 | case 1: v ^= (uint64_t)pos2[0]; 64 | h ^= mix(v); 65 | h *= m; 66 | } 67 | 68 | return mix(h); 69 | } 70 | 71 | // objsize: 0-236: 566 72 | uint32_t fasthash32(const void *buf, size_t len, uint32_t seed) 73 | { 74 | // the following trick converts the 64-bit hashcode to Fermat 75 | // residue, which shall retain information from both the higher 76 | // and lower parts of hashcode. 77 | uint64_t h = fasthash64(buf, len, seed); 78 | return h - (h >> 32); 79 | } 80 | -------------------------------------------------------------------------------- /fasthash.h: -------------------------------------------------------------------------------- 1 | /* The MIT License 2 | 3 | Copyright (C) 2012 Zilong Tan (eric.zltan@gmail.com) 4 | 5 | Permission is hereby granted, free of charge, to any person 6 | obtaining a copy of this software and associated documentation 7 | files (the "Software"), to deal in the Software without 8 | restriction, including without limitation the rights to use, copy, 9 | modify, merge, publish, distribute, sublicense, and/or sell copies 10 | of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 20 | BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 21 | ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 22 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | */ 25 | 26 | #ifndef _FASTHASH_H 27 | #define _FASTHASH_H 28 | 29 | #include 30 | #include 31 | 32 | #ifdef __cplusplus 33 | extern "C" { 34 | #endif 35 | 36 | /** 37 | * fasthash32 - 32-bit implementation of fasthash 38 | * @buf: data buffer 39 | * @len: data size 40 | * @seed: the seed 41 | */ 42 | uint32_t fasthash32(const void *buf, size_t len, uint32_t seed); 43 | 44 | /** 45 | * fasthash64 - 64-bit implementation of fasthash 46 | * @buf: data buffer 47 | * @len: data size 48 | * @seed: the seed 49 | */ 50 | uint64_t fasthash64(const void *buf, size_t len, uint64_t seed); 51 | 52 | #ifdef __cplusplus 53 | } 54 | #endif 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /fhtw-elf64.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cyan4973/smhasher/e1adb07114f5f1f156c10b70ad9255f24f42a7d1/fhtw-elf64.o -------------------------------------------------------------------------------- /fhtw-macho64.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cyan4973/smhasher/e1adb07114f5f1f156c10b70ad9255f24f42a7d1/fhtw-macho64.o -------------------------------------------------------------------------------- /fhtw.h: -------------------------------------------------------------------------------- 1 | typedef void* fhtw ; 2 | 3 | fhtw fhtw_new(int size); 4 | void fhtw_free(fhtw); 5 | int fhtw_set(fhtw, void* key, int key_len, void* value); 6 | void* fhtw_get(fhtw, void* key, int key_len); 7 | int fhtw_hash(void* key, int key_len); 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /fixupdocspeeds.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use File::Copy 'mv'; 4 | my $endtest = qr(^(?:---|\[\[\[|Input vcode 0x)); 5 | 6 | if (@ARGV) { 7 | readf($_) for @ARGV; 8 | } else { 9 | readf("log.speed"); 10 | } 11 | 12 | sub readf { 13 | my $fn = shift; 14 | my ($n,$speed,$hash); 15 | open(my $l, "<", $fn) or die "open $fn $!"; 16 | while (<$l>) { 17 | if (/^--- Testing ([\w-_]+) \"/) { 18 | if ($n) { # fixup previous name 19 | fixup($n,$speed,$hash,$fn) if $hash or $speed; 20 | } 21 | $n = $1; $speed = $hash = undef; 22 | } elsif (/^\[\[\[ 'Hashmap' Speed/) { 23 | while (<$l>) { 24 | last if $_ =~ $endtest; 25 | $hash .= $_; 26 | } 27 | } elsif (/^\[\[\[ Speed Tests/) { 28 | while (<$l>) { 29 | last if $_ =~ $endtest; 30 | $speed .= $_; 31 | } 32 | } elsif (/^\[\[\[ /) { 33 | last; 34 | } 35 | } 36 | fixup($n,$speed,$hash,$fn) if $n; # fixup last name 37 | } 38 | 39 | sub fixup { 40 | my ($n,$speed,$hash,$fn) = @_; 41 | return unless $n; 42 | return if !defined($speed) && !defined($hash); 43 | return if "doc/$n" eq $fn; 44 | open(my $I, "<", "doc/$n") or die "open doc/$n $!"; 45 | open(my $O, ">", "doc/$n.new") or die "open doc/$n.new $!"; 46 | my $found; 47 | while (<$I>) { 48 | print $O $_; 49 | # search for $n in doc 50 | if (/^--- Testing /) { 51 | $found = /^--- Testing $n /; 52 | } 53 | elsif ($found && defined $hash && /^\[\[\[ 'Hashmap' Speed/) { 54 | print $O $hash; 55 | while (<$I>) { 56 | last if $_ =~ $endtest; 57 | } 58 | print $O $_; 59 | } elsif ($found && defined $speed && /^\[\[\[ Speed Tests/) { 60 | print $O $speed; 61 | while (<$I>) { 62 | last if $_ =~ $endtest; 63 | } 64 | print $O $_; 65 | } 66 | } 67 | close $I; 68 | close $O; 69 | mv ("doc/$n.new", "doc/$n") if $found; 70 | } 71 | -------------------------------------------------------------------------------- /fixupdoctests.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use File::Copy 'mv'; 4 | use File::Basename 'basename'; 5 | my $endtest = qr(^(?:---|\[\[\[ |Input vcode 0x)); 6 | # mkdir partests; build/SMHasher --list|perl -alne'print $F[0] | parallel -j4 --bar 'build/SMHasher --test=Sparse,Permutation,Cyclic,TwoBytes,DiffDist,Text,Zeroes,Seed,Sanity,Avalanche,BIC,LongNeighbors,Diff,MomentChi2 {} >partests/{}' 7 | # build/SMHasher --list|perl -alne'print $F[0] | parallel -j4 --bar 'build/SMHasher --test=Sparse,Permutation,Cyclic,TwoBytes,DiffDist,Text,Zeroes,Seed {} >lowcoll/{}' 8 | my @keysettests = qw(Sparse Permutation Cyclic TwoBytes Window Text Zeroes Seed); 9 | my @othertests = qw(Sanity Avalanche Diff DiffDist BIC LongNeighbors MomentChi2); 10 | my %tests = map {$_ => 1} @keysettests, @othertests; 11 | my $testrx = '(' . join('|',@othertests) . ')'; 12 | $testrx = qr($testrx); 13 | 14 | if (@ARGV) { 15 | readf($_) for @ARGV; 16 | } else { 17 | readf($_) for ; 18 | } 19 | 20 | sub readf { 21 | my $fn = shift; 22 | my ($n,%r); 23 | open(my $l, "<", $fn) or die "open $fn $!"; 24 | $n = basename($fn) unless $fn =~ /^log\./; 25 | while (<$l>) { 26 | if (/^--- Testing ([\w_-]+) \"/) { 27 | if ($n) { # fixup previous name 28 | fixup($n,\%r,$fn) if %r; 29 | } 30 | $n = $1; %r = (); 31 | } 32 | NEXT: 33 | if (/^\[\[\[ Keyset '(.+?)' Tests/) { 34 | my $t = $1; 35 | while (<$l>) { 36 | goto NEXT if $_ =~ $endtest; 37 | $r{$t} .= $_; 38 | } 39 | } 40 | if (/^\[\[\[ '?$testrx'? /) { 41 | my $t = $1; 42 | while (<$l>) { 43 | goto NEXT if $_ =~ $endtest; 44 | $r{$t} .= $_; 45 | } 46 | } 47 | if (/^Input vcode 0x/) { 48 | last; 49 | } 50 | } 51 | fixup($n,\%r,$fn) if $n; # fixup last name 52 | } 53 | 54 | sub fixup { 55 | my ($n,$r,$fn) = @_; 56 | return unless $n; 57 | return if !%$r; 58 | return if "doc/$n" eq $fn; 59 | open(my $I, "<", "doc/$n") or die "open doc/$n $!"; 60 | open(my $O, ">", "doc/$n.new") or die "open doc/$n.new $!"; 61 | my $found; 62 | my %r = %$r; 63 | while (<$I>) { 64 | print $O $_; 65 | # search for $n in doc 66 | if (/^--- Testing /) { 67 | if ($found && %r) { 68 | print STDERR "tests not in doc/$n:\n", join(" ",sort keys %r), "\n"; 69 | } 70 | $found = /^--- Testing $n /; 71 | } 72 | NEXT1: 73 | if ($found && /^\[\[\[ Keyset '(.+?)' Tests/ && $r{$1}) { 74 | my $t = $1; 75 | print $O $r{$t}; 76 | delete $r{$t}; 77 | while (<$I>) { 78 | if ($_ =~ $endtest) { 79 | print $O $_; 80 | goto NEXT1; 81 | } 82 | } 83 | print $O $_; 84 | } 85 | if ($found && /^\[\[\[ '?$testrx'? / && $r{$1}) { 86 | my $t = $1; 87 | print $O $r{$t}; 88 | delete $r{$t}; 89 | while (<$I>) { 90 | if ($_ =~ $endtest) { 91 | print $O $_; 92 | goto NEXT1; 93 | } 94 | } 95 | print $O $_; 96 | } 97 | } 98 | if (%r) { 99 | print STDERR "finally tests not found in doc/$n:\n", join(" ",sort keys %r), "\n"; 100 | } 101 | close $I; 102 | close $O; 103 | mv ("doc/$n.new", "doc/$n") if $found; 104 | } 105 | -------------------------------------------------------------------------------- /floppsyhash.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Cris Stringfellow 2 | // Licensed under GPL-3.0 3 | // https://github.com/dosyago/floppsy 4 | #define _USE_MATH_DEFINES 5 | #include 6 | #include "floppsyhash.h" 7 | 8 | #if defined(_MSC_VER) 9 | 10 | #define FORCE_INLINE __forceinline 11 | 12 | // Other compilers 13 | 14 | #else // defined(_MSC_VER) 15 | 16 | #define FORCE_INLINE inline __attribute__((always_inline)) 17 | 18 | #endif // !defined(_MSC_VER) 19 | 20 | //--------- 21 | // Q function : Continued Egyptian Fraction update function 22 | 23 | FORCE_INLINE void q ( double * state, double key_val, 24 | double numerator, double denominator ) 25 | { 26 | state[0] += numerator / denominator; 27 | state[0] = 1.0 / state[0]; 28 | 29 | state[1] += key_val + M_PI; 30 | state[1] = numerator / state[1]; 31 | } 32 | 33 | //--------- 34 | // round function : process the message 35 | 36 | FORCE_INLINE void round ( const uint8_t * msg, long len, 37 | double * state ) 38 | { 39 | double numerator = 1.0; 40 | 41 | // Loop 42 | for( long i = 0; i < len; i++ ) { 43 | double val = (double)msg[i]; 44 | double denominator = (M_E * val + i + 1) / state[1]; 45 | 46 | q( state, val, numerator, denominator ); 47 | 48 | numerator = denominator + 1; 49 | } 50 | } 51 | 52 | //--------- 53 | // setup function : setup the state 54 | 55 | FORCE_INLINE void setup ( double * state, double init = 0 ) 56 | { 57 | state[0] += init != 0 ? pow(init + 1.0/init, 1.0/3) : 3.0; 58 | state[1] += init != 0 ? pow(init + 1.0/init, 1.0/7) : 1.0/7; 59 | } 60 | 61 | //--------- 62 | // floppsyhash 63 | // with 64 bit continued egyptian fractions 64 | 65 | void floppsyhash_64 ( const void * key, int len, 66 | uint32_t seed, void * out ) 67 | { 68 | const uint8_t * data = (const uint8_t *)key; 69 | uint8_t buf [16]; 70 | double * state = (double*)buf; 71 | uint32_t * state32 = (uint32_t*)buf; 72 | double seed32 = (double)seed; 73 | 74 | uint8_t * seedbuf; 75 | seedbuf = (uint8_t *)&seed; 76 | 77 | setup( state, seed32 ); 78 | round( seedbuf, 4, state ); 79 | round( data, len, state ); 80 | 81 | uint8_t output [8]; 82 | uint32_t * h = (uint32_t*)output; 83 | 84 | h[0] = state32[0] + state32[3]; 85 | h[1] = state32[1] + state32[2]; 86 | 87 | ((uint32_t*)out)[0] = h[0]; 88 | ((uint32_t*)out)[1] = h[1]; 89 | } 90 | 91 | -------------------------------------------------------------------------------- /floppsyhash.h: -------------------------------------------------------------------------------- 1 | /* https://github.com/crislin2046/floppsy */ 2 | #ifndef _FLOPPSYHASH_H_ 3 | #define _FLOPPSYHASH_H_ 4 | 5 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 6 | 7 | typedef unsigned char uint8_t; 8 | typedef unsigned int uint32_t; 9 | typedef unsigned __int64 uint64_t; 10 | 11 | // Other compilers 12 | 13 | #else // defined(_MSC_VER) 14 | 15 | #include 16 | 17 | #endif // !defined(_MSC_VER) 18 | 19 | void floppsyhash_64 ( const void * key, int len, uint32_t seed, void * out ); 20 | 21 | #endif // _FLOPPSYHASH_H_ 22 | -------------------------------------------------------------------------------- /halfsiphash.c: -------------------------------------------------------------------------------- 1 | #include "siphash.h" 2 | #include "siphash_impl.h" 3 | 4 | #ifndef U8TO64 5 | #define U8TO64 6 | static uint64_t INLINE 7 | U8TO64_LE(const unsigned char *p) { 8 | return *(const uint64_t *)p; 9 | } 10 | #endif 11 | 12 | #define TRACE 13 | #define cROUNDS 2 14 | #define dROUNDS 4 15 | #define ROTL(x, b) (uint32_t)(((x) << (b)) | ((x) >> (32 - (b)))) 16 | #define U8TO32_LE(p) \ 17 | (((uint32_t)((p)[0])) | ((uint32_t)((p)[1]) << 8) | \ 18 | ((uint32_t)((p)[2]) << 16) | ((uint32_t)((p)[3]) << 24)) 19 | #define SIPROUND \ 20 | do { \ 21 | v0 += v1; \ 22 | v1 = ROTL(v1, 5); \ 23 | v1 ^= v0; \ 24 | v0 = ROTL(v0, 16); \ 25 | v2 += v3; \ 26 | v3 = ROTL(v3, 8); \ 27 | v3 ^= v2; \ 28 | v0 += v3; \ 29 | v3 = ROTL(v3, 7); \ 30 | v3 ^= v0; \ 31 | v2 += v1; \ 32 | v1 = ROTL(v1, 13); \ 33 | v1 ^= v2; \ 34 | v2 = ROTL(v2, 16); \ 35 | } while (0) 36 | 37 | /* the faster half 32bit variant for the linux kernel */ 38 | uint32_t 39 | halfsiphash(const unsigned char key[16], const unsigned char *m, size_t len) { 40 | uint32_t v0 = 0; 41 | uint32_t v1 = 0; 42 | uint32_t v2 = 0x6c796765; 43 | uint32_t v3 = 0x74656462; 44 | uint32_t k0 = U8TO32_LE(key); 45 | uint32_t k1 = U8TO32_LE(key + 8); 46 | uint32_t mi; 47 | const uint8_t *end = m + len - (len % sizeof(uint32_t)); 48 | const int left = len & 3; 49 | uint32_t b = ((uint32_t)len) << 24; 50 | v3 ^= k1; 51 | v2 ^= k0; 52 | v1 ^= k1; 53 | v0 ^= k0; 54 | 55 | for (; m != end; m += 4) { 56 | mi = U8TO32_LE(m); 57 | v3 ^= mi; 58 | SIPROUND; 59 | SIPROUND; 60 | v0 ^= mi; 61 | } 62 | 63 | switch (left) { 64 | case 3: 65 | b |= ((uint32_t)m[2]) << 16; 66 | case 2: 67 | b |= ((uint32_t)m[1]) << 8; 68 | case 1: 69 | b |= ((uint32_t)m[0]); 70 | break; 71 | case 0: 72 | break; 73 | } 74 | 75 | v3 ^= b; 76 | SIPROUND; 77 | SIPROUND; 78 | v0 ^= b; 79 | v2 ^= 0xff; 80 | SIPROUND; 81 | SIPROUND; 82 | SIPROUND; 83 | SIPROUND; 84 | return v1 ^ v3; 85 | } 86 | 87 | -------------------------------------------------------------------------------- /hasshe2.c: -------------------------------------------------------------------------------- 1 | /* http://cessu.blogspot.com/2008/11/hashing-with-sse2-revisited-or-my-hash.html */ 2 | /* Compile with gcc -O3 -msse2 ... */ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #ifdef __SSE2__ 9 | #include 10 | 11 | static uint32_t coeffs[12] __attribute__((aligned(16))) = { 12 | /* Four carefully selected coefficients and interleaving zeros. */ 13 | 2561893793UL, 0, 1388747947UL, 0, 14 | 3077216833UL, 0, 3427609723UL, 0, 15 | /* 128 bits of random data. */ 16 | 0x564A4447, 0xC7265595, 0xE20C241D, 0x128FA608, 17 | }; 18 | 19 | #define COMBINE_AND_MIX(c_1, c_2, s_1, s_2, in) \ 20 | /* Phase 1: Perform four 32x32->64 bit multiplication with the \ 21 | input block and words 1 and 3 coeffs, respectively. This \ 22 | effectively propagates a bit change in input to 32 more \ 23 | significant bit positions. Combine into internal state by \ 24 | subtracting the result of multiplications from the internal \ 25 | state. */ \ 26 | s_1 = _mm_sub_epi64(s_1, _mm_mul_epu32(c_1, _mm_unpackhi_epi32(in, in))); \ 27 | s_2 = _mm_sub_epi64(s_2, _mm_mul_epu32(c_2, _mm_unpacklo_epi32(in, in))); \ 28 | \ 29 | /* Phase 2: Perform shifts and xors to propagate the 32-bit \ 30 | changes produced above into 64-bit (and even a little larger) \ 31 | changes in the internal state. */ \ 32 | /* state ^= state >64> 29; */ \ 33 | s_1 = _mm_xor_si128(s_1, _mm_srli_epi64(s_1, 29)); \ 34 | s_2 = _mm_xor_si128(s_2, _mm_srli_epi64(s_2, 29)); \ 35 | /* state +64= state <64< 16; */ \ 36 | s_1 = _mm_add_epi64(s_1, _mm_slli_epi64(s_1, 16)); \ 37 | s_2 = _mm_add_epi64(s_2, _mm_slli_epi64(s_2, 16)); \ 38 | /* state ^= state >64> 21; */ \ 39 | s_1 = _mm_xor_si128(s_1, _mm_srli_epi64(s_1, 21)); \ 40 | s_2 = _mm_xor_si128(s_2, _mm_srli_epi64(s_2, 21)); \ 41 | /* state +64= state <128< 32; */ \ 42 | s_1 = _mm_add_epi64(s_1, _mm_slli_si128(s_1, 4)); \ 43 | s_2 = _mm_add_epi64(s_2, _mm_slli_si128(s_2, 4)); \ 44 | \ 45 | /* Phase 3: Propagate the changes among the four 64-bit words by \ 46 | performing 64-bit subtractions and 32-bit word shuffling. */ \ 47 | s_1 = _mm_sub_epi64(s_1, s_2); \ 48 | s_2 = _mm_sub_epi64(_mm_shuffle_epi32(s_2, _MM_SHUFFLE(0, 3, 2, 1)), s_1); \ 49 | s_1 = _mm_sub_epi64(_mm_shuffle_epi32(s_1, _MM_SHUFFLE(0, 1, 3, 2)), s_2); \ 50 | s_2 = _mm_sub_epi64(_mm_shuffle_epi32(s_2, _MM_SHUFFLE(2, 1, 0, 3)), s_1); \ 51 | s_1 = _mm_sub_epi64(_mm_shuffle_epi32(s_1, _MM_SHUFFLE(2, 1, 0, 3)), s_2); \ 52 | \ 53 | /* With good coefficients any one-bit flip in the input has now \ 54 | changed all bits in the internal state with a probability \ 55 | between 45% to 55%. */ 56 | 57 | void hasshe2(const void *input_buf, int n_bytes, uint32_t seed, void *output_state) 58 | { 59 | __m128i coeffs_1, coeffs_2, rnd_data, input, state_1, state_2; 60 | 61 | assert(n_bytes % 16 == 0); 62 | 63 | coeffs_1 = _mm_load_si128((void *) coeffs); 64 | coeffs_2 = _mm_load_si128((void *) (coeffs + 4)); 65 | rnd_data = _mm_load_si128((void *) (coeffs + 8)); 66 | 67 | /* Initialize internal state to something random. (Alternatively, 68 | if hashing a chain of data, read in the previous hash result from 69 | somewhere.) */ 70 | state_1 = state_2 = rnd_data; 71 | 72 | while (n_bytes >= 16) { 73 | /* Read in 16 bytes, or 128 bits, from buf. Advance buf and 74 | decrement n_bytes accordingly. */ 75 | input = _mm_loadu_si128((void *) input_buf); 76 | input_buf += 16; 77 | n_bytes -= 16; 78 | 79 | COMBINE_AND_MIX(coeffs_1, coeffs_2, state_1, state_2, input); 80 | } 81 | 82 | /* Postprocessing. Copy half of the internal state into fake input, 83 | replace it with the constant rnd_data, and do one combine and mix 84 | phase more. */ 85 | input = state_1; 86 | state_1 = rnd_data; 87 | COMBINE_AND_MIX(coeffs_1, coeffs_2, state_1, state_2, input); 88 | 89 | _mm_storeu_si128((void *) output_state, state_1); 90 | _mm_storeu_si128((void *) (output_state + 16), state_2); 91 | } 92 | 93 | #endif 94 | -------------------------------------------------------------------------------- /jody_hash32.c: -------------------------------------------------------------------------------- 1 | /* Jody Bruchon's fast hashing function 2 | * 3 | * This function was written to generate a fast hash that also has a 4 | * fairly low collision rate. The collision rate is much higher than 5 | * a secure hash algorithm, but the calculation is drastically simpler 6 | * and faster. 7 | * 8 | * Copyright (C) 2014-2017 by Jody Bruchon 9 | * Released under The MIT License 10 | */ 11 | 12 | /* 13 | * ****** WARNING ******* 14 | * 15 | * This has been modified to integrate with SMHasher. DO NOT USE in other 16 | * projects. The proper source for jodyhash is available at: 17 | * https://github.com/jbruchon/jodyhash 18 | * 19 | * ****** WARNING ******* 20 | */ 21 | 22 | #include 23 | #include 24 | #include "jody_hash32.h" 25 | 26 | /* Width of a jody_hash. Changing this will also require 27 | * changing the width of tail masks to match. */ 28 | #define JODY_HASH_WIDTH 32 29 | 30 | /* DO NOT modify the shift unless you know what you're doing. 31 | * This shift was decided upon after lots of testing and 32 | * changing it will likely cause lots of hash collisions. */ 33 | #ifndef JODY_HASH_SHIFT 34 | #define JODY_HASH_SHIFT 14 35 | #endif 36 | 37 | /* The salt value's purpose is to cause each byte in the 38 | * jodyhash32_t word to have a positionally dependent variation. 39 | * It is injected into the calculation to prevent a string of 40 | * identical bytes from easily producing an identical hash. */ 41 | 42 | /* The tail mask table is used for block sizes that are 43 | * indivisible by the width of a jodyhash32_t. It is ANDed with the 44 | * final jodyhash32_t-sized element to zero out data in the buffer 45 | * that is not part of the data to be hashed. */ 46 | 47 | /* Set hash parameters based on requested hash width */ 48 | #if JODY_HASH_WIDTH == 64 49 | #define JODY_HASH_CONSTANT 0x1f3d5b79U 50 | static const jodyhash32_t tail_mask[] = { 51 | 0x0000000000000000, 52 | 0x00000000000000ff, 53 | 0x000000000000ffff, 54 | 0x0000000000ffffff, 55 | 0x00000000ffffffff, 56 | 0x000000ffffffffff, 57 | 0x0000ffffffffffff, 58 | 0x00ffffffffffffff, 59 | 0xffffffffffffffff 60 | }; 61 | #endif /* JODY_HASH_WIDTH == 64 */ 62 | #if JODY_HASH_WIDTH == 32 63 | #define JODY_HASH_CONSTANT 0x1f3d5b79U 64 | static const jodyhash32_t tail_mask[] = { 65 | 0x00000000, 66 | 0x000000ff, 67 | 0x0000ffff, 68 | 0x00ffffff, 69 | 0xffffffff, 70 | }; 71 | #endif /* JODY_HASH_WIDTH == 32 */ 72 | #if JODY_HASH_WIDTH == 16 73 | #define JODY_HASH_CONSTANT 0x1f5bU 74 | static const jodyhash32_t tail_mask[] = { 75 | 0x0000, 76 | 0x00ff, 77 | 0xffff, 78 | }; 79 | #endif /* JODY_HASH_WIDTH == 16 */ 80 | 81 | 82 | /* Hash a block of arbitrary size; must be divisible by sizeof(jodyhash32_t) 83 | * The first block should pass a start_hash of zero. 84 | * All blocks after the first should pass start_hash as the value 85 | * returned by the last call to this function. This allows hashing 86 | * of any amount of data. If data is not divisible by the size of 87 | * jodyhash32_t, it is MANDATORY that the caller provide a data buffer 88 | * which is divisible by sizeof(jodyhash32_t). */ 89 | extern jodyhash32_t jody_block_hash32(const jodyhash32_t *data, 90 | const jodyhash32_t start_hash, const size_t count) 91 | { 92 | jodyhash32_t hash = start_hash; 93 | jodyhash32_t element; 94 | jodyhash32_t partial_salt; 95 | size_t len; 96 | 97 | /* Don't bother trying to hash a zero-length block */ 98 | if (count == 0) return hash; 99 | 100 | len = count / sizeof(jodyhash32_t); 101 | for (; len > 0; len--) { 102 | element = *data; 103 | hash += element; 104 | hash += JODY_HASH_CONSTANT; 105 | hash = (hash << JODY_HASH_SHIFT) | hash >> (sizeof(jodyhash32_t) * 8 - JODY_HASH_SHIFT); /* bit rotate left */ 106 | hash ^= element; 107 | hash = (hash << JODY_HASH_SHIFT) | hash >> (sizeof(jodyhash32_t) * 8 - JODY_HASH_SHIFT); 108 | hash ^= JODY_HASH_CONSTANT; 109 | hash += element; 110 | data++; 111 | } 112 | 113 | /* Handle data tail (for blocks indivisible by sizeof(jodyhash32_t)) */ 114 | len = count & (sizeof(jodyhash32_t) - 1); 115 | if (len) { 116 | partial_salt = JODY_HASH_CONSTANT & tail_mask[len]; 117 | element = *data & tail_mask[len]; 118 | hash += element; 119 | hash += partial_salt; 120 | hash = (hash << JODY_HASH_SHIFT) | hash >> (sizeof(jodyhash32_t) * 8 - JODY_HASH_SHIFT); 121 | hash ^= element; 122 | hash = (hash << JODY_HASH_SHIFT) | hash >> (sizeof(jodyhash32_t) * 8 - JODY_HASH_SHIFT); 123 | hash ^= partial_salt; 124 | hash += element; 125 | } 126 | 127 | return hash; 128 | } 129 | -------------------------------------------------------------------------------- /jody_hash32.h: -------------------------------------------------------------------------------- 1 | /* Jody Bruchon's fast hashing function (headers) 2 | * See jody_hash.c for license information */ 3 | 4 | /* 5 | * ****** WARNING ******* 6 | * 7 | * This has been modified to integrate with SMHasher. DO NOT USE in other 8 | * projects. The proper source for jodyhash is available at: 9 | * https://github.com/jbruchon/jodyhash 10 | * 11 | * ****** WARNING ******* 12 | */ 13 | 14 | #ifndef JODY_HASH32_H 15 | #define JODY_HASH32_H 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | /* Required for uint64_t */ 22 | #include 23 | 24 | typedef uint32_t jodyhash32_t; 25 | 26 | /* Version increments when algorithm changes incompatibly */ 27 | #define JODY_HASH_VERSION 5 28 | 29 | extern jodyhash32_t jody_block_hash32(const jodyhash32_t *data, 30 | const jodyhash32_t start_hash, const size_t count); 31 | 32 | #ifdef __cplusplus 33 | } 34 | #endif 35 | 36 | #endif /* JODY_HASH32_H */ 37 | -------------------------------------------------------------------------------- /jody_hash64.c: -------------------------------------------------------------------------------- 1 | /* Jody Bruchon's fast hashing function 2 | * 3 | * This function was written to generate a fast hash that also has a 4 | * fairly low collision rate. The collision rate is much higher than 5 | * a secure hash algorithm, but the calculation is drastically simpler 6 | * and faster. 7 | * 8 | * Copyright (C) 2014-2017 by Jody Bruchon 9 | * Released under The MIT License 10 | */ 11 | 12 | /* 13 | * ****** WARNING ******* 14 | * 15 | * This has been modified to integrate with SMHasher. DO NOT USE in other 16 | * projects. The proper source for jodyhash is available at: 17 | * https://github.com/jbruchon/jodyhash 18 | * 19 | * ****** WARNING ******* 20 | */ 21 | 22 | #include 23 | #include 24 | #include "jody_hash64.h" 25 | 26 | /* Width of a jody_hash. Changing this will also require 27 | * changing the width of tail masks to match. */ 28 | #define JODY_HASH_WIDTH 64 29 | 30 | /* DO NOT modify the shift unless you know what you're doing. 31 | * This shift was decided upon after lots of testing and 32 | * changing it will likely cause lots of hash collisions. */ 33 | #ifndef JODY_HASH_SHIFT 34 | #define JODY_HASH_SHIFT 14 35 | #endif 36 | 37 | /* The salt value's purpose is to cause each byte in the 38 | * jodyhash_t word to have a positionally dependent variation. 39 | * It is injected into the calculation to prevent a string of 40 | * identical bytes from easily producing an identical hash. */ 41 | 42 | /* The tail mask table is used for block sizes that are 43 | * indivisible by the width of a jodyhash_t. It is ANDed with the 44 | * final jodyhash_t-sized element to zero out data in the buffer 45 | * that is not part of the data to be hashed. */ 46 | 47 | /* Set hash parameters based on requested hash width */ 48 | #if JODY_HASH_WIDTH == 64 49 | #define JODY_HASH_CONSTANT 0x1f3d5b79U 50 | static const jodyhash_t tail_mask[] = { 51 | 0x0000000000000000, 52 | 0x00000000000000ff, 53 | 0x000000000000ffff, 54 | 0x0000000000ffffff, 55 | 0x00000000ffffffff, 56 | 0x000000ffffffffff, 57 | 0x0000ffffffffffff, 58 | 0x00ffffffffffffff, 59 | 0xffffffffffffffff 60 | }; 61 | #endif /* JODY_HASH_WIDTH == 64 */ 62 | #if JODY_HASH_WIDTH == 32 63 | #define JODY_HASH_CONSTANT 0x1f3d5b79U 64 | static const jodyhash_t tail_mask[] = { 65 | 0x00000000, 66 | 0x000000ff, 67 | 0x0000ffff, 68 | 0x00ffffff, 69 | 0xffffffff, 70 | }; 71 | #endif /* JODY_HASH_WIDTH == 32 */ 72 | #if JODY_HASH_WIDTH == 16 73 | #define JODY_HASH_CONSTANT 0x1f5bU 74 | static const jodyhash_t tail_mask[] = { 75 | 0x0000, 76 | 0x00ff, 77 | 0xffff, 78 | }; 79 | #endif /* JODY_HASH_WIDTH == 16 */ 80 | 81 | 82 | /* Hash a block of arbitrary size; must be divisible by sizeof(jodyhash_t) 83 | * The first block should pass a start_hash of zero. 84 | * All blocks after the first should pass start_hash as the value 85 | * returned by the last call to this function. This allows hashing 86 | * of any amount of data. If data is not divisible by the size of 87 | * jodyhash_t, it is MANDATORY that the caller provide a data buffer 88 | * which is divisible by sizeof(jodyhash_t). */ 89 | jodyhash_t jody_block_hash(const jodyhash_t *data, 90 | const jodyhash_t start_hash, const size_t count) 91 | { 92 | jodyhash_t hash = start_hash; 93 | jodyhash_t element; 94 | jodyhash_t partial_salt; 95 | size_t len; 96 | 97 | /* Don't bother trying to hash a zero-length block */ 98 | if (count == 0) return hash; 99 | 100 | len = count / sizeof(jodyhash_t); 101 | for (; len > 0; len--) { 102 | element = *data; 103 | hash += element; 104 | hash += JODY_HASH_CONSTANT; 105 | hash = (hash << JODY_HASH_SHIFT) | hash >> (sizeof(jodyhash_t) * 8 - JODY_HASH_SHIFT); /* bit rotate left */ 106 | hash ^= element; 107 | hash = (hash << JODY_HASH_SHIFT) | hash >> (sizeof(jodyhash_t) * 8 - JODY_HASH_SHIFT); 108 | hash ^= JODY_HASH_CONSTANT; 109 | hash += element; 110 | data++; 111 | } 112 | 113 | /* Handle data tail (for blocks indivisible by sizeof(jodyhash_t)) */ 114 | len = count & (sizeof(jodyhash_t) - 1); 115 | if (len) { 116 | partial_salt = JODY_HASH_CONSTANT & tail_mask[len]; 117 | element = *data & tail_mask[len]; 118 | hash += element; 119 | hash += partial_salt; 120 | hash = (hash << JODY_HASH_SHIFT) | hash >> (sizeof(jodyhash_t) * 8 - JODY_HASH_SHIFT); 121 | hash ^= element; 122 | hash = (hash << JODY_HASH_SHIFT) | hash >> (sizeof(jodyhash_t) * 8 - JODY_HASH_SHIFT); 123 | hash ^= partial_salt; 124 | hash += element; 125 | } 126 | 127 | return hash; 128 | } 129 | -------------------------------------------------------------------------------- /jody_hash64.h: -------------------------------------------------------------------------------- 1 | /* Jody Bruchon's fast hashing function (headers) 2 | * See jody_hash.c for license information */ 3 | 4 | /* 5 | * ****** WARNING ******* 6 | * 7 | * This has been modified to integrate with SMHasher. DO NOT USE in other 8 | * projects. The proper source for jodyhash is available at: 9 | * https://github.com/jbruchon/jodyhash 10 | * 11 | * ****** WARNING ******* 12 | */ 13 | 14 | #ifndef JODY_HASH_H 15 | #define JODY_HASH_H 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | /* Required for uint64_t */ 22 | #include 23 | 24 | /* Width of a jody_hash. Changing this will also require 25 | * changing the width of tail masks to match. */ 26 | typedef uint64_t jodyhash_t; 27 | 28 | /* Version increments when algorithm changes incompatibly */ 29 | #define JODY_HASH_VERSION 5 30 | 31 | jodyhash_t jody_block_hash(const jodyhash_t *data, 32 | const jodyhash_t start_hash, const size_t count); 33 | 34 | #ifdef __cplusplus 35 | } 36 | #endif 37 | 38 | #endif /* JODY_HASH_H */ 39 | -------------------------------------------------------------------------------- /lookup3.cpp: -------------------------------------------------------------------------------- 1 | // lookup3 by Bob Jekins, code is public domain. 2 | 3 | #include "Platform.h" 4 | 5 | #define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k)))) 6 | 7 | #define mix(a,b,c) \ 8 | { \ 9 | a -= c; a ^= rot(c, 4); c += b; \ 10 | b -= a; b ^= rot(a, 6); a += c; \ 11 | c -= b; c ^= rot(b, 8); b += a; \ 12 | a -= c; a ^= rot(c,16); c += b; \ 13 | b -= a; b ^= rot(a,19); a += c; \ 14 | c -= b; c ^= rot(b, 4); b += a; \ 15 | } 16 | 17 | #define final(a,b,c) \ 18 | { \ 19 | c ^= b; c -= rot(b,14); \ 20 | a ^= c; a -= rot(c,11); \ 21 | b ^= a; b -= rot(a,25); \ 22 | c ^= b; c -= rot(b,16); \ 23 | a ^= c; a -= rot(c,4); \ 24 | b ^= a; b -= rot(a,14); \ 25 | c ^= b; c -= rot(b,24); \ 26 | } 27 | 28 | // objsize: 0x0-0x155: 341 29 | uint32_t lookup3 ( const char * key, int length, uint32_t initval ) 30 | { 31 | uint32_t a,b,c; /* internal state */ 32 | 33 | a = b = c = 0xdeadbeef + ((uint32_t)length) + initval; 34 | 35 | const uint32_t *k = (const uint32_t *)key; /* read 32-bit chunks */ 36 | 37 | /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ 38 | while (length > 12) 39 | { 40 | a += k[0]; 41 | b += k[1]; 42 | c += k[2]; 43 | mix(a,b,c); 44 | length -= 12; 45 | k += 3; 46 | } 47 | 48 | switch(length) 49 | { 50 | case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; 51 | case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break; 52 | case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break; 53 | case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break; 54 | case 8 : b+=k[1]; a+=k[0]; break; 55 | case 7 : b+=k[1]&0xffffff; a+=k[0]; break; 56 | case 6 : b+=k[1]&0xffff; a+=k[0]; break; 57 | case 5 : b+=k[1]&0xff; a+=k[0]; break; 58 | case 4 : a+=k[0]; break; 59 | case 3 : a+=k[0]&0xffffff; break; 60 | case 2 : a+=k[0]&0xffff; break; 61 | case 1 : a+=k[0]&0xff; break; 62 | case 0 : { return c; } /* zero length strings require no mixing */ 63 | } 64 | 65 | final(a,b,c); 66 | 67 | return c; 68 | } 69 | -------------------------------------------------------------------------------- /md5.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include "Types.h" 3 | 4 | // "Derived from the RSA Data Security, Inc. MD5 Message Digest Algorithm" 5 | 6 | /** 7 | * \brief MD5 context structure 8 | */ 9 | typedef struct 10 | { 11 | unsigned long total[2]; /*!< number of bytes processed */ 12 | unsigned long state[4]; /*!< intermediate digest state */ 13 | unsigned char buffer[64]; /*!< data block being processed */ 14 | 15 | unsigned char ipad[64]; /*!< HMAC: inner padding */ 16 | unsigned char opad[64]; /*!< HMAC: outer padding */ 17 | } 18 | md5_context; 19 | 20 | /** 21 | * \brief MD5 context setup 22 | * 23 | * \param ctx context to be initialized 24 | */ 25 | void md5_starts( md5_context *ctx ); 26 | 27 | /** 28 | * \brief MD5 process buffer 29 | * 30 | * \param ctx MD5 context 31 | * \param input buffer holding the data 32 | * \param ilen length of the input data 33 | */ 34 | void md5_update( md5_context *ctx, unsigned char *input, int ilen ); 35 | 36 | /** 37 | * \brief MD5 final digest 38 | * 39 | * \param ctx MD5 context 40 | * \param output MD5 checksum result 41 | */ 42 | void md5_finish( md5_context *ctx, unsigned char output[16] ); 43 | 44 | /** 45 | * \brief Output = MD5( input buffer ) 46 | * 47 | * \param input buffer holding the data 48 | * \param ilen length of the input data 49 | * \param output MD5 checksum result 50 | */ 51 | void md5( unsigned char *input, int ilen, unsigned char output[16] ); 52 | 53 | /** 54 | * \brief Output = MD5( file contents ) 55 | * 56 | * \param path input file name 57 | * \param output MD5 checksum result 58 | * 59 | * \return 0 if successful, 1 if fopen failed, 60 | * or 2 if fread failed 61 | */ 62 | int md5_file( char *path, unsigned char output[16] ); 63 | 64 | /** 65 | * \brief MD5 HMAC context setup 66 | * 67 | * \param ctx HMAC context to be initialized 68 | * \param key HMAC secret key 69 | * \param keylen length of the HMAC key 70 | */ 71 | void md5_hmac_starts( md5_context *ctx, unsigned char *key, int keylen ); 72 | 73 | /** 74 | * \brief MD5 HMAC process buffer 75 | * 76 | * \param ctx HMAC context 77 | * \param input buffer holding the data 78 | * \param ilen length of the input data 79 | */ 80 | void md5_hmac_update( md5_context *ctx, unsigned char *input, int ilen ); 81 | 82 | /** 83 | * \brief MD5 HMAC final digest 84 | * 85 | * \param ctx HMAC context 86 | * \param output MD5 HMAC checksum result 87 | */ 88 | void md5_hmac_finish( md5_context *ctx, unsigned char output[16] ); 89 | 90 | /** 91 | * \brief Output = HMAC-MD5( hmac key, input buffer ) 92 | * 93 | * \param key HMAC secret key 94 | * \param keylen length of the HMAC key 95 | * \param input buffer holding the data 96 | * \param ilen length of the input data 97 | * \param output HMAC-MD5 result 98 | */ 99 | void md5_hmac( unsigned char *key, int keylen, 100 | unsigned char *input, int ilen, 101 | unsigned char output[16] ); 102 | 103 | /** 104 | * \brief Checkup routine 105 | * 106 | * \return 0 if successful, or 1 if the test failed 107 | */ 108 | int md5_self_test( int verbose ); 109 | 110 | /* 111 | * MD5 context setup 112 | */ 113 | void md5_starts( md5_context *ctx ); 114 | 115 | /* 116 | * MD5 process buffer 117 | */ 118 | void md5_update( md5_context *ctx, unsigned char *input, int ilen ); 119 | 120 | /* 121 | * MD5 final digest 122 | */ 123 | void md5_finish( md5_context *ctx, unsigned char output[16] ); 124 | 125 | unsigned int md5hash ( const void * input, int len, uint32_t seed ); 126 | -------------------------------------------------------------------------------- /metrohash/metrohash.h: -------------------------------------------------------------------------------- 1 | // metrohash.h 2 | // 3 | // Copyright 2015-2018 J. Andrew Rogers 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | #ifndef METROHASH_METROHASH_H 18 | #define METROHASH_METROHASH_H 19 | 20 | #include "metrohash64.h" 21 | #include "metrohash64crc.h" 22 | #include "metrohash128.h" 23 | #include "metrohash128crc.h" 24 | 25 | #endif // #ifndef METROHASH_METROHASH_H 26 | -------------------------------------------------------------------------------- /metrohash/metrohash128.h: -------------------------------------------------------------------------------- 1 | // metrohash128.h 2 | // 3 | // Copyright 2015-2018 J. Andrew Rogers 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | #ifndef METROHASH_METROHASH_128_H 18 | #define METROHASH_METROHASH_128_H 19 | 20 | #include 21 | 22 | class MetroHash128 23 | { 24 | public: 25 | static const uint32_t bits = 128; 26 | 27 | // Constructor initializes the same as Initialize() 28 | MetroHash128(const uint64_t seed=0); 29 | 30 | // Initializes internal state for new hash with optional seed 31 | void Initialize(const uint64_t seed=0); 32 | 33 | // Update the hash state with a string of bytes. If the length 34 | // is sufficiently long, the implementation switches to a bulk 35 | // hashing algorithm directly on the argument buffer for speed. 36 | void Update(const uint8_t * buffer, const uint64_t length); 37 | 38 | // Constructs the final hash and writes it to the argument buffer. 39 | // After a hash is finalized, this instance must be Initialized()-ed 40 | // again or the behavior of Update() and Finalize() is undefined. 41 | void Finalize(uint8_t * const hash); 42 | 43 | // A non-incremental function implementation. This can be significantly 44 | // faster than the incremental implementation for some usage patterns. 45 | static void Hash(const uint8_t * buffer, const uint64_t length, uint8_t * const hash, const uint64_t seed=0); 46 | 47 | // Does implementation correctly execute test vectors? 48 | static bool ImplementationVerified(); 49 | 50 | // test vectors -- Hash(test_string, seed=0) => test_seed_0 51 | static const char * test_string; 52 | static const uint8_t test_seed_0[16]; 53 | static const uint8_t test_seed_1[16]; 54 | 55 | private: 56 | static const uint64_t k0 = 0xC83A91E1; 57 | static const uint64_t k1 = 0x8648DBDB; 58 | static const uint64_t k2 = 0x7BDEC03B; 59 | static const uint64_t k3 = 0x2F5870A5; 60 | 61 | struct { uint64_t v[4]; } state; 62 | struct { uint8_t b[32]; } input; 63 | uint64_t bytes; 64 | }; 65 | 66 | 67 | // Legacy 128-bit hash functions -- do not use 68 | void metrohash128_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out); 69 | void metrohash128_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out); 70 | 71 | 72 | #endif // #ifndef METROHASH_METROHASH_128_H 73 | -------------------------------------------------------------------------------- /metrohash/metrohash128crc.h: -------------------------------------------------------------------------------- 1 | // metrohash128crc.h 2 | // 3 | // Copyright 2015-2018 J. Andrew Rogers 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | #ifndef METROHASH_METROHASH_128_CRC_H 18 | #define METROHASH_METROHASH_128_CRC_H 19 | 20 | #include 21 | 22 | // Legacy 128-bit hash functions 23 | void metrohash128crc_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out); 24 | void metrohash128crc_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out); 25 | 26 | 27 | #endif // #ifndef METROHASH_METROHASH_128_CRC_H 28 | -------------------------------------------------------------------------------- /metrohash/metrohash64.h: -------------------------------------------------------------------------------- 1 | // metrohash64.h 2 | // 3 | // Copyright 2015-2018 J. Andrew Rogers 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | #ifndef METROHASH_METROHASH_64_H 18 | #define METROHASH_METROHASH_64_H 19 | 20 | #include 21 | 22 | class MetroHash64 23 | { 24 | public: 25 | static const uint32_t bits = 64; 26 | 27 | // Constructor initializes the same as Initialize() 28 | MetroHash64(const uint64_t seed=0); 29 | 30 | // Initializes internal state for new hash with optional seed 31 | void Initialize(const uint64_t seed=0); 32 | 33 | // Update the hash state with a string of bytes. If the length 34 | // is sufficiently long, the implementation switches to a bulk 35 | // hashing algorithm directly on the argument buffer for speed. 36 | void Update(const uint8_t * buffer, const uint64_t length); 37 | 38 | // Constructs the final hash and writes it to the argument buffer. 39 | // After a hash is finalized, this instance must be Initialized()-ed 40 | // again or the behavior of Update() and Finalize() is undefined. 41 | void Finalize(uint8_t * const hash); 42 | 43 | // A non-incremental function implementation. This can be significantly 44 | // faster than the incremental implementation for some usage patterns. 45 | static void Hash(const uint8_t * buffer, const uint64_t length, uint8_t * const hash, const uint64_t seed=0); 46 | 47 | // Does implementation correctly execute test vectors? 48 | static bool ImplementationVerified(); 49 | 50 | // test vectors -- Hash(test_string, seed=0) => test_seed_0 51 | static const char * test_string; 52 | static const uint8_t test_seed_0[8]; 53 | static const uint8_t test_seed_1[8]; 54 | 55 | private: 56 | static const uint64_t k0 = 0xD6D018F5; 57 | static const uint64_t k1 = 0xA2AA033B; 58 | static const uint64_t k2 = 0x62992FC1; 59 | static const uint64_t k3 = 0x30BC5B29; 60 | 61 | struct { uint64_t v[4]; } state; 62 | struct { uint8_t b[32]; } input; 63 | uint64_t bytes; 64 | uint64_t vseed; 65 | }; 66 | 67 | 68 | // Legacy 64-bit hash functions -- do not use 69 | void metrohash64_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out); 70 | void metrohash64_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out); 71 | 72 | 73 | #endif // #ifndef METROHASH_METROHASH_64_H 74 | -------------------------------------------------------------------------------- /metrohash/metrohash64crc.h: -------------------------------------------------------------------------------- 1 | // metrohash64crc.h 2 | // 3 | // Copyright 2015-2018 J. Andrew Rogers 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | #ifndef METROHASH_METROHASH_64_CRC_H 18 | #define METROHASH_METROHASH_64_CRC_H 19 | 20 | #include 21 | 22 | // Legacy 64-bit hash functions -- do not use 23 | void metrohash64crc_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out); 24 | void metrohash64crc_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out); 25 | 26 | #endif // #ifndef METROHASH_METROHASH_64_CRC_H 27 | -------------------------------------------------------------------------------- /metrohash/platform.h: -------------------------------------------------------------------------------- 1 | // platform.h 2 | // 3 | // Copyright 2015-2018 J. Andrew Rogers 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | #ifndef METROHASH_PLATFORM_H 18 | #define METROHASH_PLATFORM_H 19 | 20 | #include 21 | 22 | // rotate right idiom recognized by most compilers 23 | inline static uint64_t rotate_right(uint64_t v, unsigned k) 24 | { 25 | return (v >> k) | (v << (64 - k)); 26 | } 27 | 28 | // unaligned reads, fast and safe on Nehalem and later microarchitectures 29 | inline static uint64_t read_u64(const void * const ptr) 30 | { 31 | return static_cast(*reinterpret_cast(ptr)); 32 | } 33 | 34 | inline static uint64_t read_u32(const void * const ptr) 35 | { 36 | return static_cast(*reinterpret_cast(ptr)); 37 | } 38 | 39 | inline static uint64_t read_u16(const void * const ptr) 40 | { 41 | return static_cast(*reinterpret_cast(ptr)); 42 | } 43 | 44 | inline static uint64_t read_u8 (const void * const ptr) 45 | { 46 | return static_cast(*reinterpret_cast(ptr)); 47 | } 48 | 49 | 50 | #endif // #ifndef METROHASH_PLATFORM_H 51 | -------------------------------------------------------------------------------- /mir-hash.h: -------------------------------------------------------------------------------- 1 | /* This file is a part of MIR project. 2 | 3 | Copyright (C) 2018, 2019 Vladimir Makarov . 4 | */ 5 | 6 | /* Simple high-quality multiplicative hash passing demerphq-smhsher, 7 | faster than spooky, city, or xxhash for strings less 100 bytes. 8 | Hash for the same key can be different on different architectures. 9 | To get machine-independent hash, use mir_hash_strict which is about 10 | 1.5 times slower than mir_hash. */ 11 | #ifndef __MIR_HASH__ 12 | #define __MIR_HASH__ 13 | 14 | #include 15 | #include 16 | 17 | #if defined(__x86_64__) || defined(__i386__) || defined(__PPC64__) || defined(__s390__) \ 18 | || defined(__m32c__) || defined(cris) || defined(__CR16__) || defined(__vax__) \ 19 | || defined(__m68k__) || defined(__aarch64__) || defined(_M_AMD64) || defined(_M_IX86) 20 | #define MIR_HASH_UNALIGNED_ACCESS 1 21 | #else 22 | #define MIR_HASH_UNALIGNED_ACCESS 0 23 | #endif 24 | 25 | static inline uint64_t mir_get_key_part (const uint8_t *v, size_t len, int relax_p) { 26 | size_t i, start = 0; 27 | uint64_t tail = 0; 28 | 29 | if (relax_p || __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) { 30 | #if MIR_HASH_UNALIGNED_ACCESS 31 | if (len == sizeof (uint64_t)) return *(uint64_t *) v; 32 | if (len >= sizeof (uint32_t)) { 33 | tail = (uint64_t) * (uint32_t *) v << 32; 34 | start = 4; 35 | } 36 | #endif 37 | } 38 | for (i = start; i < len; i++) tail = (tail >> 8) | ((uint64_t) v[i] << 56); 39 | return tail; 40 | } 41 | 42 | static const uint64_t p1 = 0X65862b62bdf5ef4d, p2 = 0X288eea216831e6a7; 43 | static inline uint64_t mir_mum (uint64_t v, uint64_t c, int relax_p) { 44 | if (relax_p) { 45 | #if defined(__SIZEOF_INT128__) 46 | __uint128_t r = (__uint128_t) v * (__uint128_t) c; 47 | return (uint64_t) (r >> 64) + (uint64_t) r; 48 | #endif 49 | } 50 | uint64_t v1 = v >> 32, v2 = (uint32_t) v, c1 = c >> 32, c2 = (uint32_t) c, rm = v2 * c1 + v1 * c2; 51 | return v1 * c1 + (rm >> 32) + v2 * c2 + (rm << 32); 52 | } 53 | 54 | static inline uint64_t mir_round (uint64_t state, uint64_t v, int relax_p) { 55 | state ^= mir_mum (v, p1, relax_p); 56 | return state ^ mir_mum (state, p2, relax_p); 57 | } 58 | 59 | static inline uint64_t mir_hash_1 (const void *key, size_t len, uint64_t seed, int relax_p) { 60 | const uint8_t *v = (const uint8_t *) key; 61 | uint64_t r = seed + len; 62 | 63 | for (; len >= 16; len -= 16, v += 16) { 64 | r ^= mir_mum (mir_get_key_part (v, 8, relax_p), p1, relax_p); 65 | r ^= mir_mum (mir_get_key_part (v + 8, 8, relax_p), p2, relax_p); 66 | r ^= mir_mum (r, p1, relax_p); 67 | } 68 | if (len >= 8) { 69 | r ^= mir_mum (mir_get_key_part (v, 8, relax_p), p1, relax_p); 70 | len -= 8, v += 8; 71 | } 72 | if (len != 0) r ^= mir_mum (mir_get_key_part (v, len, relax_p), p2, relax_p); 73 | return mir_round (r, r, relax_p); 74 | } 75 | 76 | static inline uint64_t mir_hash (const void *key, size_t len, uint64_t seed) { 77 | return mir_hash_1 (key, len, seed, 1); 78 | } 79 | 80 | static inline uint64_t mir_hash_strict (const void *key, size_t len, uint64_t seed) { 81 | return mir_hash_1 (key, len, seed, 0); 82 | } 83 | 84 | static inline uint64_t mir_hash_init (uint64_t seed) { return seed; } 85 | static inline uint64_t mir_hash_step (uint64_t h, uint64_t key) { return mir_round (h, key, 1); } 86 | static inline uint64_t mir_hash_finish (uint64_t h) { return mir_round (h, h, 1); } 87 | 88 | static inline uint64_t mir_hash64 (uint64_t key, uint64_t seed) { 89 | return mir_hash_finish (mir_hash_step (mir_hash_init (seed), key)); 90 | } 91 | 92 | #endif 93 | -------------------------------------------------------------------------------- /mum.cc: -------------------------------------------------------------------------------- 1 | #include "mum.h" 2 | 3 | void mum_hash_test(const void *key, int len, uint32_t seed, void *out) { 4 | *(uint64_t *)out = mum_hash(key, len, seed); 5 | } 6 | -------------------------------------------------------------------------------- /opt_cmetrohash.h: -------------------------------------------------------------------------------- 1 | // Original: metrohash64.cpp 2 | // Port to C: Frank Denis (jedisct1@github) 3 | // Optimized variants: Paul G (paulie-g@github, paulg@perforge.net) 4 | // 5 | // 6 | // The MIT License (MIT) 7 | // 8 | // Copyright (c) 2015 J. Andrew Rogers 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the "Software"), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is 15 | // furnished to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | // 20 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 | // SOFTWARE. 27 | // 28 | 29 | #ifndef OPT_CMETROHASH_METROHASH_H 30 | #define OPT_CMETROHASH_METROHASH_H 31 | 32 | #pragma once 33 | 34 | #if defined (__cplusplus) 35 | extern "C" { 36 | #endif 37 | 38 | /* 39 | * Optimized variants of a c port of MetroHash 64-bit hash functions 40 | * 41 | * *_optshort: optimized for speed with relatively short keys at small expense to bulk hashing throughput 42 | * 43 | */ 44 | 45 | void cmetrohash64_1_optshort(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out); 46 | 47 | 48 | 49 | #define BRANCH_MAXNO(x) (sizeof x / sizeof x[0]) 50 | #define BRANCH_MAXINDEX(x) (BRANCH_MAXNO(x) - 1) 51 | #define BRANCH_THRESHHOLD(x) (BRANCH_MAXNO(x) - 2) 52 | 53 | #if defined (__cplusplus) 54 | } 55 | #endif 56 | #endif // #ifndef OPT_CMETROHASH_METROHASH_H 57 | -------------------------------------------------------------------------------- /os_dependent_stuff.asm: -------------------------------------------------------------------------------- 1 | ; syscalls 2 | %ifidn __OUTPUT_FORMAT__,elf64 3 | ; http://lxr.linux.no/linux+v3.13.5/arch/x86/syscalls/syscall_64.tbl 4 | %define SYSCALL_OPEN 2 5 | %define SYSCALL_WRITE 1 6 | %define SYSCALL_MMAP 9 7 | %define SYSCALL_MUNMAP 11 8 | %define SYSCALL_PWRITE 18 9 | %define SYSCALL_FORK 57 10 | %define SYSCALL_WAITID 247 11 | %define SYSCALL_EXIT 60 12 | %elifidn __OUTPUT_FORMAT__,macho64 13 | ; http://www.opensource.apple.com/source/xnu/xnu-1456.1.26/bsd/kern/syscalls.master 14 | %define SYSCALL_OPEN 0x2000005 15 | %define SYSCALL_WRITE 0x2000004 16 | %define SYSCALL_MMAP 0x20000C5 17 | %define SYSCALL_MUNMAP 0x2000049 18 | %define SYSCALL_PWRITE 0x200009A 19 | %define SYSCALL_FORK 0x2000002 20 | %define SYSCALL_WAITID 0x20000AD 21 | %define SYSCALL_EXIT 0x2000001 22 | %endif 23 | 24 | ; fcntls 25 | %ifidn __OUTPUT_FORMAT__,elf64 26 | ; http://lxr.linux.no/linux+v3.13.5/include/uapi/asm-generic/fcntl.h 27 | %define O_RDONLY 0o0000000 28 | %define O_WRONLY 0o0000001 29 | %define O_RDWR 0o0000002 30 | %define O_CREAT 0o0000100 31 | %define O_EXCL 0o0000200 32 | %define O_NOCTTY 0o0000400 33 | %define O_TRUNC 0o0001000 34 | %define O_APPEND 0o0002000 35 | %define O_NONBLOCK 0o0004000 36 | %define O_DSYNC 0o0010000 37 | %define FASYNC 0o0020000 38 | %define O_DIRECT 0o0040000 39 | %define O_LARGEFILE 0o0100000 40 | %define O_DIRECTORY 0o0200000 41 | %define O_NOFOLLOW 0o0400000 42 | %define O_NOATIME 0o1000000 43 | %define O_CLOEXEC 0o2000000 44 | %define O_SYNC 0o4000000 45 | %define O_PATH 0o10000000 46 | %elifidn __OUTPUT_FORMAT__,macho64 47 | ; http://www.opensource.apple.com/source/xnu/xnu-1456.1.26/bsd/sys/fcntl.h 48 | %define O_RDONLY 0x0000 49 | %define O_WRONLY 0x0001 50 | %define O_RDWR 0x0002 51 | %define O_NONBLOCK 0x0004 52 | %define O_APPEND 0x0008 53 | %define O_SHLOCK 0x0010 54 | %define O_EXLOCK 0x0020 55 | %define O_ASYNC 0x0040 56 | %define O_SYNC 0x0080 57 | %define O_NOFOLOW 0x0100 58 | %define O_CREAT 0x0200 59 | %define O_TRUNC 0x0400 60 | %define O_EXCL 0x0800 61 | %define O_NOCTTY 0x20000 62 | %define O_DIRECTORY 0x100000 63 | %define O_SYMLINK 0x200000 64 | %define O_DSYNC 0x400000 65 | %endif 66 | 67 | ; mmap() and mprotect() flags 68 | %ifidn __OUTPUT_FORMAT__,elf64 69 | ; http://lxr.linux.no/linux+v3.13.5/include/uapi/asm-generic/mman-common.h 70 | %define MAP_SHARED 0x01 71 | %define MAP_PRIVATE 0x02 72 | %define MAP_FIXED 0x10 73 | %define MAP_ANON 0x20 74 | %define PROT_NONE 0b0000 75 | %define PROT_READ 0b0001 76 | %define PROT_WRITE 0b0010 77 | %define PROT_EXEC 0b0100 78 | %define PROT_SEM 0b1000 79 | %define PROT_GROWSDOWN 0x01000000 80 | %define PROT_GROWSUP 0x02000000 81 | %elifidn __OUTPUT_FORMAT__,macho64 82 | ; http://www.opensource.apple.com/source/xnu/xnu-2050.18.24/bsd/sys/mman.h 83 | %define MAP_SHARED 0b000000000001 84 | %define MAP_PRIVATE 0b000000000010 85 | %define MAP_FIXED 0b000000010000 86 | %define MAP_RENAME 0b000000100000 87 | %define MAP_NORESERVE 0b000001000000 88 | %define MAP_INHERIT 0b000010000000 89 | %define MAP_NOEXTEND 0b000100000000 90 | %define MAP_SEMAPHORE 0b001000000000 91 | %define MAP_NOCACHE 0b010000000000 92 | %define MAP_JIT 0b100000000000 93 | %define MAP_FILE 0x0000 94 | %define MAP_ANON 0x1000 95 | %define PROT_NONE 0b000 96 | %define PROT_READ 0b001 97 | %define PROT_WRITE 0b010 98 | %define PROT_EXEC 0b100 99 | %endif 100 | 101 | ; wait() flags are shared 102 | ; http://lxr.linux.no/linux+v3.13.5/include/uapi/linux/wait.h 103 | %define WEXITED 0x04 104 | %define P_ALL 0 105 | %define P_PID 1 106 | %define P_PGID 2 107 | %define ECHILD 10 108 | 109 | default rel 110 | section .text 111 | -------------------------------------------------------------------------------- /rijndael-alg-fst.h: -------------------------------------------------------------------------------- 1 | /** 2 | * rijndael-alg-fst.h 3 | * 4 | * @version 3.0 (December 2000) 5 | * 6 | * Optimised ANSI C code for the Rijndael cipher (now AES) 7 | * 8 | * @author Vincent Rijmen 9 | * @author Antoon Bosselaers 10 | * @author Paulo Barreto 11 | * 12 | * This code is hereby placed in the public domain. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS 15 | * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 16 | * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE 18 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 19 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 20 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 21 | * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 23 | * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 | * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | */ 26 | #ifndef __RIJNDAEL_ALG_FST_H 27 | #define __RIJNDAEL_ALG_FST_H 28 | 29 | #define MAXKC (256/32) 30 | #define MAXKB (256/8) 31 | #define MAXNR 14 32 | 33 | typedef unsigned char u8; 34 | typedef unsigned short u16; 35 | typedef unsigned int u32; 36 | 37 | int rijndaelKeySetupEnc(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], int keyBits); 38 | int rijndaelKeySetupDec(u32 rk[/*4*(Nr + 1)*/], const u8 cipherKey[], int keyBits); 39 | void rijndaelEncrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 pt[16], u8 ct[16]); 40 | void rijndaelDecrypt(const u32 rk[/*4*(Nr + 1)*/], int Nr, const u8 ct[16], u8 pt[16]); 41 | 42 | #ifdef INTERMEDIATE_VALUE_KAT 43 | void rijndaelEncryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], int rounds); 44 | void rijndaelDecryptRound(const u32 rk[/*4*(Nr + 1)*/], int Nr, u8 block[16], int rounds); 45 | #endif /* INTERMEDIATE_VALUE_KAT */ 46 | 47 | #endif /* __RIJNDAEL_ALG_FST_H */ 48 | -------------------------------------------------------------------------------- /sanity.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ./build.sh 3 | (for g in `build/SMHasher --listnames`; do build/SMHasher --test=Sanity $g; done) \ 4 | | tee log.sanity 5 | ./build32.sh 6 | (for g in `build32/SMHasher --listnames`; do build32/SMHasher --test=Sanity $g; done) \ 7 | | tee -a log.sanity 8 | -------------------------------------------------------------------------------- /seahash.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #if defined(_MSC_VER) 6 | # define sea_swap64 _byteswap_uint64 7 | #elif XXH_GCC_VERSION >= 403 8 | # define sea_swap64 __builtin_bswap64 9 | #else 10 | static uint64_t sea_swap64 (uint64_t x) 11 | { 12 | return ((x << 56) & 0xff00000000000000ULL) | 13 | ((x << 40) & 0x00ff000000000000ULL) | 14 | ((x << 24) & 0x0000ff0000000000ULL) | 15 | ((x << 8) & 0x000000ff00000000ULL) | 16 | ((x >> 8) & 0x00000000ff000000ULL) | 17 | ((x >> 24) & 0x0000000000ff0000ULL) | 18 | ((x >> 40) & 0x000000000000ff00ULL) | 19 | ((x >> 56) & 0x00000000000000ffULL); 20 | } 21 | #endif 22 | 23 | static inline uint64_t diffuse(uint64_t val) 24 | { 25 | uint64_t a, b; 26 | val *= 0x6eed0e9da4d94a4fULL; 27 | a = val >> 32; 28 | b = val >> 60; 29 | val ^= a >> b; 30 | val *= 0x6eed0e9da4d94a4fULL; 31 | return val; 32 | } 33 | 34 | uint64_t seahash(const char *key, int len, uint64_t seed) { 35 | uint64_t a, b, c, d; 36 | uint64_t s = seed; 37 | uint64_t *p; 38 | uint8_t pad[8] = {0}; 39 | const uint64_t orig_len = (uint64_t)len; 40 | 41 | a = 0x16f11fe89b0d677cULL ^ s; 42 | b = 0xb480a793d8e6c86cULL; 43 | c = 0x6fe2e5aaf078ebc9ULL; 44 | d = 0x14f994a4c5259381ULL; 45 | 46 | p = (uint64_t *)key; 47 | while (len >= 32) { 48 | a ^= *p++; 49 | b ^= *p++; 50 | c ^= *p++; 51 | d ^= *p++; 52 | a = diffuse(a); 53 | b = diffuse(b); 54 | c = diffuse(c); 55 | d = diffuse(d); 56 | len -= 32; 57 | } 58 | 59 | switch (len) { 60 | // illegal msvc 15 syntax 61 | case 25 ... 31: 62 | a ^= *p++; 63 | b ^= *p++; 64 | c ^= *p++; 65 | memcpy(pad, p, len - 24); 66 | d ^= *(uint64_t *)pad; 67 | a = diffuse(a); 68 | b = diffuse(b); 69 | c = diffuse(c); 70 | d = diffuse(d); 71 | break; 72 | case 24: 73 | a ^= *p++; 74 | b ^= *p++; 75 | c ^= *p++; 76 | a = diffuse(a); 77 | b = diffuse(b); 78 | c = diffuse(c); 79 | break; 80 | case 17 ... 23: 81 | a ^= *p++; 82 | b ^= *p++; 83 | memcpy(pad, p, len - 16); 84 | c ^= *(uint64_t *)pad; 85 | a = diffuse(a); 86 | b = diffuse(b); 87 | c = diffuse(c); 88 | break; 89 | case 16: 90 | a ^= *p++; 91 | b ^= *p++; 92 | a = diffuse(a); 93 | b = diffuse(b); 94 | break; 95 | case 9 ... 15: 96 | a ^= *p++; 97 | memcpy(pad, p, len - 8); 98 | b ^= *(uint64_t *)pad; 99 | a = diffuse(a); 100 | b = diffuse(b); 101 | break; 102 | case 8: 103 | a ^= *p++; 104 | a = diffuse(a); 105 | break; 106 | case 1 ... 7: 107 | memcpy(pad, p, len); 108 | a ^= *(uint64_t *)pad; 109 | a = diffuse(a); 110 | break; 111 | case 0: 112 | break; 113 | default: 114 | assert(0); 115 | } 116 | 117 | a ^= b; 118 | c ^= d; 119 | a ^= c; 120 | a ^= orig_len; 121 | return sea_swap64(diffuse(a)); 122 | } 123 | -------------------------------------------------------------------------------- /sha1.h: -------------------------------------------------------------------------------- 1 | /* public api for steve reid's public domain SHA-1 implementation */ 2 | /* this file is in the public domain */ 3 | 4 | #pragma once 5 | 6 | #include "Platform.h" 7 | 8 | struct SHA1_CTX { 9 | uint32_t state[5]; 10 | uint32_t count[2]; 11 | uint8_t buffer[64]; 12 | }; 13 | 14 | #define SHA1_DIGEST_SIZE 20 15 | 16 | void SHA1_Init(SHA1_CTX *context); 17 | void SHA1_Update(SHA1_CTX *context, const uint8_t *data, const size_t len); 18 | void SHA1_Final(SHA1_CTX *context, uint8_t digest[SHA1_DIGEST_SIZE]); 19 | 20 | void sha1_32a(const void *key, int len, uint32_t seed, void *out); 21 | -------------------------------------------------------------------------------- /sha2/sha224.c: -------------------------------------------------------------------------------- 1 | /* LibTomCrypt, modular cryptographic library -- Tom St Denis 2 | * 3 | * LibTomCrypt is a library that provides various cryptographic 4 | * algorithms in a highly modular and flexible manner. 5 | * 6 | * The library is free for all purposes without any express 7 | * guarantee it works. 8 | */ 9 | /** 10 | @param sha224.c 11 | LTC_SHA-224 new NIST standard based off of LTC_SHA-256 truncated to 224 bits (Tom St Denis) 12 | */ 13 | 14 | #include "../tomcrypt.h" 15 | 16 | #if defined(LTC_SHA224) && defined(LTC_SHA256) 17 | 18 | const struct ltc_hash_descriptor sha224_desc = 19 | { 20 | "sha224", 21 | 10, 22 | 28, 23 | 64, 24 | 25 | /* OID */ 26 | { 2, 16, 840, 1, 101, 3, 4, 2, 4, }, 27 | 9, 28 | 29 | &sha224_init, 30 | &sha256_process, 31 | &sha224_done, 32 | &sha224_test, 33 | NULL 34 | }; 35 | 36 | /* init the sha256 er... sha224 state ;-) */ 37 | /** 38 | Initialize the hash state 39 | @param md The hash state you wish to initialize 40 | @return CRYPT_OK if successful 41 | */ 42 | int sha224_init(hash_state * md) 43 | { 44 | LTC_ARGCHK(md != NULL); 45 | 46 | md->sha256.curlen = 0; 47 | md->sha256.length = 0; 48 | md->sha256.state[0] = 0xc1059ed8UL; 49 | md->sha256.state[1] = 0x367cd507UL; 50 | md->sha256.state[2] = 0x3070dd17UL; 51 | md->sha256.state[3] = 0xf70e5939UL; 52 | md->sha256.state[4] = 0xffc00b31UL; 53 | md->sha256.state[5] = 0x68581511UL; 54 | md->sha256.state[6] = 0x64f98fa7UL; 55 | md->sha256.state[7] = 0xbefa4fa4UL; 56 | return CRYPT_OK; 57 | } 58 | 59 | /** 60 | Terminate the hash to get the digest 61 | @param md The hash state 62 | @param out [out] The destination of the hash (28 bytes) 63 | @return CRYPT_OK if successful 64 | */ 65 | int sha224_done(hash_state * md, unsigned char *out) 66 | { 67 | unsigned char buf[32]; 68 | int err; 69 | 70 | LTC_ARGCHK(md != NULL); 71 | LTC_ARGCHK(out != NULL); 72 | 73 | err = sha256_done(md, buf); 74 | XMEMCPY(out, buf, 28); 75 | #ifdef LTC_CLEAN_STACK 76 | zeromem(buf, sizeof(buf)); 77 | #endif 78 | return err; 79 | } 80 | 81 | /** 82 | Self-test the hash 83 | @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled 84 | */ 85 | int sha224_test(void) 86 | { 87 | #ifndef LTC_TEST 88 | return CRYPT_NOP; 89 | #else 90 | static const struct { 91 | const char *msg; 92 | unsigned char hash[28]; 93 | } tests[] = { 94 | { "abc", 95 | { 0x23, 0x09, 0x7d, 0x22, 0x34, 0x05, 0xd8, 96 | 0x22, 0x86, 0x42, 0xa4, 0x77, 0xbd, 0xa2, 97 | 0x55, 0xb3, 0x2a, 0xad, 0xbc, 0xe4, 0xbd, 98 | 0xa0, 0xb3, 0xf7, 0xe3, 0x6c, 0x9d, 0xa7 } 99 | }, 100 | { "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", 101 | { 0x75, 0x38, 0x8b, 0x16, 0x51, 0x27, 0x76, 102 | 0xcc, 0x5d, 0xba, 0x5d, 0xa1, 0xfd, 0x89, 103 | 0x01, 0x50, 0xb0, 0xc6, 0x45, 0x5c, 0xb4, 104 | 0xf5, 0x8b, 0x19, 0x52, 0x52, 0x25, 0x25 } 105 | }, 106 | }; 107 | 108 | int i; 109 | unsigned char tmp[28]; 110 | hash_state md; 111 | 112 | for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { 113 | sha224_init(&md); 114 | sha224_process(&md, (unsigned char*)tests[i].msg, (unsigned long)strlen(tests[i].msg)); 115 | sha224_done(&md, tmp); 116 | if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "SHA224", i)) { 117 | return CRYPT_FAIL_TESTVECTOR; 118 | } 119 | } 120 | return CRYPT_OK; 121 | #endif 122 | } 123 | 124 | #endif /* defined(LTC_SHA224) && defined(LTC_SHA256) */ 125 | 126 | 127 | /* ref: HEAD -> master, tag: v1.18.2 */ 128 | /* git commit: 7e7eb695d581782f04b24dc444cbfde86af59853 */ 129 | /* commit time: 2018-07-01 22:49:01 +0200 */ 130 | -------------------------------------------------------------------------------- /sha2/sha512_224.c: -------------------------------------------------------------------------------- 1 | /* LibTomCrypt, modular cryptographic library -- Tom St Denis 2 | * 3 | * LibTomCrypt is a library that provides various cryptographic 4 | * algorithms in a highly modular and flexible manner. 5 | * 6 | * The library is free for all purposes without any express 7 | * guarantee it works. 8 | */ 9 | /** 10 | @param sha512_224.c 11 | SHA512/224 hash included in sha512.c 12 | */ 13 | 14 | #include "../tomcrypt.h" 15 | 16 | #if defined(LTC_SHA512_224) && defined(LTC_SHA512) 17 | 18 | const struct ltc_hash_descriptor sha512_224_desc = 19 | { 20 | "sha512-224", 21 | 15, 22 | 28, 23 | 128, 24 | 25 | /* OID */ 26 | { 2, 16, 840, 1, 101, 3, 4, 2, 5, }, 27 | 9, 28 | 29 | &sha512_224_init, 30 | &sha512_process, 31 | &sha512_224_done, 32 | &sha512_224_test, 33 | NULL 34 | }; 35 | 36 | /** 37 | Initialize the hash state 38 | @param md The hash state you wish to initialize 39 | @return CRYPT_OK if successful 40 | */ 41 | int sha512_224_init(hash_state * md) 42 | { 43 | LTC_ARGCHK(md != NULL); 44 | 45 | md->sha512.curlen = 0; 46 | md->sha512.length = 0; 47 | md->sha512.state[0] = CONST64(0x8C3D37C819544DA2); 48 | md->sha512.state[1] = CONST64(0x73E1996689DCD4D6); 49 | md->sha512.state[2] = CONST64(0x1DFAB7AE32FF9C82); 50 | md->sha512.state[3] = CONST64(0x679DD514582F9FCF); 51 | md->sha512.state[4] = CONST64(0x0F6D2B697BD44DA8); 52 | md->sha512.state[5] = CONST64(0x77E36F7304C48942); 53 | md->sha512.state[6] = CONST64(0x3F9D85A86A1D36C8); 54 | md->sha512.state[7] = CONST64(0x1112E6AD91D692A1); 55 | return CRYPT_OK; 56 | } 57 | 58 | /** 59 | Terminate the hash to get the digest 60 | @param md The hash state 61 | @param out [out] The destination of the hash (48 bytes) 62 | @return CRYPT_OK if successful 63 | */ 64 | int sha512_224_done(hash_state * md, unsigned char *out) 65 | { 66 | unsigned char buf[64]; 67 | 68 | LTC_ARGCHK(md != NULL); 69 | LTC_ARGCHK(out != NULL); 70 | 71 | if (md->sha512.curlen >= sizeof(md->sha512.buf)) { 72 | return CRYPT_INVALID_ARG; 73 | } 74 | 75 | sha512_done(md, buf); 76 | XMEMCPY(out, buf, 28); 77 | #ifdef LTC_CLEAN_STACK 78 | zeromem(buf, sizeof(buf)); 79 | #endif 80 | return CRYPT_OK; 81 | } 82 | 83 | /** 84 | Self-test the hash 85 | @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled 86 | */ 87 | int sha512_224_test(void) 88 | { 89 | #ifndef LTC_TEST 90 | return CRYPT_NOP; 91 | #else 92 | static const struct { 93 | const char *msg; 94 | unsigned char hash[28]; 95 | } tests[] = { 96 | { "abc", 97 | { 0x46, 0x34, 0x27, 0x0F, 0x70, 0x7B, 0x6A, 0x54, 98 | 0xDA, 0xAE, 0x75, 0x30, 0x46, 0x08, 0x42, 0xE2, 99 | 0x0E, 0x37, 0xED, 0x26, 0x5C, 0xEE, 0xE9, 0xA4, 100 | 0x3E, 0x89, 0x24, 0xAA } 101 | }, 102 | { "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu", 103 | { 0x23, 0xFE, 0xC5, 0xBB, 0x94, 0xD6, 0x0B, 0x23, 104 | 0x30, 0x81, 0x92, 0x64, 0x0B, 0x0C, 0x45, 0x33, 105 | 0x35, 0xD6, 0x64, 0x73, 0x4F, 0xE4, 0x0E, 0x72, 106 | 0x68, 0x67, 0x4A, 0xF9 } 107 | }, 108 | }; 109 | 110 | int i; 111 | unsigned char tmp[28]; 112 | hash_state md; 113 | 114 | for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { 115 | sha512_224_init(&md); 116 | sha512_224_process(&md, (unsigned char*)tests[i].msg, (unsigned long)strlen(tests[i].msg)); 117 | sha512_224_done(&md, tmp); 118 | if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "SHA512-224", i)) { 119 | return CRYPT_FAIL_TESTVECTOR; 120 | } 121 | } 122 | return CRYPT_OK; 123 | #endif 124 | } 125 | 126 | #endif /* defined(LTC_SHA384) && defined(LTC_SHA512) */ 127 | 128 | /* ref: HEAD -> master, tag: v1.18.2 */ 129 | /* git commit: 7e7eb695d581782f04b24dc444cbfde86af59853 */ 130 | /* commit time: 2018-07-01 22:49:01 +0200 */ 131 | -------------------------------------------------------------------------------- /sha2/sha512_256.c: -------------------------------------------------------------------------------- 1 | /* LibTomCrypt, modular cryptographic library -- Tom St Denis 2 | * 3 | * LibTomCrypt is a library that provides various cryptographic 4 | * algorithms in a highly modular and flexible manner. 5 | * 6 | * The library is free for all purposes without any express 7 | * guarantee it works. 8 | */ 9 | /** 10 | @param sha512_256.c 11 | SHA512/256 hash included in sha512.c 12 | */ 13 | 14 | #include "../tomcrypt.h" 15 | 16 | #if defined(LTC_SHA512_256) && defined(LTC_SHA512) 17 | 18 | const struct ltc_hash_descriptor sha512_256_desc = 19 | { 20 | "sha512-256", 21 | 16, 22 | 32, 23 | 128, 24 | 25 | /* OID */ 26 | { 2, 16, 840, 1, 101, 3, 4, 2, 6, }, 27 | 9, 28 | 29 | &sha512_256_init, 30 | &sha512_process, 31 | &sha512_256_done, 32 | &sha512_256_test, 33 | NULL 34 | }; 35 | 36 | /** 37 | Initialize the hash state 38 | @param md The hash state you wish to initialize 39 | @return CRYPT_OK if successful 40 | */ 41 | int sha512_256_init(hash_state * md) 42 | { 43 | LTC_ARGCHK(md != NULL); 44 | 45 | md->sha512.curlen = 0; 46 | md->sha512.length = 0; 47 | md->sha512.state[0] = CONST64(0x22312194FC2BF72C); 48 | md->sha512.state[1] = CONST64(0x9F555FA3C84C64C2); 49 | md->sha512.state[2] = CONST64(0x2393B86B6F53B151); 50 | md->sha512.state[3] = CONST64(0x963877195940EABD); 51 | md->sha512.state[4] = CONST64(0x96283EE2A88EFFE3); 52 | md->sha512.state[5] = CONST64(0xBE5E1E2553863992); 53 | md->sha512.state[6] = CONST64(0x2B0199FC2C85B8AA); 54 | md->sha512.state[7] = CONST64(0x0EB72DDC81C52CA2); 55 | return CRYPT_OK; 56 | } 57 | 58 | /** 59 | Terminate the hash to get the digest 60 | @param md The hash state 61 | @param out [out] The destination of the hash (48 bytes) 62 | @return CRYPT_OK if successful 63 | */ 64 | int sha512_256_done(hash_state * md, unsigned char *out) 65 | { 66 | unsigned char buf[64]; 67 | 68 | LTC_ARGCHK(md != NULL); 69 | LTC_ARGCHK(out != NULL); 70 | 71 | if (md->sha512.curlen >= sizeof(md->sha512.buf)) { 72 | return CRYPT_INVALID_ARG; 73 | } 74 | 75 | sha512_done(md, buf); 76 | XMEMCPY(out, buf, 32); 77 | #ifdef LTC_CLEAN_STACK 78 | zeromem(buf, sizeof(buf)); 79 | #endif 80 | return CRYPT_OK; 81 | } 82 | 83 | /** 84 | Self-test the hash 85 | @return CRYPT_OK if successful, CRYPT_NOP if self-tests have been disabled 86 | */ 87 | int sha512_256_test(void) 88 | { 89 | #ifndef LTC_TEST 90 | return CRYPT_NOP; 91 | #else 92 | static const struct { 93 | const char *msg; 94 | unsigned char hash[32]; 95 | } tests[] = { 96 | { "abc", 97 | { 0x53, 0x04, 0x8E, 0x26, 0x81, 0x94, 0x1E, 0xF9, 98 | 0x9B, 0x2E, 0x29, 0xB7, 0x6B, 0x4C, 0x7D, 0xAB, 99 | 0xE4, 0xC2, 0xD0, 0xC6, 0x34, 0xFC, 0x6D, 0x46, 100 | 0xE0, 0xE2, 0xF1, 0x31, 0x07, 0xE7, 0xAF, 0x23 } 101 | }, 102 | { "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu", 103 | { 0x39, 0x28, 0xE1, 0x84, 0xFB, 0x86, 0x90, 0xF8, 104 | 0x40, 0xDA, 0x39, 0x88, 0x12, 0x1D, 0x31, 0xBE, 105 | 0x65, 0xCB, 0x9D, 0x3E, 0xF8, 0x3E, 0xE6, 0x14, 106 | 0x6F, 0xEA, 0xC8, 0x61, 0xE1, 0x9B, 0x56, 0x3A } 107 | }, 108 | }; 109 | 110 | int i; 111 | unsigned char tmp[32]; 112 | hash_state md; 113 | 114 | for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) { 115 | sha512_256_init(&md); 116 | sha512_256_process(&md, (unsigned char*)tests[i].msg, (unsigned long)strlen(tests[i].msg)); 117 | sha512_256_done(&md, tmp); 118 | if (compare_testvector(tmp, sizeof(tmp), tests[i].hash, sizeof(tests[i].hash), "SHA512-265", i)) { 119 | return CRYPT_FAIL_TESTVECTOR; 120 | } 121 | } 122 | return CRYPT_OK; 123 | #endif 124 | } 125 | 126 | #endif /* defined(LTC_SHA384) && defined(LTC_SHA512) */ 127 | 128 | /* ref: HEAD -> master, tag: v1.18.2 */ 129 | /* git commit: 7e7eb695d581782f04b24dc444cbfde86af59853 */ 130 | /* commit time: 2018-07-01 22:49:01 +0200 */ 131 | -------------------------------------------------------------------------------- /siphash.c: -------------------------------------------------------------------------------- 1 | #include "siphash.h" 2 | #include "siphash_impl.h" 3 | 4 | #ifndef U8TO64 5 | #define U8TO64 6 | static uint64_t INLINE 7 | U8TO64_LE(const unsigned char *p) { 8 | return *(const uint64_t *)p; 9 | } 10 | #endif 11 | 12 | /* 13 | static void INLINE 14 | U64TO8_LE(unsigned char *p, const uint64_t v) { 15 | *(uint64_t *)p = v; 16 | } 17 | */ 18 | 19 | #define SIPCOMPRESS \ 20 | v0 += v1; v2 += v3; \ 21 | v1 = ROTL64(v1,13); v3 = ROTL64(v3,16); \ 22 | v1 ^= v0; v3 ^= v2; \ 23 | v0 = ROTL64(v0,32); \ 24 | v2 += v1; v0 += v3; \ 25 | v1 = ROTL64(v1,17); v3 = ROTL64(v3,21); \ 26 | v1 ^= v2; v3 ^= v0; \ 27 | v2 = ROTL64(v2,32) 28 | 29 | /* The 64bit 2-4 variant */ 30 | uint64_t 31 | siphash(const unsigned char key[16], const unsigned char *m, size_t len) { 32 | uint64_t v0, v1, v2, v3; 33 | uint64_t mi, k0, k1; 34 | uint64_t last7; 35 | size_t i, blocks; 36 | 37 | k0 = U8TO64_LE(key + 0); 38 | k1 = U8TO64_LE(key + 8); 39 | v0 = k0 ^ 0x736f6d6570736575ull; 40 | v1 = k1 ^ 0x646f72616e646f6dull; 41 | v2 = k0 ^ 0x6c7967656e657261ull; 42 | v3 = k1 ^ 0x7465646279746573ull; 43 | 44 | last7 = (uint64_t)(len & 0xff) << 56; 45 | 46 | for (i = 0, blocks = (len & ~7); i < blocks; i += 8) { 47 | mi = U8TO64_LE(m + i); 48 | v3 ^= mi; 49 | SIPCOMPRESS; /* 2 c rounds */ 50 | SIPCOMPRESS; 51 | v0 ^= mi; 52 | } 53 | 54 | switch (len - blocks) { 55 | case 7: last7 |= (uint64_t)m[i + 6] << 48; 56 | case 6: last7 |= (uint64_t)m[i + 5] << 40; 57 | case 5: last7 |= (uint64_t)m[i + 4] << 32; 58 | case 4: last7 |= (uint64_t)m[i + 3] << 24; 59 | case 3: last7 |= (uint64_t)m[i + 2] << 16; 60 | case 2: last7 |= (uint64_t)m[i + 1] << 8; 61 | case 1: last7 |= (uint64_t)m[i + 0] ; 62 | case 0: 63 | default:; 64 | }; 65 | v3 ^= last7; 66 | SIPCOMPRESS; /* 2 more c rounds */ 67 | SIPCOMPRESS; 68 | v0 ^= last7; 69 | v2 ^= 0xff; 70 | SIPCOMPRESS; /* and 4 final d rounds */ 71 | SIPCOMPRESS; 72 | SIPCOMPRESS; 73 | SIPCOMPRESS; 74 | return v0 ^ v1 ^ v2 ^ v3; 75 | } 76 | 77 | /* The 64bit 1-3 variant */ 78 | uint64_t 79 | siphash13(const unsigned char key[16], const unsigned char *m, size_t len) { 80 | uint64_t v0, v1, v2, v3; 81 | uint64_t mi, k0, k1; 82 | uint64_t last7; 83 | size_t i, blocks; 84 | 85 | k0 = U8TO64_LE(key + 0); 86 | k1 = U8TO64_LE(key + 8); 87 | v0 = k0 ^ 0x736f6d6570736575ull; 88 | v1 = k1 ^ 0x646f72616e646f6dull; 89 | v2 = k0 ^ 0x6c7967656e657261ull; 90 | v3 = k1 ^ 0x7465646279746573ull; 91 | 92 | last7 = (uint64_t)(len & 0xff) << 56; 93 | 94 | for (i = 0, blocks = (len & ~7); i < blocks; i += 8) { 95 | mi = U8TO64_LE(m + i); 96 | v3 ^= mi; 97 | SIPCOMPRESS; /* 1 c round */ 98 | v0 ^= mi; 99 | } 100 | 101 | switch (len - blocks) { 102 | case 7: last7 |= (uint64_t)m[i + 6] << 48; 103 | case 6: last7 |= (uint64_t)m[i + 5] << 40; 104 | case 5: last7 |= (uint64_t)m[i + 4] << 32; 105 | case 4: last7 |= (uint64_t)m[i + 3] << 24; 106 | case 3: last7 |= (uint64_t)m[i + 2] << 16; 107 | case 2: last7 |= (uint64_t)m[i + 1] << 8; 108 | case 1: last7 |= (uint64_t)m[i + 0] ; 109 | case 0: 110 | default:; 111 | }; 112 | v3 ^= last7; 113 | SIPCOMPRESS; /* 1 more c round */ 114 | v0 ^= last7; 115 | v2 ^= 0xff; 116 | SIPCOMPRESS; /* and 3 final d rounds */ 117 | SIPCOMPRESS; 118 | SIPCOMPRESS; 119 | return v0 ^ v1 ^ v2 ^ v3; 120 | } 121 | 122 | #undef sipcompress 123 | 124 | #include "halfsiphash.c" 125 | -------------------------------------------------------------------------------- /siphash.h: -------------------------------------------------------------------------------- 1 | #ifndef SIPHASH_H 2 | #define SIPHASH_H 3 | 4 | #include 5 | #include 6 | 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | 11 | uint64_t siphash(const unsigned char key[16], const unsigned char *m, size_t len); 12 | uint64_t siphash13(const unsigned char key[16], const unsigned char *m, size_t len); 13 | uint32_t halfsiphash(const unsigned char key[16], const unsigned char *m, size_t len); 14 | 15 | #ifdef __cplusplus 16 | } 17 | #endif 18 | 19 | #endif // SIPHASH_H 20 | -------------------------------------------------------------------------------- /siphash_impl.h: -------------------------------------------------------------------------------- 1 | #ifndef SIPHASH_IMPL_H 2 | #define SIPHASH_IMPL_H 3 | 4 | #include "siphash.h" 5 | 6 | #if defined(_MSC_VER) 7 | #include 8 | 9 | #define INLINE __forceinline 10 | #define NOINLINE __declspec(noinline) 11 | #define ROTL64(a,b) _rotl64(a,b) 12 | #define MM16 __declspec(align(16)) 13 | 14 | typedef unsigned int uint32_t; 15 | 16 | #if (_MSC_VER >= 1500) 17 | #define __SSSE3__ 18 | #endif 19 | #if (_MSC_VER > 1200) || defined(_mm_free) 20 | #define __SSE2__ 21 | #endif 22 | #else 23 | #define INLINE __attribute__((always_inline)) 24 | #define NOINLINE __attribute__((noinline)) 25 | #define ROTL64(a,b) (((a)<<(b))|((a)>>(64-b))) 26 | #define MM16 __attribute__((aligned(16))) 27 | #endif 28 | 29 | #if defined(__SSE2__) 30 | #include 31 | typedef __m128i xmmi; 32 | typedef __m64 qmm; 33 | 34 | typedef union packedelem64_t { 35 | uint64_t u[2]; 36 | xmmi v; 37 | } packedelem64; 38 | 39 | typedef union packedelem8_t { 40 | unsigned char u[16]; 41 | xmmi v; 42 | } packedelem8; 43 | #endif 44 | 45 | #if defined(__SSSE3__) 46 | #include 47 | #endif 48 | 49 | #endif // SIPHASH_IMPL_H 50 | 51 | -------------------------------------------------------------------------------- /siphash_sse2.c: -------------------------------------------------------------------------------- 1 | #include "siphash_impl.h" 2 | 3 | /* 0,2,1,3 */ 4 | static const packedelem64 siphash_init[2] = { 5 | {{0x736f6d6570736575ull,0x6c7967656e657261ull}}, 6 | {{0x646f72616e646f6dull,0x7465646279746573ull}} 7 | }; 8 | 9 | static const packedelem64 siphash_final = { 10 | {0x0000000000000000ull,0x00000000000000ffull} 11 | }; 12 | 13 | uint64_t 14 | siphash(const unsigned char key[16], const unsigned char *m, size_t len) { 15 | xmmi k,v02,v20,v13,v11,v33,mi; 16 | uint64_t last7; 17 | uint32_t lo, hi; 18 | size_t i, blocks; 19 | 20 | k = _mm_loadu_si128((xmmi *)(key + 0)); 21 | v02 = siphash_init[0].v; 22 | v13 = siphash_init[1].v; 23 | v02 = _mm_xor_si128(v02, _mm_unpacklo_epi64(k, k)); 24 | v13 = _mm_xor_si128(v13, _mm_unpackhi_epi64(k, k)); 25 | 26 | last7 = (uint64_t)(len & 0xff) << 56; 27 | 28 | #define sipcompress() \ 29 | v11 = v13; \ 30 | v33 = _mm_shuffle_epi32(v13, _MM_SHUFFLE(1,0,3,2)); \ 31 | v11 = _mm_or_si128(_mm_slli_epi64(v11, 13), _mm_srli_epi64(v11, 64-13)); \ 32 | v02 = _mm_add_epi64(v02, v13); \ 33 | v33 = _mm_or_si128(_mm_slli_epi64(v33, 16), _mm_srli_epi64(v33, 64-16)); \ 34 | v13 = _mm_unpacklo_epi64(v11, v33); \ 35 | v13 = _mm_xor_si128(v13, v02); \ 36 | v20 = _mm_shuffle_epi32(v02, _MM_SHUFFLE(0,1,3,2)); \ 37 | v11 = v13; \ 38 | v33 = _mm_shuffle_epi32(v13, _MM_SHUFFLE(1,0,3,2)); \ 39 | v11 = _mm_or_si128(_mm_slli_epi64(v11, 17), _mm_srli_epi64(v11, 64-17)); \ 40 | v20 = _mm_add_epi64(v20, v13); \ 41 | v33 = _mm_or_si128(_mm_slli_epi64(v33, 21), _mm_srli_epi64(v33, 64-21)); \ 42 | v13 = _mm_unpacklo_epi64(v11, v33); \ 43 | v13 = _mm_unpacklo_epi64(v11, v33); \ 44 | v02 = _mm_shuffle_epi32(v20, _MM_SHUFFLE(0,1,3,2)); \ 45 | v13 = _mm_xor_si128(v13, v20); 46 | 47 | for (i = 0, blocks = (len & ~7); i < blocks; i += 8) { 48 | mi = _mm_loadl_epi64((xmmi *)(m + i)); 49 | v13 = _mm_xor_si128(v13, _mm_slli_si128(mi, 8)); 50 | sipcompress() 51 | sipcompress() 52 | v02 = _mm_xor_si128(v02, mi); 53 | } 54 | 55 | switch (len - blocks) { 56 | case 7: last7 |= (uint64_t)m[i + 6] << 48; 57 | case 6: last7 |= (uint64_t)m[i + 5] << 40; 58 | case 5: last7 |= (uint64_t)m[i + 4] << 32; 59 | case 4: last7 |= (uint64_t)m[i + 3] << 24; 60 | case 3: last7 |= (uint64_t)m[i + 2] << 16; 61 | case 2: last7 |= (uint64_t)m[i + 1] << 8; 62 | case 1: last7 |= (uint64_t)m[i + 0] ; 63 | case 0: 64 | default:; 65 | }; 66 | 67 | mi = _mm_unpacklo_epi32(_mm_cvtsi32_si128((uint32_t)last7),_mm_cvtsi32_si128((uint32_t)(last7 >> 32))); 68 | v13 = _mm_xor_si128(v13, _mm_slli_si128(mi, 8)); 69 | sipcompress() 70 | sipcompress() 71 | v02 = _mm_xor_si128(v02, mi); 72 | v02 = _mm_xor_si128(v02, siphash_final.v); 73 | sipcompress() 74 | sipcompress() 75 | sipcompress() 76 | sipcompress() 77 | 78 | v02 = _mm_xor_si128(v02, v13); 79 | v02 = _mm_xor_si128(v02, _mm_shuffle_epi32(v02, _MM_SHUFFLE(1,0,3,2))); 80 | lo = _mm_cvtsi128_si32(v02); 81 | hi = _mm_cvtsi128_si32(_mm_srli_si128(v02, 4)); 82 | return ((uint64_t)hi << 32) | lo; 83 | } 84 | 85 | uint64_t 86 | siphash13(const unsigned char key[16], const unsigned char *m, size_t len) { 87 | xmmi k,v02,v20,v13,v11,v33,mi; 88 | uint64_t last7; 89 | uint32_t lo, hi; 90 | size_t i, blocks; 91 | 92 | k = _mm_loadu_si128((xmmi *)(key + 0)); 93 | v02 = siphash_init[0].v; 94 | v13 = siphash_init[1].v; 95 | v02 = _mm_xor_si128(v02, _mm_unpacklo_epi64(k, k)); 96 | v13 = _mm_xor_si128(v13, _mm_unpackhi_epi64(k, k)); 97 | 98 | last7 = (uint64_t)(len & 0xff) << 56; 99 | 100 | for (i = 0, blocks = (len & ~7); i < blocks; i += 8) { 101 | mi = _mm_loadl_epi64((xmmi *)(m + i)); 102 | v13 = _mm_xor_si128(v13, _mm_slli_si128(mi, 8)); 103 | sipcompress() 104 | v02 = _mm_xor_si128(v02, mi); 105 | } 106 | 107 | switch (len - blocks) { 108 | case 7: last7 |= (uint64_t)m[i + 6] << 48; 109 | case 6: last7 |= (uint64_t)m[i + 5] << 40; 110 | case 5: last7 |= (uint64_t)m[i + 4] << 32; 111 | case 4: last7 |= (uint64_t)m[i + 3] << 24; 112 | case 3: last7 |= (uint64_t)m[i + 2] << 16; 113 | case 2: last7 |= (uint64_t)m[i + 1] << 8; 114 | case 1: last7 |= (uint64_t)m[i + 0] ; 115 | case 0: 116 | default:; 117 | }; 118 | 119 | mi = _mm_unpacklo_epi32(_mm_cvtsi32_si128((uint32_t)last7),_mm_cvtsi32_si128((uint32_t)(last7 >> 32))); 120 | v13 = _mm_xor_si128(v13, _mm_slli_si128(mi, 8)); 121 | sipcompress() 122 | v02 = _mm_xor_si128(v02, mi); 123 | v02 = _mm_xor_si128(v02, siphash_final.v); 124 | sipcompress() 125 | sipcompress() 126 | sipcompress() 127 | 128 | v02 = _mm_xor_si128(v02, v13); 129 | v02 = _mm_xor_si128(v02, _mm_shuffle_epi32(v02, _MM_SHUFFLE(1,0,3,2))); 130 | lo = _mm_cvtsi128_si32(v02); 131 | hi = _mm_cvtsi128_si32(_mm_srli_si128(v02, 4)); 132 | return ((uint64_t)hi << 32) | lo; 133 | } 134 | 135 | #include "halfsiphash.c" 136 | -------------------------------------------------------------------------------- /speed.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use File::Copy 'mv'; 4 | my $ln = @ARGV ? shift : "log.speed"; 5 | 6 | open(my $l, "<", $ln) or die "open $ln $!"; 7 | my ($n,$bulk,$small,$hash); 8 | while (<$l>) { 9 | if (/^--- Testing ([\w_-]+)[\s\n]/) { 10 | if ($n) { 11 | fixupmd($n,$bulk,$small,$hash); 12 | } 13 | $n = $1; $bulk = $small = $hash = undef; 14 | } elsif (/^Average.+ - +([\d\.]+) MiB\/sec/) { 15 | $bulk = sprintf("%12.2f",$1);; 16 | } elsif (/^Average +(\d.*)cycles\/hash/) { 17 | $small = sprintf("%8.2f",$1); 18 | } elsif (/^Running fast HashMapTest: +(\d\S+) cycles\/op \((\d.+) stdv\)/) { 19 | $hash = sprintf("%6.2f (%.0f)", $1, $2); 20 | $hash .= " " if $2 < 10.0; 21 | } elsif (/^Running std HashMapTest: SKIP/) { 22 | $hash = "too slow "; 23 | } 24 | } 25 | fixupmd($n,$bulk,$small,$hash) if $n; 26 | 27 | sub fixupmd { 28 | my ($n,$bulk,$small,$hash) = @_; 29 | open(my $I, "<", "README.md") or die "open README.md $!"; 30 | open(my $O, ">", "README.md.new") or die "open README.md.new $!"; 31 | my $found; 32 | while (<$I>) { 33 | # search for $n in README.md 34 | if (/^\| \[$n\]/) { 35 | # get old values, update line 36 | m/$n\)\s*\| (\s+\d[\d\.]+)\s*\| (\s+\d[\d\.]+)\s+\|\s+(\d[\d\.]+ \(\d+\)|skipped|too slow|-)\s*\|/; 37 | $bulk = $1 unless defined $bulk; 38 | $small = $2 unless defined $small; 39 | $hash = $3 unless defined $hash; 40 | $hash = " - " unless $hash; 41 | my $spc = " " x (38-(2*length($n))); 42 | s/$n\)\s*\|\s+\d[\d\.]+\s*\|\s+\d[\d\.]+\s+\|\s+(\d[\d\.]+ \(\d+\)|skipped|too slow|-)\s*\|/$n)$spc| $bulk | $small | $hash|/; 43 | $found++; 44 | } 45 | print $O $_; 46 | } 47 | if (!$found) { 48 | warn "$n not found in README.md\n"; 49 | } 50 | close $I; 51 | close $O; 52 | if ($found) { 53 | mv ("README.md.new", "README.md"); 54 | fixuphtml ($n,$bulk,$small,$hash); 55 | } 56 | } 57 | 58 | sub fixuphtml { 59 | my ($n,$bulk,$small,$hash) = @_; 60 | open(my $I, "<", "doc/table.html") or die "open doc/table.html $!"; 61 | open(my $O, ">", "doc/table.html.new") or die "open doc/table.html.new $!"; 62 | my $found; 63 | while (<$I>) { 64 | # search for $n in doc/table.html FIXME 65 | if (/$n/) { 66 | # get old values, update line 67 | print $O $_; 68 | 69 | $_ = <$I>; 70 | if (defined $bulk) { 71 | $bulk =~ s/^\s+//; 72 | s{(.+?)}{$bulk}; 73 | } 74 | print $O $_; 75 | 76 | $_ = <$I>; 77 | if (defined $small) { 78 | $small =~ s/^\s+//; 79 | s{(.+?)}{$small}; 80 | } 81 | print $O $_; 82 | 83 | $_ = <$I>; 84 | if (defined $hash) { 85 | $hash =~ s/^\s+//; 86 | s{(.+?)}{$hash}; 87 | } 88 | print $O $_; 89 | $_ = <$I>; 90 | $found++; 91 | } 92 | print $O $_; 93 | } 94 | if (!$found) { 95 | warn "$n not found in doc/table.html\n"; 96 | } 97 | close $I; 98 | close $O; 99 | mv ("doc/table.html.new", "doc/table.html") if $found; 100 | } 101 | -------------------------------------------------------------------------------- /speed.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | LOG=${1:-log.speed} 3 | perl -ne'/^--- Testing ([\w-]+) "/ && print "| [$1](doc/$1)".(" "x(36-(2*length($1)))); /^Average.+ - +([\d\.]+) MiB\/sec/ && printf(" | %12.2f",$1); /^Average +(\d.*)cycles\/hash/ && printf(" | %8.2f",$1); /^Running fast HashMapTest: +(\d\S+) cycles\/op \((\d.+) stdv\)/ && printf(" | %6.2f (%.0f) |\n",$1,$2);' $LOG 4 | -------------------------------------------------------------------------------- /speedall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make -C build 3 | test -f log.speed && mv log.speed log.speed.bak 4 | (for g in `build/SMHasher --listnames`; do build/SMHasher --test=Speed $g; done) | tee log.speed 5 | -------------------------------------------------------------------------------- /split.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | # ./split.pl log.hashes 3 | undef $/; 4 | $_ = <>; 5 | 6 | for $match (split(/-------------------------------------------------------------------------------/)) { 7 | $match =~ m/^--- Testing (\w+)/m; 8 | unless ($seen{$1}) { 9 | open(O, ">doc/$1") and print "doc/$1\n"; 10 | $seen{$1}++; 11 | print O $match; 12 | close(O); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /t1ha/t1ha0_ia32aes_avx.c: -------------------------------------------------------------------------------- 1 | #ifndef T1HA0_DISABLED 2 | #define T1HA_IA32AES_NAME t1ha0_ia32aes_avx 3 | #include "t1ha0_ia32aes_a.h" 4 | #endif /* T1HA0_DISABLED */ 5 | -------------------------------------------------------------------------------- /t1ha/t1ha0_ia32aes_avx2.c: -------------------------------------------------------------------------------- 1 | #ifndef T1HA0_DISABLED 2 | #define T1HA_IA32AES_NAME t1ha0_ia32aes_avx2 3 | #include "t1ha0_ia32aes_b.h" 4 | #endif /* T1HA0_DISABLED */ 5 | -------------------------------------------------------------------------------- /t1ha/t1ha0_ia32aes_noavx.c: -------------------------------------------------------------------------------- 1 | #ifndef T1HA0_DISABLED 2 | #define T1HA_IA32AES_NAME t1ha0_ia32aes_noavx 3 | #include "t1ha0_ia32aes_a.h" 4 | #endif /* T1HA0_DISABLED */ 5 | -------------------------------------------------------------------------------- /testall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | echo "rather use: ./testspeed.sh; testpar.sh, or even testpar1.sh/testpar2.sh on different machines" 3 | echo "./testall.sh will need 1.5-4 days to complete (~20m per hash)" 4 | make -C build 5 | test -f log.hashes && mv log.hashes log.hashes.bak 6 | (for g in `build/SMHasher --listnames`; do build/SMHasher $g | tee doc/$g; done) | tee log.hashes 7 | -------------------------------------------------------------------------------- /testextra.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make -C build 3 | cat extra.lst | parallel -j4 --bar 'build/SMHasher --extra {} >doc/{}' 4 | -------------------------------------------------------------------------------- /testgood-but-poor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make -C build 3 | echo "marked GOOD but FAIL (ignore HashMapTest and LongNeighbors)" 4 | for f in `build/SMHasher --list | perl -alne 'print $F[0] if /GOOD$/'`; do 5 | grep 'FAIL' doc/$f && echo $f; done 6 | 7 | echo "marked GOOD but !!!!" 8 | for f in `build/SMHasher --list | perl -alne 'print $F[0] if /GOOD$/'`; do 9 | grep -q '!!!!' doc/$f && echo $f; done 10 | 11 | echo "marked GOOD but POOR in doc" 12 | for f in `build/SMHasher --list | perl -alne 'print $F[0] if /GOOD$/'`; do 13 | grep 'POOR' doc/$f; done 14 | -------------------------------------------------------------------------------- /testhashmaps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make -C build 3 | mv log.hashmap log.hashmap~; 4 | for g in `build/SMHasher --listnames | tail -n +15` 5 | do build/SMHasher --test=Hashmap $g 6 | done | tee log.hashmap 7 | 8 | ./speed.pl log.hashmap && \ 9 | ./fixupdocspeeds.pl log.hashmap 10 | -------------------------------------------------------------------------------- /testpar-good.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make -C build 3 | build/SMHasher --list | perl -alne 'print $F[0] if /GOOD$/' | \ 4 | parallel -j4 --bar 'build/SMHasher {} >doc/{}' 5 | -------------------------------------------------------------------------------- /testpar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make -C build 3 | mkdir partests 4 | test -n "$@" && r="if /$@/" 5 | build/SMHasher --list|perl -alne"print \$F[0] $r" | \ 6 | parallel -j4 --bar \ 7 | 'build/SMHasher --test=Sparse,Permutation,Cyclic,TwoBytes,DiffDist,Text,Zeroes,Seed,Sanity,Avalanche,BIC,Diff,MomentChi2 {} 2>&1 >partests/{}' 8 | 9 | ./fixupdoctests.pl 10 | -------------------------------------------------------------------------------- /testpar1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make -C build 3 | mkdir partests1 4 | test -n "$@" && r="if /$@/" 5 | build/SMHasher --list|perl -alne"print \$F[0] $r" | \ 6 | parallel -j4 --bar \ 7 | 'build/SMHasher --test=Sparse,Permutation,Cyclic,TwoBytes,DiffDist,Text,Zeroes {} 2>&1 >partests1/{}' 8 | 9 | ./fixupdoctests.pl partests1/* 10 | -------------------------------------------------------------------------------- /testpar2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make -C build 3 | mkdir partests2 4 | test -n "$@" && r="if /$@/" 5 | build/SMHasher --list|perl -alne"print \$F[0] $r" | \ 6 | parallel -j4 --bar \ 7 | 'build/SMHasher --test=Seed,Sanity,Avalanche,BIC,Diff,MomentChi2 {} 2>&1 >partests2/{}' 8 | 9 | ./fixupdoctests.pl partests2/* 10 | -------------------------------------------------------------------------------- /testpoor-but-good.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make -C build 3 | 4 | echo "marked POOR but no FAIL" 5 | for f in `build/SMHasher --list | perl -alne 'print $F[0] if /POOR$/'`; do 6 | grep -q FAIL doc/$f || echo "no FAIL $f"; done 7 | 8 | echo "marked POOR labeled as GOOD in doc" 9 | for f in `build/SMHasher --list | perl -alne 'print $F[0] if /POOR$/'`; do 10 | grep GOOD doc/$f; done 11 | -------------------------------------------------------------------------------- /testspeed.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make -C build 3 | which performance && performance 4 | if [ -z "$@" ]; then 5 | test -f log.speed && mv log.speed log.speed.bak 6 | (for g in `build/SMHasher --listnames`; do 7 | build/SMHasher --test=Speed,Hashmap $g 2>&1; done) | tee log.speed 8 | ./speed.pl && \ 9 | ./fixupdocspeeds.pl 10 | else 11 | (for g in `build/SMHasher --listnames | egrep "$@"`; do 12 | build/SMHasher --test=Speed,Hashmap $g 2>&1; done) | tee "log.speed-$1" 13 | ./speed.pl "log.speed-$1" && \ 14 | ./fixupdocspeeds.pl "log.speed-$1" 15 | fi 16 | -------------------------------------------------------------------------------- /tifuhash.cpp: -------------------------------------------------------------------------------- 1 | // Copyright 2020 Cris Stringfellow 2 | // Licensed under GPL-3.0 3 | // https://github.com/dosyago/floppsy 4 | #include 5 | #include "tifuhash.h" 6 | 7 | #if defined(_MSC_VER) 8 | 9 | #define FORCE_INLINE __forceinline 10 | #ifndef M_PI 11 | #define M_PI 3.14159265358979323846264338327950288 12 | #endif 13 | #ifndef M_E 14 | #define M_E 2.71828182845904523536028747135266250 15 | #endif 16 | 17 | // Other compilers 18 | 19 | #else // defined(_MSC_VER) 20 | 21 | #define FORCE_INLINE inline __attribute__((always_inline)) 22 | 23 | #endif // !defined(_MSC_VER) 24 | 25 | //--------- 26 | // Q function : Continued Egyptian Fraction update function 27 | 28 | FORCE_INLINE void q ( double * state, double key_val, 29 | double numerator, double denominator ) 30 | { 31 | state[0] += numerator / denominator; 32 | state[0] = 1.0 / state[0]; 33 | 34 | state[1] += key_val; 35 | state[1] = numerator / state[1]; 36 | } 37 | 38 | //--------- 39 | // round function : process the message 40 | 41 | FORCE_INLINE void round ( const uint8_t * msg, long len, 42 | double * state ) 43 | { 44 | double numerator = 1.0; 45 | 46 | // Loop 47 | for( long i = 0; i < len; i++ ) { 48 | double val = (double)msg[i]; 49 | double denominator = (1.0 + val + i) / state[1]; 50 | 51 | q( state, val, numerator, denominator ); 52 | 53 | numerator = denominator + 1.0; 54 | } 55 | 56 | state[0] *= M_PI + state[1]; 57 | state[1] *= M_E + state[0]; 58 | } 59 | 60 | //--------- 61 | // setup function : setup the state 62 | 63 | FORCE_INLINE void setup ( double * state, double init = 0 ) 64 | { 65 | state[0] = init != 0 ? pow(init + 1.0/init, 1.0/3) : 3.0; 66 | state[1] = init != 0 ? pow(init + 1.0/init, 1.0/7) : 1.0/7; 67 | } 68 | 69 | //--------- 70 | // tifuhash - tiny fast universal hash ( a.k.a today I f&%@!d up hash ) 71 | // with 64 bit continued egyptian fractions 72 | 73 | void tifuhash_64 ( const void * key, int len, 74 | uint32_t seed, void * out ) 75 | { 76 | const uint8_t * data = (const uint8_t *)key; 77 | uint8_t buf [16]; 78 | double * state = (double*)buf; 79 | uint32_t * state32 = (uint32_t*)buf; 80 | double seed32 = (double)seed; 81 | 82 | setup( state, seed32 ); 83 | round( data, len, state ); 84 | 85 | uint32_t h[2]; 86 | 87 | h[0] = state32[0] + state32[3]; 88 | h[1] = state32[1] + state32[2]; 89 | 90 | ((uint32_t*)out)[0] = h[0]; 91 | ((uint32_t*)out)[1] = h[1]; 92 | } 93 | 94 | -------------------------------------------------------------------------------- /tifuhash.h: -------------------------------------------------------------------------------- 1 | #ifndef _TIFUHASH_H_ 2 | #define _TIFUHASH_H_ 3 | 4 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 5 | 6 | typedef unsigned char uint8_t; 7 | typedef unsigned int uint32_t; 8 | typedef unsigned __int64 uint64_t; 9 | 10 | // Other compilers 11 | 12 | #else // defined(_MSC_VER) 13 | 14 | #include 15 | 16 | #endif // !defined(_MSC_VER) 17 | 18 | void tifuhash_64 ( const void * key, int len, uint32_t seed, void * out ); 19 | 20 | #endif // _TIFUHASH_H_ 21 | -------------------------------------------------------------------------------- /tomcrypt.h: -------------------------------------------------------------------------------- 1 | /* LibTomCrypt, modular cryptographic library -- Tom St Denis 2 | * 3 | * LibTomCrypt is a library that provides various cryptographic 4 | * algorithms in a highly modular and flexible manner. 5 | * 6 | * The library is free for all purposes without any express 7 | * guarantee it works. 8 | */ 9 | 10 | #ifndef TOMCRYPT_H_ 11 | #define TOMCRYPT_H_ 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | /* use configuration data */ 22 | #include "tomcrypt_custom.h" 23 | 24 | #ifdef __cplusplus 25 | extern "C" { 26 | #endif 27 | 28 | /* version */ 29 | #define CRYPT 0x0118 30 | #define SCRYPT "1.18.2" 31 | 32 | /* max size of either a cipher/hash block or symmetric key [largest of the two] */ 33 | #define MAXBLOCKSIZE 128 34 | 35 | #ifndef TAB_SIZE 36 | /* descriptor table size */ 37 | #define TAB_SIZE 32 38 | #endif 39 | 40 | /* error codes [will be expanded in future releases] */ 41 | enum { 42 | CRYPT_OK=0, /* Result OK */ 43 | CRYPT_ERROR, /* Generic Error */ 44 | CRYPT_NOP, /* Not a failure but no operation was performed */ 45 | 46 | CRYPT_INVALID_KEYSIZE, /* Invalid key size given */ 47 | CRYPT_INVALID_ROUNDS, /* Invalid number of rounds */ 48 | CRYPT_FAIL_TESTVECTOR, /* Algorithm failed test vectors */ 49 | 50 | CRYPT_BUFFER_OVERFLOW, /* Not enough space for output */ 51 | CRYPT_INVALID_PACKET, /* Invalid input packet given */ 52 | 53 | CRYPT_INVALID_PRNGSIZE, /* Invalid number of bits for a PRNG */ 54 | CRYPT_ERROR_READPRNG, /* Could not read enough from PRNG */ 55 | 56 | CRYPT_INVALID_CIPHER, /* Invalid cipher specified */ 57 | CRYPT_INVALID_HASH, /* Invalid hash specified */ 58 | CRYPT_INVALID_PRNG, /* Invalid PRNG specified */ 59 | 60 | CRYPT_MEM, /* Out of memory */ 61 | 62 | CRYPT_PK_TYPE_MISMATCH, /* Not equivalent types of PK keys */ 63 | CRYPT_PK_NOT_PRIVATE, /* Requires a private PK key */ 64 | 65 | CRYPT_INVALID_ARG, /* Generic invalid argument */ 66 | CRYPT_FILE_NOTFOUND, /* File Not Found */ 67 | 68 | CRYPT_PK_INVALID_TYPE, /* Invalid type of PK key */ 69 | 70 | CRYPT_OVERFLOW, /* An overflow of a value was detected/prevented */ 71 | 72 | CRYPT_UNUSED1, /* UNUSED1 */ 73 | 74 | CRYPT_INPUT_TOO_LONG, /* The input was longer than expected. */ 75 | 76 | CRYPT_PK_INVALID_SIZE, /* Invalid size input for PK parameters */ 77 | 78 | CRYPT_INVALID_PRIME_SIZE,/* Invalid size of prime requested */ 79 | CRYPT_PK_INVALID_PADDING, /* Invalid padding on input */ 80 | 81 | CRYPT_HASH_OVERFLOW /* Hash applied to too many bits */ 82 | }; 83 | 84 | #include "tomcrypt_cfg.h" 85 | #include "tomcrypt_macros.h" 86 | #include "tomcrypt_cipher.h" 87 | #include "tomcrypt_hash.h" 88 | #include "tomcrypt_mac.h" 89 | #include "tomcrypt_prng.h" 90 | #include "tomcrypt_pk.h" 91 | #include "tomcrypt_math.h" 92 | #include "tomcrypt_misc.h" 93 | #include "tomcrypt_argchk.h" 94 | #include "tomcrypt_pkcs.h" 95 | 96 | #ifdef __cplusplus 97 | } 98 | #endif 99 | 100 | #endif /* TOMCRYPT_H_ */ 101 | 102 | 103 | /* ref: HEAD -> master, tag: v1.18.2 */ 104 | /* git commit: 7e7eb695d581782f04b24dc444cbfde86af59853 */ 105 | /* commit time: 2018-07-01 22:49:01 +0200 */ 106 | -------------------------------------------------------------------------------- /tomcrypt_argchk.h: -------------------------------------------------------------------------------- 1 | /* LibTomCrypt, modular cryptographic library -- Tom St Denis 2 | * 3 | * LibTomCrypt is a library that provides various cryptographic 4 | * algorithms in a highly modular and flexible manner. 5 | * 6 | * The library is free for all purposes without any express 7 | * guarantee it works. 8 | */ 9 | 10 | /* Defines the LTC_ARGCHK macro used within the library */ 11 | /* ARGTYPE is defined in tomcrypt_cfg.h */ 12 | #if ARGTYPE == 0 13 | 14 | #include 15 | 16 | /* this is the default LibTomCrypt macro */ 17 | #if defined(__clang__) || defined(__GNUC_MINOR__) 18 | #define NORETURN __attribute__ ((noreturn)) 19 | #else 20 | #define NORETURN 21 | #endif 22 | 23 | void crypt_argchk(const char *v, const char *s, int d) NORETURN; 24 | #define LTC_ARGCHK(x) do { if (!(x)) { crypt_argchk(#x, __FILE__, __LINE__); } }while(0) 25 | #define LTC_ARGCHKVD(x) do { if (!(x)) { crypt_argchk(#x, __FILE__, __LINE__); } }while(0) 26 | 27 | #elif ARGTYPE == 1 28 | 29 | /* fatal type of error */ 30 | #define LTC_ARGCHK(x) assert((x)) 31 | #define LTC_ARGCHKVD(x) LTC_ARGCHK(x) 32 | 33 | #elif ARGTYPE == 2 34 | 35 | #define LTC_ARGCHK(x) if (!(x)) { fprintf(stderr, "\nwarning: ARGCHK failed at %s:%d\n", __FILE__, __LINE__); } 36 | #define LTC_ARGCHKVD(x) LTC_ARGCHK(x) 37 | 38 | #elif ARGTYPE == 3 39 | 40 | #define LTC_ARGCHK(x) 41 | #define LTC_ARGCHKVD(x) LTC_ARGCHK(x) 42 | 43 | #elif ARGTYPE == 4 44 | 45 | #define LTC_ARGCHK(x) if (!(x)) return CRYPT_INVALID_ARG; 46 | #define LTC_ARGCHKVD(x) if (!(x)) return; 47 | 48 | #endif 49 | 50 | 51 | /* ref: HEAD -> master, tag: v1.18.2 */ 52 | /* git commit: 7e7eb695d581782f04b24dc444cbfde86af59853 */ 53 | /* commit time: 2018-07-01 22:49:01 +0200 */ 54 | -------------------------------------------------------------------------------- /tomcrypt_misc.h: -------------------------------------------------------------------------------- 1 | /* LibTomCrypt, modular cryptographic library -- Tom St Denis 2 | * 3 | * LibTomCrypt is a library that provides various cryptographic 4 | * algorithms in a highly modular and flexible manner. 5 | * 6 | * The library is free for all purposes without any express 7 | * guarantee it works. 8 | */ 9 | 10 | /* ---- LTC_BASE64 Routines ---- */ 11 | #ifdef LTC_BASE64 12 | int base64_encode(const unsigned char *in, unsigned long len, 13 | unsigned char *out, unsigned long *outlen); 14 | 15 | int base64_decode(const unsigned char *in, unsigned long len, 16 | unsigned char *out, unsigned long *outlen); 17 | int base64_strict_decode(const unsigned char *in, unsigned long len, 18 | unsigned char *out, unsigned long *outlen); 19 | #endif 20 | 21 | #ifdef LTC_BASE64_URL 22 | int base64url_encode(const unsigned char *in, unsigned long len, 23 | unsigned char *out, unsigned long *outlen); 24 | int base64url_strict_encode(const unsigned char *in, unsigned long inlen, 25 | unsigned char *out, unsigned long *outlen); 26 | 27 | int base64url_decode(const unsigned char *in, unsigned long len, 28 | unsigned char *out, unsigned long *outlen); 29 | int base64url_strict_decode(const unsigned char *in, unsigned long len, 30 | unsigned char *out, unsigned long *outlen); 31 | #endif 32 | 33 | /* ===> LTC_HKDF -- RFC5869 HMAC-based Key Derivation Function <=== */ 34 | #ifdef LTC_HKDF 35 | 36 | int hkdf_test(void); 37 | 38 | int hkdf_extract(int hash_idx, 39 | const unsigned char *salt, unsigned long saltlen, 40 | const unsigned char *in, unsigned long inlen, 41 | unsigned char *out, unsigned long *outlen); 42 | 43 | int hkdf_expand(int hash_idx, 44 | const unsigned char *info, unsigned long infolen, 45 | const unsigned char *in, unsigned long inlen, 46 | unsigned char *out, unsigned long outlen); 47 | 48 | int hkdf(int hash_idx, 49 | const unsigned char *salt, unsigned long saltlen, 50 | const unsigned char *info, unsigned long infolen, 51 | const unsigned char *in, unsigned long inlen, 52 | unsigned char *out, unsigned long outlen); 53 | 54 | #endif /* LTC_HKDF */ 55 | 56 | /* ---- MEM routines ---- */ 57 | int mem_neq(const void *a, const void *b, size_t len); 58 | //void zeromem(volatile void *dst, size_t len); 59 | void burn_stack(unsigned long len); 60 | 61 | const char *error_to_string(int err); 62 | 63 | extern const char *crypt_build_settings; 64 | 65 | /* ---- HMM ---- */ 66 | int crypt_fsa(void *mp, ...); 67 | 68 | /* ---- Dynamic language support ---- */ 69 | int crypt_get_constant(const char* namein, int *valueout); 70 | int crypt_list_all_constants(char *names_list, unsigned int *names_list_size); 71 | 72 | int crypt_get_size(const char* namein, unsigned int *sizeout); 73 | int crypt_list_all_sizes(char *names_list, unsigned int *names_list_size); 74 | 75 | #ifdef LTM_DESC 76 | void init_LTM(void); 77 | #endif 78 | #ifdef TFM_DESC 79 | void init_TFM(void); 80 | #endif 81 | #ifdef GMP_DESC 82 | void init_GMP(void); 83 | #endif 84 | 85 | #ifdef LTC_ADLER32 86 | typedef struct adler32_state_s 87 | { 88 | unsigned short s[2]; 89 | } adler32_state; 90 | 91 | void adler32_init(adler32_state *ctx); 92 | void adler32_update(adler32_state *ctx, const unsigned char *input, unsigned long length); 93 | void adler32_finish(adler32_state *ctx, void *hash, unsigned long size); 94 | int adler32_test(void); 95 | #endif 96 | 97 | #ifdef LTC_CRC32 98 | typedef struct crc32_state_s 99 | { 100 | ulong32 crc; 101 | } crc32_state; 102 | 103 | void crc32_init(crc32_state *ctx); 104 | void crc32_update(crc32_state *ctx, const unsigned char *input, unsigned long length); 105 | void crc32_finish(crc32_state *ctx, void *hash, unsigned long size); 106 | int crc32_test(void); 107 | #endif 108 | 109 | int compare_testvector(const void* is, const unsigned long is_len, const void* should, const unsigned long should_len, const char* what, int which); 110 | 111 | /* ref: HEAD -> master, tag: v1.18.2 */ 112 | /* git commit: 7e7eb695d581782f04b24dc444cbfde86af59853 */ 113 | /* commit time: 2018-07-01 22:49:01 +0200 */ 114 | -------------------------------------------------------------------------------- /tsip.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | static uint64_t inline U8TO64_LE(const unsigned char *p) { 4 | return *(const uint64_t *)p; 5 | } 6 | 7 | static uint64_t rotl64(uint64_t x, int k) { 8 | return (((x) << (k)) | ((x) >> (64 - k))); 9 | } 10 | 11 | uint64_t tsip(const unsigned char *seed, const unsigned char *m, uint64_t len) { 12 | 13 | uint64_t v0, v1; 14 | uint64_t mi, k0, k1; 15 | uint64_t last7; 16 | 17 | k0 = U8TO64_LE(seed); 18 | k1 = U8TO64_LE(seed + 8); 19 | 20 | v0 = k0 ^ 0x736f6d6570736575ull; 21 | v1 = k1 ^ 0x646f72616e646f6dull; 22 | 23 | last7 = (uint64_t)(len & 0xff) << 56; 24 | 25 | #define sipcompress() \ 26 | do { \ 27 | v0 += v1; \ 28 | v1 = rotl64(v1, 13) ^ v0; \ 29 | v0 = rotl64(v0, 35) + v1; \ 30 | v1 = rotl64(v1, 17) ^ v0; \ 31 | v0 = rotl64(v0, 21); \ 32 | } while (0) 33 | 34 | const unsigned char *end = m + (len & ~7); 35 | 36 | while (m < end) { 37 | mi = U8TO64_LE(m); 38 | v1 ^= mi; 39 | sipcompress(); 40 | v0 ^= mi; 41 | m += 8; 42 | } 43 | 44 | switch (len & 7) { 45 | case 7: 46 | last7 |= (uint64_t)m[6] << 48; 47 | case 6: 48 | last7 |= (uint64_t)m[5] << 40; 49 | case 5: 50 | last7 |= (uint64_t)m[4] << 32; 51 | case 4: 52 | last7 |= (uint64_t)m[3] << 24; 53 | case 3: 54 | last7 |= (uint64_t)m[2] << 16; 55 | case 2: 56 | last7 |= (uint64_t)m[1] << 8; 57 | case 1: 58 | last7 |= (uint64_t)m[0]; 59 | case 0: 60 | default:; 61 | }; 62 | 63 | v1 ^= last7; 64 | sipcompress(); 65 | v0 ^= last7; 66 | 67 | // finalization 68 | v1 ^= 0xff; 69 | sipcompress(); 70 | v1 = rotl64(v1, 32); 71 | sipcompress(); 72 | v1 = rotl64(v1, 32); 73 | 74 | return v0 ^ v1; 75 | } 76 | -------------------------------------------------------------------------------- /xxhash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xxHash - Fast Hash algorithm 3 | * Copyright (C) 2012-2016, Yann Collet 4 | * 5 | * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are 9 | * met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above 14 | * copyright notice, this list of conditions and the following disclaimer 15 | * in the documentation and/or other materials provided with the 16 | * distribution. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | * 30 | * You can contact the author at : 31 | * - xxHash homepage: http://www.xxhash.com 32 | * - xxHash source repository : https://github.com/Cyan4973/xxHash 33 | */ 34 | 35 | 36 | /* xxhash.c instantiates functions defined in xxhash.h 37 | */ 38 | 39 | #define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ 40 | #define XXH_IMPLEMENTATION /* access definitions */ 41 | 42 | #include "xxhash.h" 43 | --------------------------------------------------------------------------------