├── hash_comparison.png ├── LICENSE ├── testvec.c ├── README.md └── komihash.h /hash_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/avaneev/komihash/HEAD/hash_comparison.png -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021-2025 Aleksey Vaneev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /testvec.c: -------------------------------------------------------------------------------- 1 | /** 2 | * testvec.c version 5.9 3 | * 4 | * The program that lists test vectors and their hash values, for the current 5 | * version of komihash. Also prints initial outputs of the `komirand` PRNG. 6 | * 7 | * Description is available at https://github.com/avaneev/komihash 8 | * 9 | * License 10 | * 11 | * Copyright (c) 2021-2023 Aleksey Vaneev 12 | * 13 | * Permission is hereby granted, free of charge, to any person obtaining a 14 | * copy of this software and associated documentation files (the "Software"), 15 | * to deal in the Software without restriction, including without limitation 16 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 17 | * and/or sell copies of the Software, and to permit persons to whom the 18 | * Software is furnished to do so, subject to the following conditions: 19 | * 20 | * The above copyright notice and this permission notice shall be included in 21 | * all copies or substantial portions of the Software. 22 | * 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 25 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 26 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 27 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 28 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 29 | * DEALINGS IN THE SOFTWARE. 30 | */ 31 | 32 | #include 33 | #include "komihash.h" 34 | 35 | int main() 36 | { 37 | #define seedc 3 38 | const uint64_t seeds[ seedc ] = { 0, 0x0123456789ABCDEF, 256 }; 39 | 40 | #define strc 5 41 | const char* const strs[ strc ] = { 42 | "This is a 32-byte testing string", 43 | "The cat is out of the bag", 44 | "A 16-byte string", 45 | "The new string", 46 | "7 chars" 47 | }; 48 | 49 | #define bulkc 17 50 | const int bulks[ bulkc ] = { 3, 6, 8, 12, 20, 31, 32, 40, 47, 48, 56, 64, 51 | 72, 80, 112, 132, 256 }; 52 | 53 | #define bulkbc 256 54 | uint8_t bulkbuf[ bulkbc ]; 55 | int i; 56 | 57 | for( i = 0; i < bulkbc; i++ ) 58 | { 59 | bulkbuf[ i ] = (uint8_t) i; 60 | } 61 | 62 | int j; 63 | 64 | for( j = 0; j < seedc; j++ ) 65 | { 66 | printf( "komihash UseSeed = 0x%016llx:\n", seeds[ j ]); 67 | 68 | for( i = 0; i < strc; i++ ) 69 | { 70 | const char* const s = strs[ i ]; 71 | const size_t sl = strlen( s ); 72 | 73 | printf( "\"%s\" = 0x%016llx\n", s, 74 | komihash( s, sl, seeds[ j ])); 75 | } 76 | 77 | for( i = 0; i < bulkc; i++ ) 78 | { 79 | printf( "bulk(%i) = 0x%016llx\n", bulks[ i ], 80 | komihash( bulkbuf, bulks[ i ], seeds[ j ])); 81 | } 82 | 83 | printf( "\n" ); 84 | } 85 | 86 | for( j = 0; j < seedc; j++ ) 87 | { 88 | printf( "komirand Seed1/Seed2 = 0x%016llx:\n", seeds[ j ]); 89 | 90 | uint64_t Seed1 = seeds[ j ]; 91 | uint64_t Seed2 = seeds[ j ]; 92 | 93 | for( i = 0; i < 12; i++ ) 94 | { 95 | printf( "0x%016llx\n", komirand( &Seed1, &Seed2 )); 96 | } 97 | 98 | printf( "\n" ); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # KOMIHASH - Very Fast Hash Function (in C/C++) 2 | 3 | ## Introduction 4 | 5 | The `komihash()` function available in the `komihash.h` file implements a very 6 | fast 64-bit hash function, mainly designed for hash-table, hash-map, and 7 | bloom-filter uses; produces identical hashes on both big- and little-endian 8 | systems. Suitable for file and large data hashing (checksums). Function's code 9 | is portable, cross-platform, scalar, zero-allocation, is header-only, 10 | inlineable C (C++ compatible). 11 | 12 | This function features both a high large-block hashing performance (27 GB/s 13 | on Ryzen 3700X) and a high hashing throughput for small strings/messages 14 | (about 8 cycles/hash for 0-15-byte strings, hashed repeatedly). Performance on 15 | 32-bit systems is, however, quite low. Also, large-block hashing performance 16 | on big-endian systems may be 20% lower due to the need of byte-swapping (can 17 | be switched off with a define). 18 | 19 | Technically, `komihash` is close to the class of hash functions like `wyhash` 20 | and `CircleHash`, which are, in turn, close to the `lehmer64` PRNG. However, 21 | `komihash` is structurally different to them in that it accumulates the full 22 | 128-bit multiplication result, without folding into a single 64-bit state 23 | variable. Thus `komihash` does not lose differentiation between consecutive 24 | states while others may. 25 | 26 | Another important difference in `komihash` is that it parses the input message 27 | without overlaps. While overlaps allow a function to have fewer code branches, 28 | they are considered "non-ideal", potentially causing collisions and seed value 29 | flaws. Beside that, `komihash` features superior seed value handling and 30 | Perlin Noise hashing. 31 | 32 | An aspect worth noting, important to some users, is that `komihash` at its 33 | base uses a simple mathematical construct, and uses no author-intended nor 34 | author-fabricated information. The base state of the function is equal to the 35 | first mantissa bits of PI, and can be changed to any uniformly-random values. 36 | This means that statistical properties (collision resistance) of `komihash` 37 | are not based on manual fine-tuning, but are a product of the mathematical 38 | construct. 39 | 40 | Note that this function is not cryptographically-secure: in open systems, and 41 | within any server-side internal structures, it should only be used with a 42 | secret seed, to minimize the chance of a collision attack (hash flooding). 43 | However, when the default seed is used (0), this further reduces function's 44 | overhead by 1-2 cycles/hash (compiler-dependent). 45 | 46 | This function passes all [SMHasher](https://github.com/rurban/smhasher) and 47 | [SMHasher3](https://gitlab.com/fwojcik/smhasher3/-/tree/main/results) tests. 48 | The function was also tested with the [xxHash collision tester](https://github.com/Cyan4973/xxHash/tree/dev/tests/collisions) 49 | at various settings, with the collision statistics satisfying the expectations. 50 | The performance (expressed in cycles/byte) of this hash function on various 51 | platforms is best evaluated at the 52 | [ECRYPT/eBASH project](https://bench.cr.yp.to/results-hash.html). Thanks to 53 | [Daniel J. Bernstein](https://en.wikipedia.org/wiki/Daniel_J._Bernstein) for 54 | maintaining the benchmark. 55 | 56 | This function and the source code (which conforms to 57 | [ISO C99](https://en.wikipedia.org/wiki/C99)) were quality-tested on: 58 | Clang, GCC, MSVC, and Intel C++ compilers; x86, x86-64 (Intel, AMD), and 59 | AArch64 (Apple Silicon) architectures; Windows 10, AlmaLinux 9.3, and 60 | macOS 13.3. Full C++ compliance is enabled conditionally and automatically 61 | when the source code is compiled with a C++ compiler. 62 | 63 | ## Usage 64 | 65 | ```c 66 | #include 67 | #include "komihash.h" 68 | 69 | int main(void) 70 | { 71 | const char s1[] = "This is a test of komihash."; 72 | const char s2[] = "7 chars"; 73 | 74 | printf( "%016llx\n", komihash( s1, strlen( s1 ), 0 )); // 5b13177fc68b4f96 75 | printf( "%016llx\n", komihash( s2, strlen( s2 ), 0 )); // 2c514f6e5dcb11cb 76 | } 77 | ``` 78 | 79 | ## Discrete-Incremental Hashing 80 | 81 | A correct way to hash an array of independent values, and which does not 82 | require pre-buffering, is to pass previous hash value as a seed value. This 83 | method may be as fast or faster than pre-buffering, especially if lengths of 84 | values in the array are not small. An additional 1-2 cycles/hash advantage is 85 | obtained if fixed-size values are being hashed incrementally (due to 86 | compiler's branching optimization). In most cases, incremental hashing of even 87 | a few 2-8-byte values may be faster than using pre-buffering if the overall 88 | input length is not known in advance. 89 | 90 | ```c 91 | uint64_t HashVal = komihash( &val1, sizeof( val1 ), Seed ); 92 | HashVal = komihash( &val2, sizeof( val2 ), HashVal ); 93 | ... 94 | HashVal = komihash( &valN, sizeof( valN ), HashVal ); 95 | ``` 96 | 97 | Note that this approach is not the same as "streamed" hashing since this 98 | approach implicitly encodes the length of each independent value. Such kind of 99 | hashing can be beneficial when a database record is being hashed, when it is 100 | necessary to separate fields by means of encoding their lengths. 101 | 102 | Discrete-incremental hashing of nested structures requires a "hash value 103 | stack" where the current hash value is pushed into it upon each nesting, the 104 | nested level starts at hash value 0, and the resulting value is hashed with a 105 | popped previous hash value upon exiting the nesting level. 106 | 107 | ## Streamed Hashing 108 | 109 | The `komihash.h` file also features a fast continuously-streamed 110 | implementation of the `komihash` hash function. Streamed hashing expects any 111 | number of `update` calls inbetween the `init` and `final` calls: 112 | 113 | ```c 114 | komihash_stream_t ctx; 115 | komihash_stream_init( &ctx, UseSeed ); 116 | 117 | komihash_stream_update( &ctx, &val1, sizeof( val1 )); 118 | komihash_stream_update( &ctx, &val2, sizeof( val2 )); 119 | ... 120 | komihash_stream_update( &ctx, &valN, sizeof( valN )); 121 | 122 | uint64_t Hash = komihash_stream_final( &ctx ); 123 | ``` 124 | 125 | Since the `final` function is non-destructive to the context structure, the 126 | function can be used to obtain intermediate "incremental" hashes of the data 127 | stream being hashed, and the hashing can then be resumed. 128 | 129 | The hash value produced via streamed hashing can be used in the 130 | discrete-incremental hashing outlined above (e.g., for files and blobs). 131 | 132 | You may also consider using [PRVHASH64S](https://github.com/avaneev/prvhash) 133 | which provides 8.5 GB/s hashing throughput on Ryzen 3700X, and is able to 134 | produce a hash value of any required bit-size. 135 | 136 | ## Ports 137 | 138 | * [C++, vcpkg](https://vcpkg.link/ports/komihash) 139 | * [C3, by NotsoanoNimus](https://github.com/c3lang/c3c/blob/master/lib/std/hash/komi.c3) 140 | * [Crystal, by Coral Pink](https://codeberg.org/coralpink/komihash.cr) 141 | * [Java, by Dynatrace](https://github.com/dynatrace-oss/hash4j) 142 | * [LUA, by rangercyh](https://github.com/rangercyh/lua-komihash) 143 | * [.NET, by TommasoBelluzzo](https://www.nuget.org/packages/FastHashes/) 144 | * [Perl, by scottchiefbaker](https://github.com/scottchiefbaker/perl-Crypt-Komihash) 145 | * [Rust, by thynson](https://crates.io/crates/komihash) 146 | 147 | ## Customizing C++ namespace 148 | 149 | In C++ environments where it is undesirable to export `komihash` symbols into 150 | the global namespace, the `KOMIHASH_NS_CUSTOM` macro can be defined 151 | externally: 152 | 153 | ```c++ 154 | #define KOMIHASH_NS_CUSTOM komihash 155 | #include "komihash.h" 156 | ``` 157 | 158 | Similarly, `komihash` symbols can be placed into any other custom namespace 159 | (e.g., a namespace with hash functions): 160 | 161 | ```c++ 162 | #define KOMIHASH_NS_CUSTOM my_hashes 163 | #include "komihash.h" 164 | ``` 165 | 166 | This way, `komihash` functions can be referenced like 167 | `my_hashes::komihash(...)`. Note that since all `komihash` functions have the 168 | `static` specifier, there can be no ABI conflicts, even if the `komihash.h` 169 | header is included in unrelated, mixed C/C++, compilation units. 170 | 171 | ## Comparisons 172 | 173 | These are the performance comparisons made and used by the author during the 174 | development of `komihash`, on different compilers and platforms. 175 | 176 | 1. LLVM clang-cl 18.1.8 x86-64, Windows 10, Ryzen 3700X (Zen2), 4.2 GHz. 177 | Compiler options: `/Ox -msse2`. 178 | 2. LLVM clang-cl 18.1.8 x86-64, Windows 10, Ryzen 3700X (Zen2), 4.2 GHz. 179 | Compiler options: `/Ox -mavx2`. 180 | 3. ICC 19.0 x86-64, Windows 10, Ryzen 3700X (Zen2), 4.2 GHz. 181 | Compiler options: `/O3 /QxSSE2`. 182 | 4. LLVM clang 18.1.8 x86-64, AlmaLinux 9.3, Xeon E-2386G (RocketLake), 5.1 GHz. 183 | Compiler options: `-O3 -mavx2`. 184 | 5. GCC 11.4.1 x86-64, AlmaLinux 9.3, Xeon E-2386G (RocketLake), 5.1 GHz. 185 | Compiler options: `-O3 -msse2`. 186 | 6. GCC 11.4.1 x86-64, AlmaLinux 9.3, Xeon E-2386G (RocketLake), 5.1 GHz. 187 | Compiler options: `-O3 -mavx2`. 188 | 7. LLVM clang-cl 18.1.8 x86-64, Windows 10, Core i7-7700K (KabyLake), 4.5 GHz. 189 | Compiler options: `/Ox -mavx2`. 190 | 8. ICC 19.0 x86-64, Windows 10, Core i7-7700K (KabyLake), 4.5 GHz. 191 | Compiler options: `/O3 /QxSSE2`. 192 | 9. Apple clang 15.0.0 arm64, macOS 13.3.2, Apple M1, 3.5 GHz. 193 | Compiler options: `-O3`. 194 | 10. LLVM clang-cl 18.1.8 x86-64, Windows 11, Ryzen 9950X (Zen5), 5.7 GHz. 195 | Compiler options: `/Ox -msse2`. 196 | 197 | |Platform |1 |1 |1 |2 |2 |2 |3 |3 |3 |4 |4 |4 |5 |5 |5 |6 |6 |6 |7 |7 |7 |8 |8 |8 |9 |9 |9 |10 |10 |10 | 198 | |-----------------|-----------------|---------------|---------------|-------|-----|----|-------|-----|----|-------|-----|----|-------|-----|----|-------|-----|----|-------|-----|----|-------|-----|----|-------|-----|----|-------|-----|----| 199 | |Hash function |`0-15b, cycles/h`|8-28b, cycles/h|bulk, GB/s |`0-15b`|8-28b|bulk|`0-15b`|8-28b|bulk|`0-15b`|8-28b|bulk|`0-15b`|8-28b|bulk|`0-15b`|8-28b|bulk|`0-15b`|8-28b|bulk|`0-15b`|8-28b|bulk|`0-15b`|8-28b|bulk|`0-15b`|8-28b|bulk| 200 | |**komihash 5.28**|`9.7` |11.2 |27.1 |`9.7` |11.2 |27.1|`11.8` |13.9 |23.2|`9.7` |11.3 |31.7|`10.0` |11.4 |31.0|`10.0` |11.4 |31.0|`12.0` |13.2 |22.8|`15.0` |18.0 |19.4|`8.1` |7.9 |23.6|`7.0` |8.0 |42.7| 201 | |wyhash_final4 |`14.5` |18.2 |29.3 |`14.7` |18.2 |29.3|`25.9` |32.9 |12.5|`16.8` |21.6 |34.6|`17.2` |23.1 |35.3|`17.3` |23.2 |35.5|`15.5` |20.4 |29.8|`21.1` |26.1 |19.4|`7.9` |8.1 |26.1|`13.9` |18.5 |41.7| 202 | |XXH3_64 0.8.0 |`15.5` |28.8 |30.0 |`15.5` |28.7 |61.8|`21.8` |27.2 |29.6|`18.5` |25.6 |68.3|`19.2` |25.3 |33.8|`19.7` |26.3 |63.6|`18.4` |23.0 |48.3|`19.9` |25.8 |28.0|`8.2` |8.2 |30.5|`15.4` |31.0 |50.3| 203 | |XXH64 0.8.0 |`12.5` |17.5 |17.2 |`12.5` |17.5 |17.3|`24.3` |36.6 |8.9 |`10.5` |14.2 |20.1|`11.2` |14.6 |20.1|`11.2` |14.6 |20.0|`13.2` |17.3 |17.7|`18.8` |24.7 |16.0|`8.8` |10.4 |14.5|`9.1` |12.7 |31.4| 204 | |(overhead) |1.8 |1.8 |0 |1.8 |1.8 |0 |1.9 |1.9 |0 |3.9 |3.9 |0 |2.8 |2.8 |0 |3.6 |3.6 |0 |5.5 |5.5 |0 |5.9 |5.9 |0 |2.0 |2.0 |0 |1.0 |1.0 |0 | 205 | 206 | Notes: `XXH3_64` is unseeded (seeded variant is 1 cycle/h higher). `bulk` is 207 | 256000 bytes: this means it is mainly a cache-bound performance, not 208 | reflective of high-load situations. `GB/s` should not be misinterpreted as 209 | `GiB/s`. `cycles/h` means `processor clock ticks per hash value`, including 210 | overhead. Measurement error is approximately 3%. 211 | 212 | ### Averages over all measurements (overhead excluded) 213 | 214 | |Hash function |0-15b, cycles/h|8-28b, cycles/h| 215 | |---- |---- |---- | 216 | |**komihash 5.28**|**7.3** |**8.7** | 217 | |komihash 5.10 |8.2 |9.8 | 218 | |komihash 4.5 |9.5 |11.4 | 219 | |komihash 4.3 |10.4 |12.1 | 220 | |komihash 3.6 |10.9 |15.4 | 221 | |komihash 2.8 |11.8 |16.7 | 222 | |wyhash_final4 |13.5 |18.0 | 223 | |XXH3_64 0.8.0 |14.2 |22.0 | 224 | |XXH64 0.8.0 |10.2 |15.0 | 225 | 226 | This is the throughput comparison of hash functions on Ryzen 3700X. The used 227 | measurement method actually measures hash function's "latencied throughput", 228 | or sequential hashing, due to the use of the "volatile" variable specifiers 229 | and result accumulation. 230 | 231 | 232 | 233 | The following method was used to obtain the `cycles/h` values. Note that this 234 | method measures a "raw" throughput, when processor's branch predictor tunes to 235 | a specific message length and a specific memory address. Practical performance 236 | depends on actual statistics of strings (messages) being hashed, including 237 | memory access patterns. Note that particular hash functions may "over-favor" 238 | specific message lengths. In this respect, `komihash` does not "favor" any 239 | specific length, thus it may be more universal. Throughput aside, hashing 240 | quality is also an important factor since it drives a hash-map's creation and 241 | subsequent accesses. This, and many other synthetic hash function tests should 242 | be taken with a grain of salt. Only an actual use-case can reveal which hash 243 | function is preferrable. 244 | 245 | ```c++ 246 | const uint64_t rc = 1ULL << 26; 247 | const int minl = 8; const int maxl = 28; 248 | volatile uint64_t msg[ 8 ] = { 0 }; 249 | uint64_t v = 0; 250 | 251 | const TClock t1( CSystem :: getClock() ); 252 | 253 | for( int k = minl; k <= maxl; k++ ) 254 | { 255 | volatile size_t msgl = k; 256 | volatile uint64_t sd = k + 1; 257 | 258 | for( uint64_t i = 0; i < rc; i++ ) 259 | { 260 | v ^= komihash( (uint8_t*) &msg, msgl, sd ); 261 | // v ^= wyhash( (uint8_t*) &msg, msgl, sd, _wyp ); 262 | // v ^= XXH3_64bits( (uint8_t*) &msg, msgl ); 263 | // v ^= msg[ 0 ]; // Used to estimate the overhead. 264 | msg[ 0 ]++; 265 | } 266 | } 267 | 268 | printf( "%016llx\n", v ); 269 | printf( "%.1f\n", CSystem :: getClockDiffSec( t1 ) * 4.2e9 / 270 | ( rc * ( maxl - minl + 1 ))); // 5.1 on Xeon, 4.5 on i7700K, 3.5 on M1 271 | ``` 272 | 273 | ## Discussion 274 | 275 | Does `komihash` feature identity hashing? No, it does not. If you are using 276 | fixed-size keys, it is by all means advisable to use direct key values, and 277 | not the ones obtained via a hash function. Adding the identity hashing feature 278 | to any hash function increases the overhead. 279 | 280 | You may wonder, why `komihash` does not include a quite common `^MsgLen` XOR 281 | instruction at some place in the code? The main reason is that due to the way 282 | `komihash` parses the input message such instruction is not necessary. Another 283 | reason is that for a non-cryptographic hash function such instruction provides 284 | no additional security: while it may seem like such instruction protects from 285 | simple "state XORing" collision attacks, in practice it offers no protection, 286 | if one considers how powerful [SAT solvers](https://github.com/pysathq/pysat) 287 | are: in less than a second they can "forge" a preimage which produces a 288 | required hash value. It is also important to note that in such "fast" hash 289 | functions like `komihash` the input message has complete control over the 290 | state variables and the result. 291 | 292 | Is 128-bit version of this hash function planned? Most probably, no, it is 293 | not. While such version may be reasonable for data structure compatibility 294 | reasons, there is no much practical sense to use 128-bit hashes at a local 295 | level: a reliable 64-bit hash allows one to have 2.1 billion diverse binary 296 | objects (e.g. files in a file system, or entries in a hash-map) without 297 | collisions, on average. On the other hand, on a worldwide scale, having 298 | 128-bit hashes is clearly not enough considering the number of existing 299 | digital devices and the number of diverse binary objects (e.g. files, records 300 | in databases) on each of them. 301 | 302 | An opinion on the "bulk" performance of "fast" hash functions: in most 303 | practical situations, when processor's total memory bandwidth is limited to 304 | e.g. 41 GB/s, a "bulk" single-threaded hashing performance on the order of 305 | 30 GB/s is excessive considering memory bandwidth has to be spread over 306 | multiple cores. So, practically, such "fast" hash function, working on a 307 | high-load 8-core server, rarely receives more than 8 GB/s of bandwidth. 308 | Another factor worth a mention is that a server rarely has more than 10 Gb/s 309 | network connectivity, thus further reducing practical hashing performance of 310 | incoming data. The same applies to disk system's throughput, if on-disk data 311 | is not yet in memory. 312 | 313 | ## KOMIRAND 314 | 315 | The `komirand()` function available in the `komihash.h` file implements a 316 | simple, but reliable, self-starting, and fast (`0.62` cycles/byte) 64-bit 317 | pseudo-random number generator (PRNG) with a `2^64` period. It is based on the 318 | same mathematical construct as the `komihash` hash function. `komirand` 319 | passes `PractRand` tests (at least up to 1 TB length, at the default settings) 320 | and `SmokeRand` tests (full setting). 321 | 322 | The PRNG has a 128-bit state which is divided into two 64-bit unsigned integer 323 | variables ($s_{1}$ and $s_{2}$). 324 | 325 | $$ m_{128}=s_{1} * s_{2} $$ 326 | 327 | $$ s_{2}'=(s_{2}+\lfloor m_{128} / 2^{64} \rfloor +C) \mod 2^{64} $$ 328 | 329 | $$ s_{1}'=(m_{128} \mod 2^{64}) \oplus s_{2}' $$ 330 | 331 | $C$ is any optional 64-bit constant (to facilitate PRNG auto-start from 332 | $m_{128}=0$ state), but can be zero if such auto-start is not needed. 333 | $s_{1}'$ is used as PRNG output. 334 | 335 | This construct can be scaled to any even-sized registers beside 64-bit ones 336 | (e.g., 32, 48) - it is invariant to the register size. The $C$ used in 337 | `komirand` (`0xAAAA...`) is a good choice as such constant carries no bit-wise 338 | spectral information, and its influence on the statistics and modes is 339 | minimal. 340 | 341 | For hashing, the following expression for $m_{128}$ is used: 342 | 343 | $$ m_{128}=(s_{1} \oplus x_{1}) * (s_{2} \oplus x_{2}) $$ 344 | 345 | Where $x_{1}$ and $x_{2}$ are 64-bit parts of a message/string being hashed. 346 | Since $s_{1}$ and $s_{2}$ are uniformly-distributed values, such mixing is 347 | equivalent to mixing a message with a cryptographic one-time-pad (bitwise 348 | modulo 2 addition). Message's statistics and distribution become unimportant, 349 | and do not change the uniform distribution of $s_{1}$ and $s_{2}$. 350 | 351 | ```c 352 | #include 353 | #include "komihash.h" 354 | 355 | int main(void) 356 | { 357 | uint64_t Seed1 = 0, Seed2 = 0; 358 | int i; 359 | 360 | for( i = 0; i < 8; i++ ) 361 | { 362 | printf( "%016llx\n", komirand( &Seed1, &Seed2 )); 363 | } 364 | } 365 | ``` 366 | 367 | Output: 368 | 369 | ``` 370 | aaaaaaaaaaaaaaaa 371 | fffffffffffffffe 372 | 4924924924924910 373 | baebaebaebaeba00 374 | 400c62cc4727496b 375 | 35a969173e8f925b 376 | db47f6bae9a247ad 377 | 98e0f6cece6711fe 378 | ``` 379 | 380 | ## Other 381 | 382 | This function is named the way it is named is to honor 383 | the [Komi Republic](https://en.wikipedia.org/wiki/Komi_Republic) (located in 384 | Russia), native to the author. 385 | 386 | According to OMNIUM REGIONUM MOSCOVIAE DESCRIPTIO by ALEXANDRI GWAGNINI 387 | VERONENSIS 1611, the territory of the Komi Republic was internationally 388 | known as Condora Regio inhabited by people worshipping a golden 389 | (sun-shining) woman-goddess, usually depicted as holding a child and 390 | a spear, Zarni Ana (Зарни Ань), and usually roughly translated as 391 | Zlatababa. Condora is also mentioned as a territory near the Mezena 392 | (Мезень) river on the map produced by Englishman Anthony Jenkinson in 393 | 1562. "Condora" may have meant a "pine country" (from Komi "conda"-pine 394 | and "dor"-region, land area, country). Southern part of the modern Komi 395 | Republic near the Voichegda and Vischera rivers, baptized by bishop 396 | Stephen Velickopermsky, was known as Permia, which does not correspond 397 | to the modern Perm region near the Cama river. 398 | 399 | ## Test Vectors 400 | 401 | Test vectors for the current version of `komihash`, string-hash pairs (note 402 | that the parentheses are not included in the calculation). The `bulk` is a 403 | buffer with increasing 8-bit values; `bulk` hashes are calculated from this 404 | buffer using various lengths. See the `testvec.c` file for details. 405 | 406 | ``` 407 | komihash UseSeed = 0x0000000000000000: 408 | "This is a 32-byte testing string" = 0x05ad960802903a9d 409 | "The cat is out of the bag" = 0xd15723521d3c37b1 410 | "A 16-byte string" = 0x467caa28ea3da7a6 411 | "The new string" = 0xf18e67bc90c43233 412 | "7 chars" = 0x2c514f6e5dcb11cb 413 | bulk(3) = 0x7a9717e9eea4be8b 414 | bulk(6) = 0xa56469564c2ea0ff 415 | bulk(8) = 0x00b4313a24431306 416 | bulk(12) = 0x64c2ad96013f70fe 417 | bulk(20) = 0x7a3888bc95545364 418 | bulk(31) = 0xc77e02ed4b201b9a 419 | bulk(32) = 0x256d74350303a1ba 420 | bulk(40) = 0x59609c71697bb9df 421 | bulk(47) = 0x36eb9e6a4c2c5e4b 422 | bulk(48) = 0x8dd56c332850baa6 423 | bulk(56) = 0xcbb722192b353999 424 | bulk(64) = 0x90b07e2158f88cc0 425 | bulk(72) = 0x24c9621701603741 426 | bulk(80) = 0x1d4c1d97ca684334 427 | bulk(112) = 0xd1a425d530652287 428 | bulk(132) = 0x72623be342c20ab5 429 | bulk(256) = 0x94c3dbdca59ddf57 430 | 431 | komihash UseSeed = 0x0123456789abcdef: 432 | "This is a 32-byte testing string" = 0x6ce66a2e8d4979a5 433 | "The cat is out of the bag" = 0x5b1da0b43545d196 434 | "A 16-byte string" = 0x26af914213d0c915 435 | "The new string" = 0x62d9ca1b73250cb5 436 | "7 chars" = 0x90ab7c9f831cd940 437 | bulk(3) = 0x84ae4eb65b96617e 438 | bulk(6) = 0xaceebc32a3c0d9e4 439 | bulk(8) = 0xdaa1a90ecb95f6f8 440 | bulk(12) = 0xec8eb3ef4af380b4 441 | bulk(20) = 0x07045bd31abba34c 442 | bulk(31) = 0xd5f619fb2e62c4ae 443 | bulk(32) = 0x5a336fd2c4c39abe 444 | bulk(40) = 0x0e870b4623eea8ec 445 | bulk(47) = 0xe552edd6bf419d1d 446 | bulk(48) = 0x37d170ddcb1223e6 447 | bulk(56) = 0x1cd89e708e5098b6 448 | bulk(64) = 0x765490569ccd77f2 449 | bulk(72) = 0x19e9d77b86d01ee8 450 | bulk(80) = 0x25f83ee520c1d241 451 | bulk(112) = 0xd6007417091cd4c0 452 | bulk(132) = 0x3e49c2d3727b9cc9 453 | bulk(256) = 0xb2b3405ee5d65f4c 454 | 455 | komihash UseSeed = 0x0000000000000100: 456 | "This is a 32-byte testing string" = 0x5f197b30bcec1e45 457 | "The cat is out of the bag" = 0xa761280322bb7698 458 | "A 16-byte string" = 0x11c31ccabaa524f1 459 | "The new string" = 0x3a43b7f58281c229 460 | "7 chars" = 0xcff90b0466b7e3a2 461 | bulk(3) = 0x8ab53f45cc9315e3 462 | bulk(6) = 0xea606e43d1976ccf 463 | bulk(8) = 0x889b2f2ceecbec73 464 | bulk(12) = 0xacbec1886cd23275 465 | bulk(20) = 0x57c3affd1b71fcdb 466 | bulk(31) = 0x7ef6ba49a3b068c3 467 | bulk(32) = 0x49dbca62ed5a1ddf 468 | bulk(40) = 0x192848484481e8c0 469 | bulk(47) = 0x420b43a5edba1bd7 470 | bulk(48) = 0xd6e8400a9de24ce3 471 | bulk(56) = 0xbea291b225ff384d 472 | bulk(64) = 0x0ec94062b2f06960 473 | bulk(72) = 0xfa613272ecd49985 474 | bulk(80) = 0x76f0bb380bc207be 475 | bulk(112) = 0x4afb4e08ca77c020 476 | bulk(132) = 0x410f9c129ad88aea 477 | bulk(256) = 0x066c7b25f4f569ae 478 | 479 | komirand Seed1/Seed2 = 0x0000000000000000: 480 | 0xaaaaaaaaaaaaaaaa 481 | 0xfffffffffffffffe 482 | 0x4924924924924910 483 | 0xbaebaebaebaeba00 484 | 0x400c62cc4727496b 485 | 0x35a969173e8f925b 486 | 0xdb47f6bae9a247ad 487 | 0x98e0f6cece6711fe 488 | 0x97ffa2397fda534b 489 | 0x11834262360df918 490 | 0x34e53df5399f2252 491 | 0xecaeb74a81d648ed 492 | 493 | komirand Seed1/Seed2 = 0x0123456789abcdef: 494 | 0x776ad9718078ca64 495 | 0x737aa5d5221633d0 496 | 0x685046cca30f6f44 497 | 0xfb725cb01b30c1ba 498 | 0xc501cc999ede619f 499 | 0x8427298e525db507 500 | 0xd9baf3c54781f75e 501 | 0x7f5a4e5b97b37c7b 502 | 0xde8a0afe8e03b8c1 503 | 0xb6ed3e72b69fc3d6 504 | 0xa68727902f7628d0 505 | 0x44162b63af484587 506 | 507 | komirand Seed1/Seed2 = 0x0000000000000100: 508 | 0xaaaaaaaaaaababaa 509 | 0xfffffffff8fcf8fe 510 | 0xdb6dba1e4dbb1134 511 | 0xf5b7d3aec37f4cb1 512 | 0x66a571da7ded7051 513 | 0x2d59ec9245bf03d9 514 | 0x5c06a41bd510aed8 515 | 0xea5e7ea9d2bd07a2 516 | 0xe395015ddce7756f 517 | 0xc07981aaeaae3b38 518 | 0x2e120ebfee59a5a2 519 | 0x9001eee495244dba 520 | 521 | ``` 522 | -------------------------------------------------------------------------------- /komihash.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file komihash.h 3 | * 4 | * @version 5.28 5 | * 6 | * @brief The header file for the "komihash" 64-bit hash function, 7 | * the "komirand" 64-bit PRNG, and the streamed "komihash" implementation. 8 | * 9 | * The source code is written in ISO C99, with full C++ compliance enabled 10 | * conditionally and automatically when compiled with a C++ compiler. 11 | * 12 | * This function is named the way it is named is to honor the Komi Republic 13 | * (located in Russia), native to the author. 14 | * 15 | * Description is available at https://github.com/avaneev/komihash 16 | * 17 | * Email: aleksey.vaneev@gmail.com or info@voxengo.com 18 | * 19 | * LICENSE: 20 | * 21 | * Copyright (c) 2021-2025 Aleksey Vaneev 22 | * 23 | * Permission is hereby granted, free of charge, to any person obtaining a 24 | * copy of this software and associated documentation files (the "Software"), 25 | * to deal in the Software without restriction, including without limitation 26 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, 27 | * and/or sell copies of the Software, and to permit persons to whom the 28 | * Software is furnished to do so, subject to the following conditions: 29 | * 30 | * The above copyright notice and this permission notice shall be included in 31 | * all copies or substantial portions of the Software. 32 | * 33 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 34 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 35 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 36 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 37 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 38 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 39 | * DEALINGS IN THE SOFTWARE. 40 | */ 41 | 42 | #ifndef KOMIHASH_INCLUDED 43 | #define KOMIHASH_INCLUDED 44 | 45 | #define KOMIHASH_VER_STR "5.28" ///< KOMIHASH source code version string. 46 | 47 | /** 48 | * @def KOMIHASH_NS_CUSTOM 49 | * @brief If this macro is defined externally, all symbols will be placed into 50 | * the C++ namespace specified by the macro, and won't be exported to the 51 | * global namespace. WARNING: if the defined value of the macro is empty, the 52 | * symbols will be placed into the global namespace anyway. 53 | */ 54 | 55 | /** 56 | * @def KOMIHASH_U64_C( x ) 57 | * @brief Macro that defines a numeric value as unsigned 64-bit value. 58 | * 59 | * @param x Value. 60 | */ 61 | 62 | /** 63 | * @def KOMIHASH_NOEX 64 | * @brief Macro that defines the "noexcept" function specifier for C++ 65 | * environment. 66 | */ 67 | 68 | /** 69 | * @def KOMIHASH_NS 70 | * @brief Macro that defines an actual implementation namespace in C++ 71 | * environment, with export of relevant symbols to the global namespace 72 | * (if @ref KOMIHASH_NS_CUSTOM is undefined). 73 | */ 74 | 75 | #if defined( __cplusplus ) 76 | 77 | #include 78 | 79 | #if __cplusplus >= 201103L 80 | 81 | #include 82 | 83 | #define KOMIHASH_U64_C( x ) UINT64_C( x ) 84 | #define KOMIHASH_NOEX noexcept 85 | 86 | #else // __cplusplus >= 201103L 87 | 88 | #include 89 | 90 | #define KOMIHASH_U64_C( x ) (uint64_t) x 91 | #define KOMIHASH_NOEX throw() 92 | 93 | #endif // __cplusplus >= 201103L 94 | 95 | #if defined( KOMIHASH_NS_CUSTOM ) 96 | #define KOMIHASH_NS KOMIHASH_NS_CUSTOM 97 | #else // defined( KOMIHASH_NS_CUSTOM ) 98 | #define KOMIHASH_NS komihash_impl 99 | #endif // defined( KOMIHASH_NS_CUSTOM ) 100 | 101 | #else // defined( __cplusplus ) 102 | 103 | #include 104 | #include 105 | 106 | #define KOMIHASH_U64_C( x ) (uint64_t) x 107 | #define KOMIHASH_NOEX 108 | 109 | #endif // defined( __cplusplus ) 110 | 111 | #if defined( _MSC_VER ) 112 | #include 113 | #endif // defined( _MSC_VER ) 114 | 115 | /** 116 | * @{ 117 | * @brief Unsigned 64-bit constant that defines the initial state of the 118 | * hash function (first mantissa bits of PI). 119 | */ 120 | 121 | #define KOMIHASH_IVAL1 KOMIHASH_U64_C( 0x243F6A8885A308D3 ) 122 | #define KOMIHASH_IVAL2 KOMIHASH_U64_C( 0x13198A2E03707344 ) 123 | #define KOMIHASH_IVAL3 KOMIHASH_U64_C( 0xA4093822299F31D0 ) 124 | #define KOMIHASH_IVAL4 KOMIHASH_U64_C( 0x082EFA98EC4E6C89 ) 125 | #define KOMIHASH_IVAL5 KOMIHASH_U64_C( 0x452821E638D01377 ) 126 | #define KOMIHASH_IVAL6 KOMIHASH_U64_C( 0xBE5466CF34E90C6C ) 127 | #define KOMIHASH_IVAL7 KOMIHASH_U64_C( 0xC0AC29B7C97C50DD ) 128 | #define KOMIHASH_IVAL8 KOMIHASH_U64_C( 0x3F84D5B5B5470917 ) 129 | 130 | /** @} */ 131 | 132 | /** 133 | * @def KOMIHASH_VAL01 134 | * @brief Unsigned 64-bit constant with `01` bit-pair replication. 135 | */ 136 | 137 | #define KOMIHASH_VAL01 KOMIHASH_U64_C( 0x5555555555555555 ) 138 | 139 | /** 140 | * @def KOMIHASH_VAL10 141 | * @brief Unsigned 64-bit constant with `10` bit-pair replication. 142 | */ 143 | 144 | #define KOMIHASH_VAL10 KOMIHASH_U64_C( 0xAAAAAAAAAAAAAAAA ) 145 | 146 | /** 147 | * @def KOMIHASH_LITTLE_ENDIAN 148 | * @brief Endianness definition macro, can be used as a logical constant. 149 | * Equals 0, if C++20 `endian` is in use. 150 | * 151 | * Can be defined externally (e.g., =1, if endianness-correction and 152 | * hash-value portability are unnecessary in any case, to reduce overhead). 153 | */ 154 | 155 | /** 156 | * @def KOMIHASH_COND_EC( vl, vb ) 157 | * @brief Macro that emits either `vl` or `vb`, depending on platform's 158 | * endianness. 159 | */ 160 | 161 | #if !defined( KOMIHASH_LITTLE_ENDIAN ) 162 | #if ( defined( __BYTE_ORDER__ ) && \ 163 | __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ ) || \ 164 | ( defined( __BYTE_ORDER ) && __BYTE_ORDER == __LITTLE_ENDIAN ) || \ 165 | defined( __LITTLE_ENDIAN__ ) || defined( _LITTLE_ENDIAN ) || \ 166 | defined( _WIN32 ) || defined( i386 ) || defined( __i386 ) || \ 167 | defined( __i386__ ) || defined( _M_IX86 ) || defined( _M_AMD64 ) || \ 168 | defined( _X86_ ) || defined( __x86_64 ) || defined( __x86_64__ ) || \ 169 | defined( __amd64 ) || defined( __amd64__ ) || defined( _M_ARM ) 170 | 171 | #define KOMIHASH_LITTLE_ENDIAN 1 172 | 173 | #elif ( defined( __BYTE_ORDER__ ) && \ 174 | __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ) || \ 175 | ( defined( __BYTE_ORDER ) && __BYTE_ORDER == __BIG_ENDIAN ) || \ 176 | defined( __BIG_ENDIAN__ ) || defined( _BIG_ENDIAN ) || \ 177 | defined( __SYSC_ZARCH__ ) || defined( __zarch__ ) || \ 178 | defined( __s390x__ ) || defined( __sparc ) || defined( __sparc__ ) 179 | 180 | #define KOMIHASH_LITTLE_ENDIAN 0 181 | #define KOMIHASH_COND_EC( vl, vb ) ( vb ) 182 | 183 | #elif defined( __cplusplus ) && __cplusplus >= 202002L 184 | 185 | #include 186 | 187 | #define KOMIHASH_LITTLE_ENDIAN 0 188 | #define KOMIHASH_COND_EC( vl, vb ) ( std :: endian :: native == \ 189 | std :: endian :: little ? vl : vb ) 190 | 191 | #else // defined( __cplusplus ) 192 | 193 | #warning KOMIHASH: cannot determine endianness, assuming little-endian. 194 | 195 | #define KOMIHASH_LITTLE_ENDIAN 1 196 | 197 | #endif // defined( __cplusplus ) 198 | #endif // !defined( KOMIHASH_LITTLE_ENDIAN ) 199 | 200 | /** 201 | * @def KOMIHASH_ICC_GCC 202 | * @brief Macro that denotes the use of the ICC classic compiler with 203 | * GCC-style built-in functions. 204 | */ 205 | 206 | #if defined( __INTEL_COMPILER ) && __INTEL_COMPILER >= 1300 && \ 207 | !defined( _MSC_VER ) 208 | 209 | #define KOMIHASH_ICC_GCC 210 | 211 | #endif // ICC check 212 | 213 | /** 214 | * @def KOMIHASH_GCC_BUILTINS 215 | * @brief Macro that denotes availability of GCC-style built-in functions. 216 | */ 217 | 218 | #if defined( __GNUC__ ) || defined( __clang__ ) || \ 219 | defined( __IBMC__ ) || defined( __IBMCPP__ ) || \ 220 | defined( __COMPCERT__ ) || defined( KOMIHASH_ICC_GCC ) 221 | 222 | #define KOMIHASH_GCC_BUILTINS 223 | 224 | #endif // GCC built-ins check 225 | 226 | /** 227 | * @def KOMIHASH_BMI2 228 | * @brief Macro that denotes availability of `mulx` intrinsic (MSVC-compatible 229 | * compilers only). 230 | */ 231 | 232 | #if defined( _MSC_VER ) 233 | #if defined( __BMI2__ ) || ( !defined( KOMIHASH_GCC_BUILTINS ) && \ 234 | defined( _M_AMD64 ) && defined( __AVX2__ ) && \ 235 | ( defined( __INTEL_COMPILER ) || _MSC_VER >= 1900 )) 236 | 237 | #include 238 | #define KOMIHASH_BMI2 239 | 240 | #else // BMI2 241 | 242 | #include 243 | 244 | #endif // BMI2 245 | #endif // defined( _MSC_VER ) 246 | 247 | /** 248 | * @def KOMIHASH_EC32( v ) 249 | * @brief Macro that appies 32-bit byte-swapping, for endianness-correction. 250 | * Undefined for unknown compilers, if big-endian. 251 | * 252 | * @param v Value to byte-swap. 253 | */ 254 | 255 | /** 256 | * @def KOMIHASH_EC64( v ) 257 | * @brief Macro that appies 64-bit byte-swapping, for endianness-correction. 258 | * Undefined for unknown compilers, if big-endian. 259 | * 260 | * @param v Value to byte-swap. 261 | */ 262 | 263 | #if KOMIHASH_LITTLE_ENDIAN 264 | 265 | #define KOMIHASH_EC32( v ) ( v ) 266 | #define KOMIHASH_EC64( v ) ( v ) 267 | 268 | #else // KOMIHASH_LITTLE_ENDIAN 269 | 270 | #if defined( KOMIHASH_GCC_BUILTINS ) 271 | 272 | #define KOMIHASH_EC32( v ) KOMIHASH_COND_EC( v, __builtin_bswap32( v )) 273 | #define KOMIHASH_EC64( v ) KOMIHASH_COND_EC( v, __builtin_bswap64( v )) 274 | 275 | #elif defined( _MSC_VER ) 276 | 277 | #if defined( __cplusplus ) 278 | #include 279 | #else // defined( __cplusplus ) 280 | #include 281 | #endif // defined( __cplusplus ) 282 | 283 | #define KOMIHASH_EC32( v ) KOMIHASH_COND_EC( v, _byteswap_ulong( v )) 284 | #define KOMIHASH_EC64( v ) KOMIHASH_COND_EC( v, _byteswap_uint64( v )) 285 | 286 | #elif defined( __cplusplus ) && __cplusplus >= 202302L 287 | 288 | #include 289 | 290 | #define KOMIHASH_EC32( v ) KOMIHASH_COND_EC( v, std :: byteswap( v )) 291 | #define KOMIHASH_EC64( v ) KOMIHASH_COND_EC( v, std :: byteswap( v )) 292 | 293 | #endif // defined( __cplusplus ) 294 | 295 | #endif // KOMIHASH_LITTLE_ENDIAN 296 | 297 | /** 298 | * @def KOMIHASH_LIKELY( x ) 299 | * @brief Likelihood macro that is used for manually-guided 300 | * micro-optimization. 301 | * 302 | * @param x Expression that is likely to be evaluated to `true`. 303 | */ 304 | 305 | /** 306 | * @def KOMIHASH_UNLIKELY( x ) 307 | * @brief Unlikelihood macro that is used for manually-guided 308 | * micro-optimization. 309 | * 310 | * @param x Expression that is unlikely to be evaluated to `true`. 311 | */ 312 | 313 | #if defined( KOMIHASH_GCC_BUILTINS ) 314 | 315 | #define KOMIHASH_LIKELY( x ) ( __builtin_expect( x, 1 )) 316 | #define KOMIHASH_UNLIKELY( x ) ( __builtin_expect( x, 0 )) 317 | 318 | #elif defined( __cplusplus ) && __cplusplus >= 202002L 319 | 320 | #define KOMIHASH_LIKELY( x ) ( x ) [[likely]] 321 | #define KOMIHASH_UNLIKELY( x ) ( x ) [[unlikely]] 322 | 323 | #else // defined( __cplusplus ) 324 | 325 | #define KOMIHASH_LIKELY( x ) ( x ) 326 | #define KOMIHASH_UNLIKELY( x ) ( x ) 327 | 328 | #endif // defined( __cplusplus ) 329 | 330 | /** 331 | * @def KOMIHASH_PREFETCH( a ) 332 | * @brief Memory address prefetch macro, to preload some data into CPU cache. 333 | * 334 | * Temporal locality=3, in case a collision resolution would be necessary, 335 | * or for a subsequent disk write. 336 | * 337 | * @param a Prefetch address. 338 | */ 339 | 340 | #if defined( KOMIHASH_GCC_BUILTINS ) && !defined( __COMPCERT__ ) 341 | 342 | #define KOMIHASH_PREFETCH( a ) __builtin_prefetch( a, 0, 3 ) 343 | 344 | #elif defined( _MSC_VER ) && defined( _M_AMD64 ) && \ 345 | !defined( __INTEL_COMPILER ) 346 | 347 | #include 348 | 349 | #define KOMIHASH_PREFETCH( a ) _mm_prefetch( (const char*) ( a ), \ 350 | _MM_HINT_T0 ) 351 | 352 | #else // defined( _MSC_VER ) 353 | 354 | #define KOMIHASH_PREFETCH( a ) (void) 0 355 | 356 | #endif // defined( _MSC_VER ) 357 | 358 | /** 359 | * @def KOMIHASH_INLINE 360 | * @brief Macro that defines a function as inlinable at compiler's discretion. 361 | */ 362 | 363 | #if ( defined( __cplusplus ) && __cplusplus >= 201703L ) || \ 364 | ( defined( __STDC_VERSION__ ) && __STDC_VERSION__ >= 202311L ) 365 | 366 | #define KOMIHASH_INLINE [[maybe_unused]] static inline 367 | 368 | #elif defined( KOMIHASH_GCC_BUILTINS ) 369 | 370 | #define KOMIHASH_INLINE static __attribute__((unused)) inline 371 | 372 | #else // defined( KOMIHASH_GCC_BUILTINS ) 373 | 374 | #define KOMIHASH_INLINE static inline 375 | 376 | #endif // defined( KOMIHASH_GCC_BUILTINS ) 377 | 378 | /** 379 | * @def KOMIHASH_INLINE_F 380 | * @brief Macro to force code inlining. 381 | */ 382 | 383 | #if defined( KOMIHASH_GCC_BUILTINS ) 384 | 385 | #define KOMIHASH_INLINE_F KOMIHASH_INLINE __attribute__((always_inline)) 386 | 387 | #elif defined( _MSC_VER ) 388 | 389 | #define KOMIHASH_INLINE_F KOMIHASH_INLINE __forceinline 390 | 391 | #else // defined( _MSC_VER ) 392 | 393 | #define KOMIHASH_INLINE_F KOMIHASH_INLINE 394 | 395 | #endif // defined( _MSC_VER ) 396 | 397 | #if defined( KOMIHASH_NS ) 398 | 399 | namespace KOMIHASH_NS { 400 | 401 | using std :: memcpy; 402 | using std :: size_t; 403 | 404 | #if __cplusplus >= 201103L 405 | 406 | using std :: uint32_t; 407 | using std :: uint64_t; 408 | using uint8_t = unsigned char; ///< For C++ type aliasing compliance. 409 | 410 | #endif // __cplusplus >= 201103L 411 | 412 | #endif // defined( KOMIHASH_NS ) 413 | 414 | /** 415 | * @{ 416 | * @brief Load unsigned value of corresponding bit-size, with 417 | * endianness-correction. 418 | * 419 | * An auxiliary function that returns an unsigned value created out of a 420 | * sequence of bytes in memory. This function is used to convert endianness 421 | * of in-memory unsigned values, and to avoid unaligned memory accesses. 422 | * 423 | * @param p Pointer to bytes in memory. Alignment is unimportant. 424 | * @return Endianness-corrected value from memory, typecasted to `uint64_t`. 425 | */ 426 | 427 | KOMIHASH_INLINE_F uint64_t kh_lu32ec( const uint8_t* const p ) KOMIHASH_NOEX 428 | { 429 | #if defined( KOMIHASH_EC32 ) 430 | 431 | uint32_t v; 432 | memcpy( &v, p, 4 ); 433 | 434 | return( KOMIHASH_EC32( v )); 435 | 436 | #else // defined( KOMIHASH_EC32 ) 437 | 438 | return( (uint32_t) ( p[ 0 ] | p[ 1 ] << 8 | p[ 2 ] << 16 | p[ 3 ] << 24 )); 439 | 440 | #endif // defined( KOMIHASH_EC32 ) 441 | } 442 | 443 | KOMIHASH_INLINE_F uint64_t kh_lu64ec( const uint8_t* const p ) KOMIHASH_NOEX 444 | { 445 | #if defined( KOMIHASH_EC64 ) 446 | 447 | uint64_t v; 448 | memcpy( &v, p, 8 ); 449 | 450 | return( KOMIHASH_EC64( v )); 451 | 452 | #else // defined( KOMIHASH_EC64 ) 453 | 454 | return( kh_lu32ec( p ) | kh_lu32ec( p + 4 ) << 32 ); 455 | 456 | #endif // defined( KOMIHASH_EC64 ) 457 | } 458 | 459 | /** @} */ 460 | 461 | /** 462 | * @def KOMIHASH_M128_IMPL 463 | * @brief Aux macro for kh_m128() implementation. 464 | */ 465 | 466 | /** 467 | * @def KOMIHASH_EMULU( u, v ) 468 | * @brief Aux macro for `__emulu()` intrinsic. 469 | * 470 | * @param u Multiplier 1. 471 | * @param v Multiplier 2. 472 | */ 473 | 474 | #if defined( KOMIHASH_BMI2 ) 475 | 476 | #define KOMIHASH_M128_IMPL \ 477 | unsigned long long rh; \ 478 | *rl = _mulx_u64( u, v, &rh ); \ 479 | *rha += rh; 480 | 481 | #elif defined( _MSC_VER ) && \ 482 | ( defined( _M_ARM64 ) || defined( _M_ARM64EC ) || \ 483 | ( defined( __INTEL_COMPILER ) && defined( _M_AMD64 ))) 484 | 485 | #define KOMIHASH_M128_IMPL \ 486 | const uint64_t rh = __umulh( u, v ); \ 487 | *rl = u * v; \ 488 | *rha += rh; 489 | 490 | #elif defined( _MSC_VER ) && ( defined( _M_AMD64 ) || defined( _M_IA64 )) 491 | 492 | #pragma intrinsic(_umul128) 493 | 494 | #define KOMIHASH_M128_IMPL \ 495 | uint64_t rh; \ 496 | *rl = _umul128( u, v, &rh ); \ 497 | *rha += rh; 498 | 499 | #elif defined( __SIZEOF_INT128__ ) || \ 500 | ( defined( KOMIHASH_ICC_GCC ) && defined( __x86_64__ )) 501 | 502 | #define KOMIHASH_M128_IMPL \ 503 | __uint128_t r = u; \ 504 | r *= v; \ 505 | const uint64_t rh = (uint64_t) ( r >> 64 ); \ 506 | *rl = (uint64_t) r; \ 507 | *rha += rh; 508 | 509 | #elif ( defined( __IBMC__ ) || defined( __IBMCPP__ )) && defined( __LP64__ ) 510 | 511 | #define KOMIHASH_M128_IMPL \ 512 | const uint64_t rh = __mulhdu( u, v ); \ 513 | *rl = u * v; \ 514 | *rha += rh; 515 | 516 | #else // defined( __IBMC__ ) 517 | 518 | #if defined( _MSC_VER ) && !defined( __INTEL_COMPILER ) && \ 519 | !defined( _M_ARM ) 520 | 521 | #pragma intrinsic(__emulu) 522 | 523 | #define KOMIHASH_EMULU( u, v ) __emulu( u, v ) 524 | 525 | #else // __emulu 526 | 527 | #define KOMIHASH_EMULU( u, v ) ( (uint64_t) ( u ) * ( v )) 528 | 529 | #endif // __emulu 530 | 531 | #endif // defined( __IBMC__ ) 532 | 533 | /** 534 | * @brief 64-bit by 64-bit unsigned multiplication with result accumulation. 535 | * 536 | * @param u Multiplier 1. 537 | * @param v Multiplier 2. 538 | * @param[out] rl The lower half of the 128-bit result. 539 | * @param[in,out] rha The accumulator to receive the higher half of the 540 | * 128-bit result. 541 | */ 542 | 543 | #if defined( KOMIHASH_M128_IMPL ) 544 | KOMIHASH_INLINE_F 545 | #else // defined( KOMIHASH_M128_IMPL ) 546 | KOMIHASH_INLINE 547 | #endif // defined( KOMIHASH_M128_IMPL ) 548 | 549 | void kh_m128( const uint64_t u, const uint64_t v, 550 | uint64_t* const rl, uint64_t* const rha ) KOMIHASH_NOEX 551 | { 552 | #if defined( KOMIHASH_M128_IMPL ) 553 | 554 | KOMIHASH_M128_IMPL 555 | 556 | #undef KOMIHASH_M128_IMPL 557 | 558 | #else // defined( KOMIHASH_M128_IMPL ) 559 | 560 | // _umul128() code for 32-bit systems, adapted from Hacker's Delight, 561 | // Henry S. Warren, Jr. 562 | 563 | *rl = u * v; 564 | 565 | const uint32_t u0 = (uint32_t) u; 566 | const uint32_t v0 = (uint32_t) v; 567 | const uint64_t w0 = KOMIHASH_EMULU( u0, v0 ); 568 | const uint32_t u1 = (uint32_t) ( u >> 32 ); 569 | const uint32_t v1 = (uint32_t) ( v >> 32 ); 570 | const uint64_t t = KOMIHASH_EMULU( u1, v0 ) + (uint32_t) ( w0 >> 32 ); 571 | const uint64_t w1 = KOMIHASH_EMULU( u0, v1 ) + (uint32_t) t; 572 | 573 | *rha += KOMIHASH_EMULU( u1, v1 ) + (uint32_t) ( w1 >> 32 ) + 574 | (uint32_t) ( t >> 32 ); 575 | 576 | #undef KOMIHASH_EMULU 577 | 578 | #endif // defined( KOMIHASH_M128_IMPL ) 579 | } 580 | 581 | /** 582 | * @def KOMIHASH_HASHROUND() 583 | * @brief Macro for a common hashing round without input. 584 | * 585 | * The three instructions in this macro (multiply, add, and XOR) represent the 586 | * simplest constantless PRNG, scalable to any even-sized state variables, 587 | * with the `Seed1` being the PRNG output (2^64 PRNG period). It passes 588 | * `PractRand` tests with rare non-systematic "unusual" evaluations. 589 | * 590 | * To make this PRNG reliable, self-starting, and eliminate a risk of 591 | * stopping, the following variant can be used, which adds a "register 592 | * checker-board", a source of raw entropy. The PRNG is available as the 593 | * komirand() function. Not required for hashing (but works for it) since the 594 | * input entropy is usually available in abundance during hashing. 595 | * 596 | * `Seed5 += 0xAAAAAAAAAAAAAAAA;` 597 | * 598 | * (the `0xAAAA...` constant should match register's size; essentially, it is 599 | * a replication of the `10` bit-pair; it is not an arbitrary constant). 600 | */ 601 | 602 | #define KOMIHASH_HASHROUND() \ 603 | kh_m128( Seed1, Seed5, &Seed1, &Seed5 ); \ 604 | Seed1 ^= Seed5 605 | 606 | /** 607 | * @def KOMIHASH_HASH16( m ) 608 | * @brief Macro for a common hashing round with 16-byte input. 609 | * 610 | * @param m Message pointer, alignment is unimportant. 611 | */ 612 | 613 | #define KOMIHASH_HASH16( m ) \ 614 | kh_m128( kh_lu64ec( m ) ^ Seed1, \ 615 | kh_lu64ec( m + 8 ) ^ Seed5, &Seed1, &Seed5 ); \ 616 | Seed1 ^= Seed5 617 | 618 | /** 619 | * @def KOMIHASH_HASHFIN() 620 | * @brief Macro for common hashing finalization round. 621 | * 622 | * The final hashing input is expected in the `r1h` and `r2h` temporary 623 | * variables. The macro inserts the function return instruction. 624 | */ 625 | 626 | #define KOMIHASH_HASHFIN() \ 627 | kh_m128( r1h, r2h, &Seed1, &Seed5 ); \ 628 | Seed1 ^= Seed5; \ 629 | KOMIHASH_HASHROUND(); \ 630 | return( Seed1 ) 631 | 632 | /** 633 | * @def KOMIHASH_HASHLOOP64() 634 | * @brief Macro for a common 64-byte full-performance hashing loop. 635 | * 636 | * Expects `Msg` and `MsgLen` values (greater than 63), requires initialized 637 | * `Seed1-8` values. 638 | * 639 | * The "shifting" arrangement of `Seed1-4` XORs (below) does not increase 640 | * individual `SeedN` PRNG period beyond 2^64, but reduces a chance of any 641 | * occassional synchronization between PRNG lanes happening. Practically, 642 | * `Seed1-4` together become a single "fused" 256-bit PRNG value, having 2^66 643 | * summary PRNG period. 644 | */ 645 | 646 | #define KOMIHASH_HASHLOOP64() \ 647 | do \ 648 | { \ 649 | kh_m128( kh_lu64ec( Msg ) ^ Seed1, \ 650 | kh_lu64ec( Msg + 32 ) ^ Seed5, &Seed1, &Seed5 ); \ 651 | \ 652 | kh_m128( kh_lu64ec( Msg + 8 ) ^ Seed2, \ 653 | kh_lu64ec( Msg + 40 ) ^ Seed6, &Seed2, &Seed6 ); \ 654 | \ 655 | kh_m128( kh_lu64ec( Msg + 16 ) ^ Seed3, \ 656 | kh_lu64ec( Msg + 48 ) ^ Seed7, &Seed3, &Seed7 ); \ 657 | \ 658 | kh_m128( kh_lu64ec( Msg + 24 ) ^ Seed4, \ 659 | kh_lu64ec( Msg + 56 ) ^ Seed8, &Seed4, &Seed8 ); \ 660 | \ 661 | Msg += 64; \ 662 | MsgLen -= 64; \ 663 | \ 664 | KOMIHASH_PREFETCH( Msg ); \ 665 | \ 666 | Seed4 ^= Seed7; \ 667 | Seed1 ^= Seed8; \ 668 | Seed3 ^= Seed6; \ 669 | Seed2 ^= Seed5; \ 670 | \ 671 | } while KOMIHASH_LIKELY( MsgLen > 63 ) 672 | 673 | /** 674 | * @brief The hashing epilogue function (for internal use). 675 | * 676 | * @param Msg Pointer to the remaining part of the message. 677 | * @param MsgLen Remaining part's length, can be 0. 678 | * @param Seed1 Latest `Seed1` value. 679 | * @param Seed5 Latest `Seed5` value. 680 | * @return 64-bit hash value. 681 | */ 682 | 683 | KOMIHASH_INLINE_F uint64_t komihash_epi( const uint8_t* Msg, size_t MsgLen, 684 | uint64_t Seed1, uint64_t Seed5 ) KOMIHASH_NOEX 685 | { 686 | uint64_t r1h, r2h; 687 | 688 | if( MsgLen > 31 ) 689 | { 690 | KOMIHASH_HASH16( Msg ); 691 | KOMIHASH_HASH16( Msg + 16 ); 692 | 693 | MsgLen -= 32; 694 | Msg += 32; 695 | } 696 | 697 | if( MsgLen > 15 ) 698 | { 699 | KOMIHASH_HASH16( Msg ); 700 | 701 | MsgLen -= 16; 702 | Msg += 16; 703 | } 704 | 705 | int ml8 = (int) ( MsgLen * 8 ); 706 | 707 | if( MsgLen < 8 ) 708 | { 709 | ml8 ^= 56; 710 | r1h = kh_lu64ec( Msg + MsgLen - 8 ) >> 8 | (uint64_t) 1 << 56; 711 | r2h = Seed5; 712 | r1h = ( r1h >> ml8 ) ^ Seed1; 713 | } 714 | else 715 | { 716 | r2h = kh_lu64ec( Msg + MsgLen - 8 ) >> 8 | (uint64_t) 1 << 56; 717 | ml8 ^= 120; 718 | r1h = kh_lu64ec( Msg ) ^ Seed1; 719 | r2h = ( r2h >> ml8 ) ^ Seed5; 720 | } 721 | 722 | KOMIHASH_HASHFIN(); 723 | } 724 | 725 | /** 726 | * @brief KOMIHASH 64-bit hash function. 727 | * 728 | * Produces and returns a 64-bit hash value of the specified message, string, 729 | * or binary data block. Designed for 64-bit hash-table and hash-map uses, and 730 | * can be also used for checksums. Produces identical hashes on both big- and 731 | * little-endian systems. 732 | * 733 | * @param Msg0 The message to produce a hash from. The alignment of this 734 | * pointer is unimportant. It is valid to pass 0 when `MsgLen` equals 0 735 | * (assuming that compiler's implementation of the address prefetch is 736 | * non-failing, as per GCC specification). 737 | * @param MsgLen Message's length, in bytes, can be zero. 738 | * @param UseSeed Optional value, to use instead of the default seed. To use 739 | * the default seed, set to 0. This value can have any number of significant 740 | * bits, and any statistical quality. May need endianness-correction via 741 | * KOMIHASH_EC64(), if this value is shared between big- and little-endian 742 | * systems. 743 | * @return 64-bit hash of the input data. Should be endianness-corrected when 744 | * this value is shared between big- and little-endian systems. 745 | */ 746 | 747 | KOMIHASH_INLINE uint64_t komihash( const void* const Msg0, size_t MsgLen, 748 | const uint64_t UseSeed ) KOMIHASH_NOEX 749 | { 750 | const uint8_t* Msg = (const uint8_t*) Msg0; 751 | 752 | uint64_t Seed1 = KOMIHASH_IVAL1 ^ ( UseSeed & KOMIHASH_VAL01 ); 753 | uint64_t Seed5 = KOMIHASH_IVAL5 ^ ( UseSeed & KOMIHASH_VAL10 ); 754 | uint64_t r1h, r2h; 755 | 756 | KOMIHASH_PREFETCH( Msg ); 757 | 758 | KOMIHASH_HASHROUND(); // Required for Perlin Noise. 759 | 760 | if KOMIHASH_LIKELY( MsgLen < 16 ) 761 | { 762 | r1h = Seed1; 763 | r2h = Seed5; 764 | 765 | if( MsgLen > 7 ) 766 | { 767 | // The following XOR instructions are equivalent to mixing a 768 | // message with a cryptographic one-time-pad (bitwise modulo 2 769 | // addition). Message's statistics and distribution are thus 770 | // unimportant. 771 | 772 | r1h ^= kh_lu64ec( Msg ); 773 | 774 | if( MsgLen < 12 ) 775 | { 776 | int ml8 = (int) ( MsgLen * 8 ); 777 | const uint64_t m = (uint64_t) ( Msg[ MsgLen - 3 ] | 778 | Msg[ MsgLen - 1 ] << 16 | 1 << 24 | 779 | Msg[ MsgLen - 2 ] << 8 ); 780 | 781 | ml8 ^= 88; 782 | r2h ^= m >> ml8; 783 | } 784 | else 785 | { 786 | const int mhs = (int) ( 128 - MsgLen * 8 ); 787 | const uint64_t mh = ( kh_lu32ec( Msg + MsgLen - 4 ) | 788 | (uint64_t) 1 << 32 ) >> mhs; 789 | 790 | const uint64_t ml = kh_lu32ec( Msg + 8 ); 791 | 792 | r2h ^= mh << 32 | ml; 793 | } 794 | } 795 | else 796 | if KOMIHASH_LIKELY( MsgLen != 0 ) 797 | { 798 | const int ml8 = (int) ( MsgLen * 8 ); 799 | 800 | if( MsgLen < 4 ) 801 | { 802 | r1h ^= (uint64_t) 1 << ml8; 803 | r1h ^= (uint64_t) Msg[ 0 ]; 804 | 805 | if( MsgLen != 1 ) 806 | { 807 | r1h ^= (uint64_t) Msg[ 1 ] << 8; 808 | 809 | if( MsgLen != 2 ) 810 | { 811 | r1h ^= (uint64_t) Msg[ 2 ] << 16; 812 | } 813 | } 814 | } 815 | else 816 | { 817 | const int mhs = 64 - ml8; 818 | const uint64_t mh = ( kh_lu32ec( Msg + MsgLen - 4 ) | 819 | (uint64_t) 1 << 32 ) >> mhs; 820 | 821 | const uint64_t ml = kh_lu32ec( Msg ); 822 | 823 | r1h ^= mh << 32 | ml; 824 | } 825 | } 826 | } 827 | else 828 | { 829 | if KOMIHASH_UNLIKELY( MsgLen > 31 ) 830 | { 831 | goto _long; 832 | } 833 | 834 | KOMIHASH_HASH16( Msg ); 835 | 836 | int ml8 = (int) ( MsgLen * 8 ); 837 | 838 | if( MsgLen < 24 ) 839 | { 840 | ml8 ^= 184; 841 | r1h = kh_lu64ec( Msg + MsgLen - 8 ) >> 8 | (uint64_t) 1 << 56; 842 | r2h = Seed5; 843 | r1h = ( r1h >> ml8 ) ^ Seed1; 844 | 845 | KOMIHASH_HASHFIN(); 846 | } 847 | else 848 | { 849 | r2h = kh_lu64ec( Msg + MsgLen - 8 ) >> 8 | (uint64_t) 1 << 56; 850 | ml8 ^= 248; 851 | r1h = kh_lu64ec( Msg + 16 ) ^ Seed1; 852 | r2h = ( r2h >> ml8 ) ^ Seed5; 853 | } 854 | } 855 | 856 | KOMIHASH_HASHFIN(); 857 | 858 | _long: 859 | if KOMIHASH_LIKELY( MsgLen > 63 ) 860 | { 861 | uint64_t Seed2 = KOMIHASH_IVAL2 ^ Seed1; 862 | uint64_t Seed3 = KOMIHASH_IVAL3 ^ Seed1; 863 | uint64_t Seed4 = KOMIHASH_IVAL4 ^ Seed1; 864 | uint64_t Seed6 = KOMIHASH_IVAL6 ^ Seed5; 865 | uint64_t Seed7 = KOMIHASH_IVAL7 ^ Seed5; 866 | uint64_t Seed8 = KOMIHASH_IVAL8 ^ Seed5; 867 | 868 | KOMIHASH_HASHLOOP64(); 869 | 870 | Seed5 ^= Seed6 ^ Seed7 ^ Seed8; 871 | Seed1 ^= Seed2 ^ Seed3 ^ Seed4; 872 | } 873 | 874 | return( komihash_epi( Msg, MsgLen, Seed1, Seed5 )); 875 | } 876 | 877 | /** 878 | * @brief KOMIRAND 64-bit pseudo-random number generator. 879 | * 880 | * Simple, reliable, self-starting, yet efficient PRNG with a 2^64 period. 881 | * 0.62 cycles/byte performance. It self-starts in 4 iterations, which is the 882 | * suggested "warm-up" period before using its output when seeds are 883 | * initialized with an arbitrary value. If initialized with high-quality, 884 | * uniformly random value (e.g., from the operating system's entropy or a 885 | * hash function's output), the PRNG output is valid from the start. 886 | * 887 | * @param[in,out] Seed1 Seed value 1. Can be initialized to any value 888 | * (even 0). This is the usual "PRNG seed" value. 889 | * @param[in,out] Seed2 Seed value 2, a supporting variable. Must be 890 | * initialized to the same value as `Seed1`. Should not be used as the PRNG 891 | * value. 892 | * @return The next uniformly random 64-bit value. 893 | */ 894 | 895 | KOMIHASH_INLINE_F uint64_t komirand( uint64_t* const Seed1, 896 | uint64_t* const Seed2 ) KOMIHASH_NOEX 897 | { 898 | uint64_t s1 = *Seed1; 899 | uint64_t s2 = *Seed2; 900 | 901 | kh_m128( s1, s2, &s1, &s2 ); 902 | s2 += KOMIHASH_VAL10; 903 | s1 ^= s2; 904 | 905 | *Seed2 = s2; 906 | *Seed1 = s1; 907 | 908 | return( s1 ); 909 | } 910 | 911 | /** 912 | * @def KOMIHASH_BUFSIZE 913 | * @brief Streamed hashing's buffer size, in bytes. 914 | * 915 | * Must be a multiple of 64, and not less than 128. Can be defined externally. 916 | */ 917 | 918 | #if !defined( KOMIHASH_BUFSIZE ) 919 | 920 | #define KOMIHASH_BUFSIZE 768 921 | 922 | #endif // !defined( KOMIHASH_BUFSIZE ) 923 | 924 | /** 925 | * @brief Context structure for the streamed "komihash" hashing. 926 | * 927 | * The komihash_init() function should be called to initalize the structure 928 | * before hashing. Note that the default buffer size is modest, permitting 929 | * placement of this structure on stack. `Seed[ 0 ]` is used as `UseSeed` 930 | * value storage. 931 | */ 932 | 933 | typedef struct { 934 | uint8_t pb[ 8 ]; ///< Buffer's padding bytes, to avoid OOB. 935 | uint8_t Buf[ KOMIHASH_BUFSIZE ]; ///< Buffer. 936 | uint64_t Seed[ 8 ]; ///< Hashing state variables. 937 | size_t BufFill; ///< Buffer fill count (position), in bytes. 938 | size_t IsHashing; ///< 0 or 1, equals 1 if the actual hashing was started. 939 | } komihash_stream_t; 940 | 941 | /** 942 | * @brief Function initializes the streamed "komihash" hashing session. 943 | * 944 | * @param[out] ctx Pointer to the context structure. 945 | * @param UseSeed Optional value, to use instead of the default seed. To use 946 | * the default seed, set to 0. This value can have any number of significant 947 | * bits, and any statistical quality. May need endianness-correction via 948 | * KOMIHASH_EC64(), if this value is shared between big- and little-endian 949 | * systems. 950 | */ 951 | 952 | KOMIHASH_INLINE void komihash_stream_init( komihash_stream_t* const ctx, 953 | const uint64_t UseSeed ) KOMIHASH_NOEX 954 | { 955 | ctx -> Seed[ 0 ] = UseSeed; 956 | ctx -> BufFill = 0; 957 | ctx -> IsHashing = 0; 958 | } 959 | 960 | /** 961 | * @brief Function updates the streamed hashing state with a new input data. 962 | * 963 | * @param[in,out] ctx Pointer to the context structure. The structure must be 964 | * initialized via the komihash_stream_init() function. 965 | * @param Msg0 The next part of the whole message being hashed. The alignment 966 | * of this pointer is unimportant. It is valid to pass 0 when `MsgLen` equals 967 | * 0. 968 | * @param MsgLen Message's length, in bytes, can be zero. 969 | */ 970 | 971 | KOMIHASH_INLINE void komihash_stream_update( komihash_stream_t* const ctx, 972 | const void* const Msg0, size_t MsgLen ) KOMIHASH_NOEX 973 | { 974 | const uint8_t* Msg = (const uint8_t*) Msg0; 975 | 976 | const uint8_t* SwMsg = Msg; 977 | size_t SwMsgLen = 0; 978 | size_t BufFill = ctx -> BufFill; 979 | 980 | if( BufFill + MsgLen >= KOMIHASH_BUFSIZE && BufFill != 0 ) 981 | { 982 | const size_t CopyLen = KOMIHASH_BUFSIZE - BufFill; 983 | memcpy( ctx -> Buf + BufFill, Msg, CopyLen ); 984 | BufFill = 0; 985 | 986 | SwMsg += CopyLen; 987 | SwMsgLen = MsgLen - CopyLen; 988 | 989 | Msg = ctx -> Buf; 990 | MsgLen = KOMIHASH_BUFSIZE; 991 | } 992 | 993 | if( BufFill == 0 ) 994 | { 995 | while( MsgLen > 127 ) 996 | { 997 | uint64_t Seed1, Seed2, Seed3, Seed4; 998 | uint64_t Seed5, Seed6, Seed7, Seed8; 999 | 1000 | KOMIHASH_PREFETCH( Msg ); 1001 | 1002 | if( ctx -> IsHashing ) 1003 | { 1004 | Seed1 = ctx -> Seed[ 0 ]; 1005 | Seed2 = ctx -> Seed[ 1 ]; 1006 | Seed3 = ctx -> Seed[ 2 ]; 1007 | Seed4 = ctx -> Seed[ 3 ]; 1008 | Seed5 = ctx -> Seed[ 4 ]; 1009 | Seed6 = ctx -> Seed[ 5 ]; 1010 | Seed7 = ctx -> Seed[ 6 ]; 1011 | Seed8 = ctx -> Seed[ 7 ]; 1012 | } 1013 | else 1014 | { 1015 | ctx -> IsHashing = 1; 1016 | 1017 | const uint64_t UseSeed = ctx -> Seed[ 0 ]; 1018 | Seed1 = KOMIHASH_IVAL1 ^ ( UseSeed & KOMIHASH_VAL01 ); 1019 | Seed5 = KOMIHASH_IVAL5 ^ ( UseSeed & KOMIHASH_VAL10 ); 1020 | 1021 | KOMIHASH_HASHROUND(); 1022 | 1023 | Seed2 = KOMIHASH_IVAL2 ^ Seed1; 1024 | Seed3 = KOMIHASH_IVAL3 ^ Seed1; 1025 | Seed4 = KOMIHASH_IVAL4 ^ Seed1; 1026 | Seed6 = KOMIHASH_IVAL6 ^ Seed5; 1027 | Seed7 = KOMIHASH_IVAL7 ^ Seed5; 1028 | Seed8 = KOMIHASH_IVAL8 ^ Seed5; 1029 | } 1030 | 1031 | KOMIHASH_HASHLOOP64(); 1032 | 1033 | ctx -> Seed[ 0 ] = Seed1; 1034 | ctx -> Seed[ 1 ] = Seed2; 1035 | ctx -> Seed[ 2 ] = Seed3; 1036 | ctx -> Seed[ 3 ] = Seed4; 1037 | ctx -> Seed[ 4 ] = Seed5; 1038 | ctx -> Seed[ 5 ] = Seed6; 1039 | ctx -> Seed[ 6 ] = Seed7; 1040 | ctx -> Seed[ 7 ] = Seed8; 1041 | 1042 | if( SwMsgLen == 0 ) 1043 | { 1044 | if( MsgLen != 0 ) 1045 | { 1046 | break; 1047 | } 1048 | 1049 | ctx -> BufFill = 0; 1050 | return; 1051 | } 1052 | 1053 | Msg = SwMsg; 1054 | MsgLen = SwMsgLen; 1055 | SwMsgLen = 0; 1056 | } 1057 | } 1058 | 1059 | ctx -> BufFill = BufFill + MsgLen; 1060 | uint8_t* op = ctx -> Buf + BufFill; 1061 | 1062 | while( MsgLen != 0 ) 1063 | { 1064 | *op = *Msg; 1065 | Msg++; 1066 | op++; 1067 | MsgLen--; 1068 | } 1069 | } 1070 | 1071 | /** 1072 | * @brief Function finalizes the streamed "komihash" hashing session. 1073 | * 1074 | * Returns the resulting hash value of the previously hashed data. This value 1075 | * is equal to the value returned by the komihash() function for the same 1076 | * overall provided data. 1077 | * 1078 | * Note that since this function is non-destructive to the context structure, 1079 | * the function can be used to obtain intermediate hashes of the data stream 1080 | * being hashed, and the hashing can then be resumed. 1081 | * 1082 | * @param[in] ctx Pointer to the context structure. The structure must be 1083 | * initialized via the komihash_stream_init() function. 1084 | * @return 64-bit hash value. Should be endianness-corrected when this value 1085 | * is shared between big- and little-endian systems. 1086 | */ 1087 | 1088 | KOMIHASH_INLINE uint64_t komihash_stream_final( komihash_stream_t* const ctx ) 1089 | KOMIHASH_NOEX 1090 | { 1091 | const uint8_t* Msg = ctx -> Buf; 1092 | size_t MsgLen = ctx -> BufFill; 1093 | 1094 | if( ctx -> IsHashing == 0 ) 1095 | { 1096 | return( komihash( Msg, MsgLen, ctx -> Seed[ 0 ])); 1097 | } 1098 | 1099 | ctx -> pb[ 4 ] = 0; 1100 | ctx -> pb[ 5 ] = 0; 1101 | ctx -> pb[ 6 ] = 0; 1102 | ctx -> pb[ 7 ] = 0; 1103 | 1104 | uint64_t Seed1 = ctx -> Seed[ 0 ]; 1105 | uint64_t Seed2 = ctx -> Seed[ 1 ]; 1106 | uint64_t Seed3 = ctx -> Seed[ 2 ]; 1107 | uint64_t Seed4 = ctx -> Seed[ 3 ]; 1108 | uint64_t Seed5 = ctx -> Seed[ 4 ]; 1109 | uint64_t Seed6 = ctx -> Seed[ 5 ]; 1110 | uint64_t Seed7 = ctx -> Seed[ 6 ]; 1111 | uint64_t Seed8 = ctx -> Seed[ 7 ]; 1112 | 1113 | if( MsgLen > 63 ) 1114 | { 1115 | KOMIHASH_HASHLOOP64(); 1116 | } 1117 | 1118 | Seed5 ^= Seed6 ^ Seed7 ^ Seed8; 1119 | Seed1 ^= Seed2 ^ Seed3 ^ Seed4; 1120 | 1121 | return( komihash_epi( Msg, MsgLen, Seed1, Seed5 )); 1122 | } 1123 | 1124 | /** 1125 | * @brief FOR TESTING PURPOSES ONLY - use the komihash() function instead. 1126 | * 1127 | * @param Msg The message to produce a hash from. 1128 | * @param MsgLen Message's length, in bytes. 1129 | * @param UseSeed Seed to use. 1130 | * @return 64-bit hash value. 1131 | */ 1132 | 1133 | KOMIHASH_INLINE uint64_t komihash_stream_oneshot( const void* const Msg, 1134 | const size_t MsgLen, const uint64_t UseSeed ) KOMIHASH_NOEX 1135 | { 1136 | komihash_stream_t ctx; 1137 | 1138 | komihash_stream_init( &ctx, UseSeed ); 1139 | komihash_stream_update( &ctx, Msg, MsgLen ); 1140 | 1141 | return( komihash_stream_final( &ctx )); 1142 | } 1143 | 1144 | #if defined( KOMIHASH_NS ) 1145 | 1146 | } // namespace KOMIHASH_NS 1147 | 1148 | #if !defined( KOMIHASH_NS_CUSTOM ) 1149 | 1150 | namespace { 1151 | 1152 | using KOMIHASH_NS :: komihash; 1153 | using KOMIHASH_NS :: komirand; 1154 | using KOMIHASH_NS :: komihash_stream_t; 1155 | using KOMIHASH_NS :: komihash_stream_init; 1156 | using KOMIHASH_NS :: komihash_stream_update; 1157 | using KOMIHASH_NS :: komihash_stream_final; 1158 | using KOMIHASH_NS :: komihash_stream_oneshot; 1159 | 1160 | } // namespace 1161 | 1162 | #endif // !defined( KOMIHASH_NS_CUSTOM ) 1163 | 1164 | #endif // defined( KOMIHASH_NS ) 1165 | 1166 | // Defines for Doxygen. 1167 | 1168 | #if !defined( KOMIHASH_NS_CUSTOM ) 1169 | #define KOMIHASH_NS_CUSTOM 1170 | #endif // !defined( KOMIHASH_NS_CUSTOM ) 1171 | 1172 | #undef KOMIHASH_NS_CUSTOM 1173 | #undef KOMIHASH_U64_C 1174 | #undef KOMIHASH_NOEX 1175 | #undef KOMIHASH_IVAL1 1176 | #undef KOMIHASH_IVAL2 1177 | #undef KOMIHASH_IVAL3 1178 | #undef KOMIHASH_IVAL4 1179 | #undef KOMIHASH_IVAL5 1180 | #undef KOMIHASH_IVAL6 1181 | #undef KOMIHASH_IVAL7 1182 | #undef KOMIHASH_IVAL8 1183 | #undef KOMIHASH_VAL01 1184 | #undef KOMIHASH_VAL10 1185 | #undef KOMIHASH_COND_EC 1186 | #undef KOMIHASH_ICC_GCC 1187 | #undef KOMIHASH_GCC_BUILTINS 1188 | #undef KOMIHASH_BMI2 1189 | #undef KOMIHASH_EC32 1190 | #undef KOMIHASH_LIKELY 1191 | #undef KOMIHASH_UNLIKELY 1192 | #undef KOMIHASH_PREFETCH 1193 | #undef KOMIHASH_INLINE 1194 | #undef KOMIHASH_INLINE_F 1195 | #undef KOMIHASH_HASHROUND 1196 | #undef KOMIHASH_HASH16 1197 | #undef KOMIHASH_HASHFIN 1198 | #undef KOMIHASH_HASHLOOP64 1199 | 1200 | #endif // KOMIHASH_INCLUDED 1201 | --------------------------------------------------------------------------------