├── LICENSE.md ├── Makefile ├── README.md ├── apps ├── hash-cpp │ ├── Makefile │ └── main.c ├── hash-perf │ ├── Makefile │ └── main-hperf.c └── hash-stats │ ├── Makefile │ └── main-hstats.c └── lib └── librte_tch_hash ├── Makefile ├── jenkins_lookup3.c ├── rte_cmp_x86_v1604.h ├── rte_cmp_x86_v1702.h ├── rte_cuckoo_hash_bloom.c ├── rte_cuckoo_hash_cond.c ├── rte_cuckoo_hash_horton.c ├── rte_cuckoo_hash_lazy_bloom.c ├── rte_cuckoo_hash_lazy_cond.c ├── rte_cuckoo_hash_lazy_no.c ├── rte_cuckoo_hash_lazy_uncond.c ├── rte_cuckoo_hash_template.c ├── rte_cuckoo_hash_uncond.c ├── rte_cuckoo_hash_v1604.c ├── rte_cuckoo_hash_v1702.c ├── rte_cuckoo_hash_v1702.h ├── rte_cuckoo_hash_x86_v1702.h ├── rte_hash64.h ├── rte_hash_bloom.h ├── rte_hash_commons.h ├── rte_hash_cond.h ├── rte_hash_horton.h ├── rte_hash_lazy_bloom.h ├── rte_hash_lazy_cond.h ├── rte_hash_lazy_no.h ├── rte_hash_lazy_uncond.h ├── rte_hash_template.h ├── rte_hash_uncond.h ├── rte_hash_v1604.h ├── rte_hash_v1702.h ├── rte_tch_hash.h ├── rte_tch_utils.h └── rte_tchh_structs.h /LICENSE.md: -------------------------------------------------------------------------------- 1 | Clear BSD license 2 | 3 | Copyright (c) 2018 – Thomson Licensing, SAS 4 | 5 | Redistribution and use in source and binary forms, with or without modification, 6 | are permitted (subject to the limitations in the disclaimer below) provided that 7 | the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of Thomson Licensing, or Technicolor, nor the names of its 17 | contributors may be used to endorse or promote products derived from this 18 | software without specific prior written permission. 19 | 20 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY THIS 21 | LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 23 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 25 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 27 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 28 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 29 | TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 30 | THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: __apps 2 | 3 | __apps : __hash_perf __hash_stats __hash_cpp 4 | 5 | __libs: 6 | make -f lib/librte_tch_hash/Makefile S=lib/librte_tch_hash O=build 7 | 8 | __hash_perf: __libs 9 | make -f apps/hash-perf/Makefile S=apps/hash-perf O=build EXTRA_CFLAGS="-I$(CURDIR)/build/include -L$(CURDIR)/build/lib" EXTRA_CPPFLAGS=-I$(CURDIR)/build/include EXTRA_LDFLAGS="-L$(CURDIR)/build/lib -lrte_tch_hash -lm" V=1 10 | 11 | __hash_stats: __libs 12 | make -f apps/hash-stats/Makefile S=apps/hash-stats O=build EXTRA_CFLAGS="-I$(CURDIR)/build/include -L$(CURDIR)/build/lib" EXTRA_CPPFLAGS=-I$(CURDIR)/build/include EXTRA_LDFLAGS="-L$(CURDIR)/build/lib -lrte_tch_hash -lm" V=1 13 | 14 | __hash_cpp: __libs 15 | make -f apps/hash-cpp/Makefile S=apps/hash-cpp O=build EXTRA_CXXFLAGS="-I$(CURDIR)/build/include -L$(CURDIR)/build/lib" EXTRA_CPPFLAGS=-I$(CURDIR)/build/include EXTRA_LDFLAGS="-L$(CURDIR)/build/lib -lrte_tch_hash -lm" V=1 16 | 17 | clean: 18 | rm -rf build 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Cuckoo++ 2 | ======== 3 | Cuckoo++ is a high-performance hash table implementation targetted at network packet processing applications. The hash table implementation provides highly-optimized code for batched lookups, typical of packet processing applications. Indeed, when a batch of keys is looked up in a hash table, it allows for specific optimization (prefetching,...) that significantly improve performance. It further implements algorithmical optimization described in [Cuckoo++ Hash Tables: High-Performance Hash Tables for Networking Applications, Nicolas Le Scouarnec, IEEE/ACM ANCS, 2018](https://dl.acm.org/citation.cfm?doid=3230718.3232629). If you are interested in design decisions (e.g., using SSE vs AVX, adding a bloom filter, having 8 slots per bucket) or performance evaluation, you are invited to refer to the aforementionned paper. 4 | 5 | If you use this work, please cite: 6 | > **Cuckoo++ Hash Tables: High-Performance Hash Tables for Networking Applications** 7 | > Nicolas Le Scouarnec 8 | > *ACM/IEEE Symposium on Architectures for Networking and Communications Systems (ANCS)* 9 | > July 2018 10 | > https://doi.org/10.1145/3230718.3232629 11 | 12 | License 13 | ======= 14 | Most of the code of Cuckoo++ is licensed under Clear BSD license (see LICENSE.md) and copyrighted by Thomson Licensing. It contains portions of code from DPDK, which are licensed under BSD license and copyrighted by others as stated in file headers. 15 | 16 | Building and testing 17 | ==================== 18 | You need to have the tools and libraries listed as dependencies of DPDK (see [DPDK documentation](http://doc.dpdk.org/guides/linux_gsg/sys_reqs.html#compilation-of-the-dpdk)). Then, you must get a copy of DPDK (preferably version 17.08) and compile it. You can then compile the additional library and example programs. It has been tested on Ubuntu 16.04 and 18.04 but should work on other distributions and versions supported by DPDK. 19 | 20 | ``` 21 | # Export variables necessary to compilation 22 | mkdir ~/dpdk-cuckoopp 23 | cd ~/dpdk-cuckoopp 24 | export RTE_SDK=~/dpdk-cuckoopp/dpdk 25 | export RTE_TARGET=x86_64-native-linuxapp-gcc 26 | 27 | # Get and compile DPDK 28 | git clone --branch v17.08 git://dpdk.org/dpdk 29 | make config T=x86_64-native-linuxapp-gcc 30 | make 31 | 32 | # Get and compile Cuckoo++ (depends on DPDK) 33 | git clone https://github.com/technicolor-research/cuckoopp.git 34 | make 35 | ``` 36 | 37 | Built applications are available in `build/apps/`. These are DPDK applications and thus require hugepages and usual DPDK arguments (see [DPDK documentation](http://doc.dpdk.org/guides/linux_gsg/sys_reqs.html#running-dpdk-applications)). As an example, one can reserve hugepages, make them available and run the hash_perf application with the following commands. Do not forget: **(i) you need hugepages (check `/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages`)** and **(ii) you must run the program as root**. 38 | 39 | ``` 40 | # All commands (including the app) must be run as root (super user) 41 | # Reserve hugepages (can also be done at boot time or reserve 1GB hugepages) 42 | echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages 43 | mkdir /mnt/huge 44 | mount -t hugetlbfs nodev /mnt/huge 45 | 46 | # Measure the performance for a hash-table of capacity of 1000000 entries and load factor of 0.9 47 | cd build/app 48 | ./hash-perf --lcores 0@0 --socket-mem 1024,0 -w99:0.0 -- 1000000 0.9 49 | ``` 50 | 51 | For optimal performance, additional tunning may be needed such as using 1G hugepages, isolating CPU cores, configuring Memory Snoop Mode. Settings for optimal performance are described in DPDK documentation. 52 | 53 | Benckmarking 54 | ============ 55 | The benchmarking application used for the paper on Cuckoo++ is `hash-cpp`. An example usage is the following: 56 | ``` 57 | sudo build/hash_cpp -- -c 500000 -l 0.8 -i 0.5 BLOOM 58 | ``` 59 | 60 | DPDK arguments come before the -- 61 | Application arguments come after the -- 62 | * _-c_ gives the hash table capacity 63 | * _-l_ gives the load factor 64 | * _-i_ gives the fraction of negative lookups 65 | * _-t_ can be used to test multiple threads 66 | * _BLOOM_ is the implementation to benchmark 67 | 68 | The available implementations are 69 | * **DPDK_1604**, almost vanilla DPDK 16.04 implementation changed to support 128 bit values. 70 | * **DPDK_1702**, almost vanilla DPDK 16.04 implementation changed to support 128 bit values. 71 | * **COND**, our optimized implementation with an optimistic prefetching strategy 72 | * **UNCOND**, our optimized implementation with a pessimistic prefetching strategy 73 | * **BLOOM**, our Cuckoo++ implementation (which should be used most of the time) 74 | * **HORTON**, our own optimized implementation of Horton tables for CPUs 75 | * **LAZY_BLOOM, LAZY_COND, LAZY_UNCOND**, same as previous but with builtin timers. 76 | 77 | _Note: the program must be run as root and there must be sufficient available memory (hugepages) on all sockets (especially if you benchmark larger hash tables)._ 78 | 79 | Using it in your application 80 | ============================ 81 | Cuckoo++ is meant to be used, as is, in DPDK-based applications. Hence, it depends on DPDK (see building section). Yet, it could easily be ported to plain C. 82 | 83 | The API is relatively close to the original hash-table from DPDK and it could be used as a drop-in replacement in numerous cases. The main limit is that Cuckoo++ was developped for an application that uses sharding to the core (see [Krononat - USENIX ATC 2018](https://www.usenix.org/conference/atc18/presentation/andre)) and thus does not implement concurrent access to the hash-table from multiple threads. 84 | 85 | It however implements additional features such as built-in timers (described in [Cuckoo++ Hash Tables - arXiv 2017](https://arxiv.org/abs/1712.09624)) and iterators (described in [Krononat - USENIX ATC 2018](https://www.usenix.org/conference/atc18/presentation/andre)). 86 | 87 | This library implements several highly-optimized variants (Vanilla "Pessimistic" Cuckoo Hash-Table, Vanilla "Optimistic" Cuckoo Hash-Table, **Cuckoo++ Hash Tables**, and our implementation of [Horton Hash Tables](https://www.usenix.org/conference/atc16/technical-sessions/presentation/breslow) for CPUs.). Furthermore, all implementations exists with or without built-in entry expiration (lazy variants). The performance and benefits of all variants are discussed in [Cuckoo++ - ANCS 2018]((https://dl.acm.org/citation.cfm?doid=3230718.3232629)). 88 | 89 | Using this library requires to choose the implementation you want to use; we recommand Cuckoo++ as it achieves excellent performance for all workloads. This choice can be done, either at compile time by including the header of the desired implementation (e.g., `rte_cuckoo_hash_bloom.h` for a Cuckoo++ hash-table), or at runtime by including `rte_tch_hash.h` which allows selecting the implementation. Example usages of `rte_tch_hash.h` are provided in `apps/hash-perf` and `apps/hash-stats`. 90 | 91 | The API is described in `lib/librte_tch_hash/rte_hash_template.h`. 92 | 93 | The available implementations are 94 | * **COND**, our optimized implementation with an optimistic prefetching strategy 95 | * **UNCOND**, our optimized implementation with a pessimistic prefetching strategy 96 | * **BLOOM**, our Cuckoo++ implementation (which should be used most of the time) 97 | * **HORTON**, our own optimized implementation of Horton tables for CPUs 98 | * **LAZY_BLOOM, LAZY_COND, LAZY_UNCOND**, same as previous but with builtin timers. 99 | 100 | 101 | References 102 | ========== 103 | > **Cuckoo++ Hash Tables: High-Performance Hash Tables for Networking Applications** 104 | > Nicolas Le Scouarnec 105 | > *ACM/IEEE Symposium on Architectures for Networking and Communications Systems (ANCS)* 106 | > July 2018 107 | > https://doi.org/10.1145/3230718.3232629 108 | 109 | > **Don't share, Don't lock: Large-scale Software Connection Tracking with Krononat** 110 | > Fabien André, Stéphane Gouache, Nicolas Le Scouarnec, and Antoine Monsifrot 111 | > *USENIX ATC'18 - USENIX Annual Technical Conference* 112 | > July 2018 113 | > https://doi.org/10.1145/3230718.3232629 114 | 115 | > **Cuckoo++ Hash Tables: High-Performance Hash Tables for Networking Applications** 116 | > **_(Technical report that describes built-in timers)_** 117 | > Nicolas Le Scouarnec 118 | > *arXiv 1712.09624* 119 | > December 2017 120 | > https://doi.org/10.1145/3230718.3232629 -------------------------------------------------------------------------------- /apps/hash-cpp/Makefile: -------------------------------------------------------------------------------- 1 | # Modifications by Thomson Licensing licensed under Clear BSD License. 2 | # Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | # 4 | # Original license: 5 | # BSD LICENSE 6 | # 7 | # Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 8 | # All rights reserved. 9 | # 10 | # Redistribution and use in source and binary forms, with or without 11 | # modification, are permitted provided that the following conditions 12 | # are met: 13 | # 14 | # * Redistributions of source code must retain the above copyright 15 | # notice, this list of conditions and the following disclaimer. 16 | # * Redistributions in binary form must reproduce the above copyright 17 | # notice, this list of conditions and the following disclaimer in 18 | # the documentation and/or other materials provided with the 19 | # distribution. 20 | # * Neither the name of Intel Corporation nor the names of its 21 | # contributors may be used to endorse or promote products derived 22 | # from this software without specific prior written permission. 23 | # 24 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 27 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 28 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 30 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 | 36 | ifeq ($(RTE_SDK),) 37 | $(error "Please define RTE_SDK environment variable") 38 | endif 39 | 40 | # Default target, can be overriden by command line or environment 41 | RTE_TARGET ?= x86_64-native-linuxapp-gcc 42 | 43 | include $(RTE_SDK)/mk/rte.vars.mk 44 | 45 | # binary name 46 | APP = hash_cpp 47 | 48 | # all source are stored in SRCS-y 49 | SRCS-y := main.c 50 | 51 | # CFLAGS += -I$(SRCDIR) 52 | # CFLAGS += -O3 $(USER_FLAGS) $(TCH_FLAGS) 53 | # CFLAGS += $(WERROR_FLAGS) 54 | 55 | # Hack to force DPDK to compile a C++ file (note that main.c despite its extension is C++) 56 | CC=g++ 57 | 58 | CFLAGS += $(EXTRA_CXXFLAGS) -std=c++14 59 | LDFLAGS += -lstdc++ 60 | 61 | # this application needs libraries first 62 | DEPDIRS-y += lib drivers 63 | 64 | include $(RTE_SDK)/mk/rte.extapp.mk 65 | -------------------------------------------------------------------------------- /apps/hash-cpp/main.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | extern "C" { 30 | #include 31 | } 32 | 33 | /* Pattern sizes */ 34 | const int batch_size = 32; 35 | const int pattern_prime = 6131; 36 | const unsigned pattern_size = batch_size * pattern_prime; 37 | 38 | /* Keys generation and management */ 39 | struct key_list { 40 | 41 | key_list() : 42 | keys(nullptr, rte_free), key_count(0) { 43 | } 44 | ; 45 | 46 | key_list(unsigned key_count_, int socked_id) : 47 | keys( 48 | reinterpret_cast(rte_malloc_socket(nullptr, 49 | key_count_ * sizeof(hash_key_t), 50 | RTE_CACHE_LINE_SIZE, socked_id)), rte_free), key_count( 51 | 0) { 52 | if(keys.get() == nullptr) { 53 | fprintf(stderr, "Could not allocate %lu bytes on socket %d for key list\n", 54 | key_count_ * sizeof(hash_key_t), socked_id); 55 | exit(1); 56 | } 57 | } 58 | ; 59 | 60 | std::unique_ptr keys; 61 | unsigned key_count; 62 | }; 63 | 64 | void keys_random_fill(hash_key_t keys[], unsigned key_count, 65 | std::mt19937_64& rnd, float non_zero_percent = 1.0f) { 66 | unsigned key_i; 67 | unsigned non_zero_count = key_count * non_zero_percent; 68 | for (key_i = 0; key_i < non_zero_count; ++key_i) { 69 | keys[key_i].a = rnd(); 70 | keys[key_i].b = rnd(); 71 | } 72 | for (; key_i < key_count; ++key_i) { 73 | keys[key_i].a = 0; 74 | keys[key_i].b = 0; 75 | } 76 | if(non_zero_percent != 1.0f) { 77 | std::shuffle(keys, keys + key_count, rnd); 78 | } 79 | } 80 | 81 | /* Hash table operations */ 82 | template 83 | void hashtable_insert(rte_tch_hash& hash, 84 | hash_key_t keys[], unsigned key_count, 85 | __rte_unused hash_key_t patterns[], __rte_unused unsigned op_count, float& success_rate) { 86 | 87 | hash_data_t data; 88 | data.mm = _mm_set1_epi16(16); 89 | 90 | for (unsigned key_i = 0; key_i < key_count; ++key_i) { 91 | rte_tch_hash_add_key_data(Variant, &hash, keys[key_i], data, 16, 0); 92 | } 93 | success_rate = 1.0f; 94 | } 95 | 96 | typedef decltype(&hashtable_insert) hashtable_operation; 97 | 98 | volatile uint64_t global_hash = 0; 99 | 100 | template 101 | void hashtable_hash(__rte_unused rte_tch_hash& hash, 102 | hash_key_t keys[], unsigned key_count, 103 | __rte_unused hash_key_t patterns[], 104 | unsigned op_count, float& success_rate) { 105 | 106 | uint64_t hash_value = 0; 107 | unsigned key_i = 0; 108 | for(unsigned op_i = 0; op_i < op_count; ++op_i) { 109 | key_i++; 110 | if (key_i >= key_count) { 111 | key_i = 0; 112 | } 113 | hash_value += dcrc_hash_m128(keys[key_i]); 114 | } 115 | global_hash = hash_value; 116 | success_rate = 1.0f; 117 | } 118 | 119 | template 120 | void hashtable_lookup(rte_tch_hash& hash, hash_key_t keys[], unsigned key_count, 121 | __rte_unused hash_key_t patterns[], unsigned op_count, float& success_rate) { 122 | 123 | hash_data_t data; 124 | 125 | unsigned key_i = 0; 126 | unsigned patt_i = 0; 127 | unsigned success_op = 0; 128 | hash_key_t xored_key; 129 | 130 | for (unsigned op_i = 0; op_i < op_count; ++op_i) { 131 | key_i++; 132 | if (key_i >= key_count) { 133 | key_i = 0; 134 | } 135 | 136 | patt_i++; 137 | if (patt_i >= pattern_size) { 138 | patt_i = 0; 139 | } 140 | 141 | xored_key.mm = _mm_xor_si128(keys[key_i].mm, patterns[patt_i].mm); 142 | 143 | success_op += (rte_tch_hash_lookup_data(Variant, &hash, xored_key, 144 | &data, 0) >= 0); 145 | } 146 | success_rate = static_cast(success_op) / op_count; 147 | } 148 | 149 | template 150 | void hashtable_lookup_batch(rte_tch_hash& hash, hash_key_t keys[], 151 | unsigned key_count, hash_key_t patterns[], unsigned op_count, 152 | float& success_rate) { 153 | 154 | hash_data_t data[BatchSize]; 155 | hash_key_t xored_keys[BatchSize]; 156 | 157 | unsigned batch_i = 0; 158 | unsigned patt_i = 0; 159 | unsigned success_op = 0; 160 | for (unsigned op_i = 0; op_i < op_count; op_i += BatchSize) { 161 | 162 | batch_i += BatchSize; 163 | if (batch_i + BatchSize > key_count) { 164 | batch_i = 0; 165 | } 166 | 167 | patt_i += BatchSize; 168 | if (patt_i + BatchSize > pattern_size) { 169 | patt_i = 0; 170 | } 171 | 172 | const hash_key_t* keys_batch = keys + batch_i; 173 | const hash_key_t* patt_batch = patterns + patt_i; 174 | 175 | for (int i = 0; i < BatchSize; ++i) { 176 | xored_keys[i].mm = _mm_xor_si128(keys_batch[i].mm, patt_batch[i].mm); 177 | } 178 | 179 | uint64_t hits; 180 | rte_tch_hash_lookup_bulk_data(Variant, &hash, xored_keys, BatchSize, 181 | &hits, data, 0); 182 | success_op += __builtin_popcount(hits); 183 | } 184 | success_rate = static_cast(success_op) / op_count; 185 | } 186 | 187 | 188 | /* Benchmark functions */ 189 | 190 | struct core_loop_args { 191 | float op_cycles; 192 | float op_rate; 193 | float success_rate; 194 | hashtable_operation op_func; 195 | const key_list* list; 196 | const key_list* patterns; 197 | rte_tch_hash* hash; 198 | int key_count; 199 | int op_count; 200 | }; 201 | 202 | int core_loop(void* args) { 203 | core_loop_args* core_args = static_cast(args); 204 | uint64_t startq = rte_get_tsc_cycles(); 205 | core_args->op_func(*(core_args->hash), core_args->list->keys.get(), 206 | core_args->key_count, core_args->patterns->keys.get(), 207 | core_args->op_count, core_args->success_rate); 208 | uint64_t endq = rte_get_tsc_cycles(); 209 | float diff_tsc = endq - startq; 210 | core_args->op_cycles = diff_tsc / static_cast(core_args->op_count); 211 | core_args->op_rate = static_cast(core_args->op_count) 212 | / (diff_tsc / rte_get_tsc_hz()); 213 | return 0; 214 | } 215 | 216 | struct bench_desc { 217 | const char* implem_name; 218 | const char* op_name; 219 | float load_factor; 220 | unsigned capacity; 221 | }; 222 | 223 | static void run_bench_multicore(hashtable_operation op_func, 224 | const key_list list[], const key_list patterns[], rte_tch_hash* hash[], 225 | const bench_desc& desc, int core_count, 226 | unsigned key_count, unsigned op_count, std::ostream& outstream) { 227 | 228 | std::unique_ptr core_args(new core_loop_args[core_count]); 229 | 230 | // Prepare arguments 231 | for (int core_i = 0; core_i < core_count; ++core_i) { 232 | core_args[core_i].op_cycles = 0; 233 | core_args[core_i].op_rate = 0; 234 | core_args[core_i].success_rate = 0; 235 | core_args[core_i].op_func = op_func; 236 | core_args[core_i].list = list + core_i; 237 | core_args[core_i].patterns = patterns + core_i; 238 | core_args[core_i].hash = hash[core_i]; 239 | core_args[core_i].key_count = key_count; 240 | core_args[core_i].op_count = op_count; 241 | } 242 | 243 | // Launch on every slave core 244 | int core_i = 0; 245 | const int skip_master = 1; 246 | const int wrap = 0; 247 | int lcore_id = rte_get_next_lcore(-1, skip_master, wrap); 248 | for (; core_i < core_count && lcore_id != RTE_MAX_LCORE; ++core_i) { 249 | rte_eal_remote_launch(core_loop, 250 | static_cast(core_args.get() + core_i), lcore_id); 251 | lcore_id = rte_get_next_lcore(lcore_id, skip_master, wrap); 252 | } 253 | 254 | 255 | // If needed, lauch on master core 256 | if(core_i == core_count - 2) { 257 | core_loop(static_cast(core_args.get() + core_count - 1)); 258 | } 259 | 260 | // Wait for slaves 261 | core_i = 0; 262 | lcore_id = rte_get_next_lcore(-1, skip_master, wrap); 263 | for(; core_i < core_count && lcore_id != RTE_MAX_LCORE; ++core_i) { 264 | if (rte_eal_wait_lcore(lcore_id) < 0) { 265 | rte_exit(EXIT_FAILURE, "One thread failed\n"); 266 | } 267 | lcore_id = rte_get_next_lcore(lcore_id, skip_master, wrap); 268 | } 269 | 270 | // Compute statistics 271 | double rate_avg = 0; 272 | double cycles_avg = 0; 273 | double sucess_avg = 0; 274 | 275 | for(core_i = 0; core_i < core_count; ++core_i) { 276 | cycles_avg += core_args[core_i].op_cycles; 277 | rate_avg += core_args[core_i].op_rate; 278 | sucess_avg += core_args[core_i].success_rate; 279 | } 280 | cycles_avg /= core_count; 281 | rate_avg /= core_count; 282 | sucess_avg /= core_count; 283 | 284 | // Display statistics 285 | outstream << desc.implem_name << "," << desc.op_name << "," << desc.capacity 286 | << "," << desc.load_factor << "," << core_count << "," << sucess_avg 287 | << "," << cycles_avg << "," << rate_avg << "\n"; 288 | /* printf("%s,%s,%d,%f,%f,%f\n", implem_name, op_name, core_count, sucess_avg, 289 | cycles_avg, rate_avg);*/ 290 | } 291 | 292 | struct cmdargs { 293 | rte_tch_hash_variants implementation; 294 | std::vector capacities; 295 | std::vector load_factors; 296 | std::vector core_counts; 297 | std::vector unsucessful_rate; 298 | float max_unsucessful_rate; 299 | std::string outfile; 300 | }; 301 | 302 | const unsigned hash_op_count = 10000000; 303 | const unsigned lookup_op_count = 10000000; 304 | const unsigned lookup_batch_op_count = 100000000; 305 | 306 | template 307 | static void make_hash(rte_tch_hash* hash[], unsigned capacity, int core_i, int lcore_id) { 308 | rte_tch_hash_parameters nat_hash_params = { 309 | .entries = capacity, 310 | .socket_id = static_cast(rte_lcore_to_socket_id(lcore_id)) 311 | }; 312 | hash[core_i] = rte_tch_hash_create(Variant, &nat_hash_params); 313 | if (hash == nullptr) { 314 | fprintf(stderr, 315 | "Failed to init hashtable for lcore_id %d on socket_id %d - Check that CPU and memory are on the same socket\n", 316 | lcore_id, rte_lcore_to_socket_id(lcore_id)); 317 | } 318 | } 319 | 320 | template 321 | void benchmark_for_size(const cmdargs& args, 322 | key_list list[], key_list patterns[], 323 | int max_cores, 324 | std::mt19937_64& rnd, 325 | unsigned capacity, float load_factor, 326 | std::ostream& outstream) { 327 | 328 | unsigned size = capacity * load_factor; 329 | 330 | 331 | fprintf(stderr, "-- Benchmark: capacity=%d load=%f size=%d\n", capacity, load_factor, size); 332 | 333 | 334 | for(int core_count: args.core_counts) { 335 | 336 | key_list* shifted_patterns = patterns; 337 | 338 | // Init. hash tables 339 | std::unique_ptr hash(new rte_tch_hash*[core_count]); 340 | 341 | int core_i = 0; 342 | const int skip_master = 1; 343 | const int wrap = 0; 344 | int lcore_id = rte_get_next_lcore(-1, skip_master, wrap); 345 | for(; core_i < core_count && lcore_id != RTE_MAX_LCORE; ++core_i) { 346 | make_hash(hash.get(), capacity, core_i, lcore_id); 347 | lcore_id = rte_get_next_lcore(lcore_id, skip_master, 0); 348 | } 349 | 350 | if (core_i < core_count - 2) { 351 | fprintf(stderr, 352 | "Not enough available cores to run benchmarks on %d cores\n", 353 | core_count); 354 | exit(1); 355 | } else if (core_i == core_count - 2) { 356 | ++core_i; 357 | lcore_id = rte_get_master_lcore(); 358 | make_hash(hash.get(), capacity, core_i, lcore_id); 359 | } 360 | 361 | bench_desc desc; 362 | desc.implem_name = rte_tch_hash_str(args.implementation); 363 | desc.load_factor = load_factor; 364 | desc.capacity = capacity; 365 | 366 | // Do benchmarks 367 | desc.op_name = "hash"; 368 | run_bench_multicore(hashtable_hash, list, shifted_patterns, 369 | hash.get(), desc, core_count, size, hash_op_count, outstream); 370 | desc.op_name = "insert"; 371 | run_bench_multicore(hashtable_insert, list, shifted_patterns, 372 | hash.get(), desc, core_count, size, size, outstream); 373 | 374 | // Shuffle key lists for lookups 375 | for(core_i = 0; core_i < core_count; ++core_i) { 376 | //fprintf(stderr, "Shuffle list core %d/%d %p\n", core_i, core_count, list[core_i].keys.get()); 377 | std::shuffle(list[core_i].keys.get(), 378 | list[core_i].keys.get() + size, rnd); 379 | } 380 | 381 | for (float rate : args.unsucessful_rate) { 382 | fprintf(stderr, "Invalid rate: %f %d\n", rate, size); 383 | desc.op_name = "lookup"; 384 | run_bench_multicore(hashtable_lookup, list, 385 | shifted_patterns, hash.get(), desc, core_count, size, 386 | lookup_op_count, outstream); 387 | desc.op_name = "lookup_batch_32"; 388 | run_bench_multicore(hashtable_lookup_batch, list, 389 | shifted_patterns, hash.get(), desc, core_count, size, 390 | lookup_batch_op_count, outstream); 391 | shifted_patterns += max_cores; 392 | } 393 | 394 | // Check integrity 395 | for(core_i = 0; core_i < core_count; ++core_i) { 396 | fprintf(stderr, "Checking for lcore_id %d\n", core_i); 397 | //rte_tch_hash_print_stats(Variant, hash[core_i], 0); 398 | rte_tch_hash_check_integrity(Variant, hash[core_i], 0); 399 | } 400 | 401 | // Reset hash 402 | for(core_i = 0; core_i < core_count; ++core_i) { 403 | rte_tch_hash_reset(Variant,hash[core_i]); 404 | } 405 | 406 | // Free structures 407 | for(core_i = 0; core_i < core_count; ++core_i) { 408 | rte_tch_hash_free(Variant,hash[core_i]); 409 | } 410 | } 411 | } 412 | 413 | template 414 | void benchmark(const cmdargs& args, std::ostream& outstream) { 415 | 416 | const int max_capacity = *std::max_element(args.capacities.begin(), 417 | args.capacities.end()); 418 | const float max_load_factor = *std::max_element(args.load_factors.begin(), 419 | args.load_factors.end()); 420 | const int max_size = max_capacity * max_load_factor; 421 | const int max_cores = *std::max_element(args.core_counts.begin(), 422 | args.core_counts.end()); 423 | 424 | fprintf(stderr, "max_capacity=%d, max_load_factor=%f, max_size=%d\n", 425 | max_capacity, max_load_factor, max_size); 426 | 427 | std::mt19937_64 rnd; 428 | 429 | // Generate key lists 430 | const int skip_master = 1; 431 | const int wrap = 0; 432 | int lcore_id = rte_get_next_lcore(-1, skip_master, wrap); 433 | std::unique_ptr lists(new key_list[max_cores]); 434 | for(int core_i = 0; core_i < max_cores && lcore_id != RTE_MAX_LCORE; ++core_i) { 435 | lists[core_i] = key_list(max_size, rte_lcore_to_socket_id(lcore_id)); 436 | //fprintf(stderr, "List %d %p\n", max_size, lists[core_i].keys.get()); 437 | keys_random_fill(lists[core_i].keys.get(), max_size, rnd); 438 | lcore_id = rte_get_next_lcore(lcore_id, skip_master, wrap); 439 | } 440 | 441 | // Generate patterns lists 442 | int un_count = args.unsucessful_rate.size(); 443 | std::unique_ptr patterns(new key_list[max_cores * un_count]); 444 | for (int un_i = 0; un_i < un_count; ++un_i) { 445 | lcore_id = rte_get_next_lcore(-1, skip_master, wrap); 446 | for(int core_i = 0; core_i < max_cores && lcore_id != RTE_MAX_LCORE; ++core_i) { 447 | patterns[un_i * max_cores + core_i] = key_list(pattern_size, 448 | rte_lcore_to_socket_id(lcore_id)); 449 | hash_key_t* patts = patterns[un_i * max_cores + core_i].keys.get(); 450 | keys_random_fill(patts, pattern_size, rnd, 451 | args.unsucessful_rate[un_i]); 452 | lcore_id = rte_get_next_lcore(lcore_id, skip_master, wrap); 453 | } 454 | } 455 | 456 | for (int cap : args.capacities) { 457 | for (float load_f : args.load_factors) { 458 | benchmark_for_size(args, lists.get(), patterns.get(), 459 | max_cores, rnd, cap, load_f, outstream); 460 | } 461 | } 462 | } 463 | 464 | void parse_float_list(char* str, std::vector& list, const float min, 465 | const float max, const char* param_name, const char* sep = ",") { 466 | char* tok = strtok(str, sep); 467 | list.push_back(std::stof(tok)); 468 | while ((tok = strtok(nullptr, sep)) != nullptr) { 469 | const float val = std::stof(tok); 470 | if (val > max || val < min) { 471 | fprintf(stderr, "Invalid %s: %f (%f-%f)\n", param_name, val, min, 472 | max); 473 | exit(1); 474 | } 475 | list.push_back(val); 476 | } 477 | } 478 | 479 | void parse_int_list(char* str, std::vector& list, const int min, 480 | const int max, const char* param_name, const char* sep = ",") { 481 | char* tok = strtok(str, sep); 482 | list.push_back(std::stoi(tok)); 483 | while ((tok = strtok(nullptr, sep)) != nullptr) { 484 | const int val = std::stoi(tok); 485 | if (val > max || val < min) { 486 | fprintf(stderr, "Invalid %s: %d (%d-%d)\n", param_name, val, min, 487 | max); 488 | exit(1); 489 | } 490 | list.push_back(val); 491 | } 492 | } 493 | 494 | rte_tch_hash_variants parse_implementation(const char* implem) { 495 | unsigned variants_count = sizeof(variants_names) 496 | / sizeof(variants_names[0]); 497 | for (unsigned i = 0; i < variants_count; ++i) { 498 | if (!strcmp(variants_names[i], implem)) { 499 | return static_cast(i); 500 | } 501 | } 502 | fprintf(stderr, "Invalid implementation: %s\n", implem); 503 | exit(1); 504 | return static_cast(0); 505 | } 506 | 507 | void usage() { 508 | fprintf(stderr, 509 | "Usage: hash-cpp [-c CAPACITY_LIST] [-l LOAD_FACTOR_LIST] " 510 | "[-t CORE_COUNT_LIST] [-i INVALID_LOOKUP_RATE_LIST] IMPLEMENTATION [OUT_FILE]\n"); 511 | } 512 | 513 | void parse_args(cmdargs& args, int argc, char* argv[]) { 514 | 515 | // Iterate over CLI arguments 516 | int opt; 517 | optind = 1; 518 | while ((opt = getopt(argc, argv, "c:l:t:i:")) != -1) { 519 | switch (opt) { 520 | case 'c': 521 | parse_int_list(optarg, args.capacities, 1, 1000000000, "capacity"); 522 | break; 523 | case 'l': 524 | parse_float_list(optarg, args.load_factors, 0.01, 1.0, 525 | "load factor"); 526 | break; 527 | case 't': 528 | parse_int_list(optarg, args.core_counts, 1, 1000, "core count"); 529 | break; 530 | case 'i': 531 | parse_float_list(optarg, args.unsucessful_rate, 0.0, 1.0, 532 | "lookup rate"); 533 | break; 534 | default: 535 | usage(); 536 | exit(1); 537 | } 538 | } 539 | 540 | // Default value for arguments 541 | if (args.capacities.empty()) { 542 | args.capacities.push_back(1000000); 543 | } 544 | 545 | if (args.load_factors.empty()) { 546 | args.load_factors.push_back(0.5f); 547 | } 548 | 549 | if (args.core_counts.empty()) { 550 | args.core_counts.push_back(1); 551 | } 552 | 553 | if (args.unsucessful_rate.empty()) { 554 | args.unsucessful_rate.push_back(0.0f); 555 | } 556 | 557 | if (argc - optind < 1 || argc - optind > 2) { 558 | usage(); 559 | exit(1); 560 | } 561 | 562 | args.implementation = parse_implementation(argv[optind]); 563 | 564 | // Open output stream 565 | if(argc - optind == 2) { 566 | args.outfile = argv[optind + 1]; 567 | } else { 568 | args.outfile = "-"; 569 | } 570 | fprintf(stderr, "Output file: %s\n", args.outfile.c_str()); 571 | } 572 | 573 | const char* header_str = "implementation,operation,capacity,load_factor,core_count,success_rate,op_cycles,op_rate"; 574 | 575 | int main(int argc, char* argv[]) { 576 | 577 | // Check if root 578 | if(getuid() != 0) { 579 | fprintf(stderr, "Please run this program as root\n"); 580 | exit(1); 581 | } 582 | 583 | // Init EAL 584 | int ret = rte_eal_init(argc, argv); 585 | if (ret < 0) { 586 | rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 587 | } 588 | argc -= ret; 589 | argv += ret; 590 | 591 | cmdargs args; 592 | parse_args(args, argc, argv); 593 | 594 | std::ostream* outstream; 595 | std::ofstream outfile; 596 | 597 | if(args.outfile == "-") { 598 | outstream = &(std::cout); 599 | fprintf(stderr, "Opened stdout\n"); 600 | } else { 601 | outfile.open(args.outfile, std::ofstream::out); 602 | fprintf(stderr, "Opened %s\n", args.outfile.c_str()); 603 | outstream = &(outfile); 604 | } 605 | 606 | switch(args.implementation) { 607 | case H_V1604: 608 | (*outstream) << header_str << "\n"; 609 | benchmark(args, *outstream); 610 | break; 611 | case H_V1702: 612 | (*outstream) << header_str << "\n"; 613 | benchmark(args, *outstream); 614 | break; 615 | case H_LAZY_BLOOM: 616 | (*outstream) << header_str << "\n"; 617 | benchmark(args, *outstream); 618 | break; 619 | case H_LAZY_COND: 620 | (*outstream) << header_str << "\n"; 621 | benchmark(args, *outstream); 622 | break; 623 | case H_LAZY_UNCOND: 624 | (*outstream) << header_str << "\n"; 625 | benchmark(args, *outstream); 626 | break; 627 | case H_LAZY_NO: 628 | (*outstream) << header_str << "\n"; 629 | benchmark(args, *outstream); 630 | break; 631 | case H_HORTON: 632 | (*outstream) << header_str << "\n"; 633 | benchmark(args, *outstream); 634 | break; 635 | case H_BLOOM: 636 | (*outstream) << header_str << "\n"; 637 | benchmark(args, *outstream); 638 | break; 639 | case H_COND: 640 | (*outstream) << header_str << "\n"; 641 | benchmark(args, *outstream); 642 | break; 643 | case H_UNCOND: 644 | (*outstream) << header_str << "\n"; 645 | benchmark(args, *outstream); 646 | break; 647 | default: 648 | fprintf(stderr, "Unsupported implementation: %d\n", args.implementation); 649 | exit(1); 650 | } 651 | return 0; 652 | } 653 | -------------------------------------------------------------------------------- /apps/hash-perf/Makefile: -------------------------------------------------------------------------------- 1 | # Modifications by Thomson Licensing licensed under Clear BSD License. 2 | # Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | # 4 | # Original license: 5 | # BSD LICENSE 6 | # 7 | # Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 8 | # All rights reserved. 9 | # 10 | # Redistribution and use in source and binary forms, with or without 11 | # modification, are permitted provided that the following conditions 12 | # are met: 13 | # 14 | # * Redistributions of source code must retain the above copyright 15 | # notice, this list of conditions and the following disclaimer. 16 | # * Redistributions in binary form must reproduce the above copyright 17 | # notice, this list of conditions and the following disclaimer in 18 | # the documentation and/or other materials provided with the 19 | # distribution. 20 | # * Neither the name of Intel Corporation nor the names of its 21 | # contributors may be used to endorse or promote products derived 22 | # from this software without specific prior written permission. 23 | # 24 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 27 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 28 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 30 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 | 36 | ifeq ($(RTE_SDK),) 37 | $(error "Please define RTE_SDK environment variable") 38 | endif 39 | 40 | # Default target, can be overriden by command line or environment 41 | RTE_TARGET ?= x86_64-native-linuxapp-gcc 42 | 43 | include $(RTE_SDK)/mk/rte.vars.mk 44 | 45 | # binary name 46 | APP = hash_perf 47 | 48 | # all source are stored in SRCS-y 49 | SRCS-y := main-hperf.c 50 | 51 | CFLAGS += -I$(SRCDIR) 52 | CFLAGS += -O3 $(USER_FLAGS) $(TCH_FLAGS) 53 | CFLAGS += $(WERROR_FLAGS) 54 | 55 | # this application needs libraries first 56 | DEPDIRS-y += lib drivers 57 | 58 | include $(RTE_SDK)/mk/rte.extapp.mk 59 | -------------------------------------------------------------------------------- /apps/hash-perf/main-hperf.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | 43 | #include 44 | 45 | #include 46 | #include 47 | #include 48 | 49 | 50 | //const enum rte_tch_hash_variants v = H_LAZY_BLOOM; 51 | //const enum rte_tch_hash_variants v = H_V1702; 52 | //const enum rte_tch_hash_variants v = H_TCH2016; 53 | enum rte_tch_hash_variants v; 54 | 55 | #define RTE_LOGTYPE_L3NAT RTE_LOGTYPE_USER1 56 | 57 | #define MAX_PKT_BURST 32u 58 | 59 | int N_capacity = 1024*1024; 60 | int N_insertions = 1024*850; 61 | int N_lookups = 1024*1024*100; 62 | int inv_Failure = 8; // must be power of 2. 63 | 64 | struct hh { 65 | volatile uint64_t v; 66 | } __rte_cache_aligned; 67 | 68 | volatile struct hh hv[32]; 69 | 70 | 71 | // Up to 32 threads 72 | struct rte_tch_hash * h[32]; 73 | #pragma message "Random state is shared between state, need to refactor application" 74 | struct rte_tch_rand_state rand_s; 75 | 76 | //double diff_tsc[32]; 77 | volatile double cycles[32]; 78 | volatile double rate[32]; 79 | volatile int nops; 80 | volatile int hashmap_size; 81 | int total_inserts=0; 82 | 83 | //enum rte_tch_hash_variants var[] = {H_HORTON, H_COND, H_BLOOM, H_LAZY_BLOOM,H_UNCOND, H_V1702, H_LAZY_UNCOND, H_LAZY_COND, H_LAZY_NO }; 84 | enum rte_tch_hash_variants var[] = {H_BLOOM, H_HORTON}; 85 | 86 | 87 | static void insert(struct rte_tch_hash* h, int nops){ 88 | int i; 89 | hash_data_t data; 90 | data.mm = _mm_set1_epi16(16); 91 | hash_key_t key; 92 | int err_count=0; 93 | 94 | for(i=0;i= hashmap_size){ 131 | i=0; 132 | } 133 | successes += (rte_tch_hash_lookup_data(v,h, key, &data, 0) == RHL_FOUND_NOTUPDATED ); 134 | } 135 | if(successes < nops) printf("%.2f successful lookup rate (ind) - %d/%d\n", ((double)successes)/(double)nops,successes,nops); 136 | 137 | } 138 | 139 | static inline void lookup_batch(struct rte_tch_hash * h, int nops, int batch, int neg){ 140 | // Perform N_lookups 141 | int j,k,i=0; 142 | int successes=0; 143 | hash_data_t data[batch]; 144 | hash_key_t key[batch]; 145 | 146 | for(j=0;j= hashmap_size){ 152 | i=0; 153 | } 154 | } 155 | uint64_t hits=0; 156 | rte_tch_hash_lookup_bulk_data(v,h, key, batch, &hits, data, 0); 157 | successes += __builtin_popcount(hits); 158 | } 159 | if((!neg) && successes < nops) printf("%.2f successful lookup rate (batch) - %d/%d\n", ((double)successes)/(double)nops,successes,nops); 160 | if((neg) && successes > 0) printf("%.2f successful lookup rate (batch) - %d/%d\n", ((double)successes)/(double)nops,successes,nops); 161 | 162 | } 163 | 164 | 165 | //static void lookup_batch1(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,1, 0); } 166 | //static void lookup_batch2(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,2, 0); } 167 | //static void lookup_batch4(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,4, 0); } 168 | //static void lookup_batch8(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,8, 0); } 169 | //static void lookup_batch12(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,12, 0); } 170 | //static void lookup_batch16(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,16, 0); } 171 | //static void lookup_batch24(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,24, 0); } 172 | __rte_unused static void lookup_batch32(struct rte_tch_hash* h, int nops){ lookup_batch(h, nops,32, 0); } 173 | //static void lookup_batch48(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,48, 0); } 174 | //static void lookup_batch64(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,64, 0); } 175 | 176 | 177 | //static void lookup_batch1_neg(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,1, 1); } 178 | //static void lookup_batch2_neg(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,2, 1); } 179 | //static void lookup_batch4_neg(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,4, 1); } 180 | //static void lookup_batch8_neg(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,8, 1); } 181 | //static void lookup_batch12_neg(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,12, 1); } 182 | //static void lookup_batch16_neg(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,16, 1); } 183 | //static void lookup_batch24_neg(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,24, 1); } 184 | __rte_unused static void lookup_batch32_neg(struct rte_tch_hash* h, int nops){ lookup_batch(h, nops,32, 1); } 185 | //static void lookup_batch48_neg(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,48, 1); } 186 | //static void lookup_batch64_neg(struct rte_hash_lazy* h, int nops){ lookup_batch(h, nops,64, 1); } 187 | 188 | 189 | 190 | 191 | 192 | 193 | typedef void (*tBenchFunc)(struct rte_tch_hash * h , int nops); 194 | 195 | static int main_loop(void * args){ 196 | unsigned lcore_id= rte_lcore_id(); 197 | uint64_t startq = rte_get_tsc_cycles(); 198 | ((tBenchFunc)args)(h[lcore_id],nops); 199 | uint64_t endq = rte_get_tsc_cycles(); 200 | double diff_tsc = endq-startq; 201 | 202 | cycles[lcore_id] = diff_tsc/(double)nops; 203 | rate[lcore_id] = ((double)nops)/(double)(diff_tsc/rte_get_tsc_hz()); 204 | return 0; 205 | } 206 | 207 | static void call_bench(tBenchFunc f, const char* title, int ncores, int n){ 208 | nops=n; 209 | int launched_cores = 0; 210 | int lcore_id; 211 | /* Reset stats to 0 */ 212 | RTE_LCORE_FOREACH(lcore_id) { 213 | cycles[lcore_id] = 0; 214 | rate[lcore_id] = 0; 215 | } 216 | 217 | /* launch on every slave lcore */ 218 | RTE_LCORE_FOREACH_SLAVE(lcore_id) { 219 | if(launched_cores++ >= ncores) break; 220 | rte_eal_remote_launch(main_loop, f, lcore_id); 221 | 222 | } 223 | 224 | /* If needed launch on master */ 225 | if(launched_cores < ncores) main_loop(f); 226 | 227 | /* Wait for slaves */ 228 | RTE_LCORE_FOREACH_SLAVE(lcore_id) { 229 | if (rte_eal_wait_lcore(lcore_id) < 0) { 230 | rte_exit(EXIT_FAILURE, "One thread failed\n"); 231 | } 232 | } 233 | 234 | double rate_avg=0; 235 | double cycles_avg=0; 236 | RTE_LCORE_FOREACH(lcore_id) { 237 | cycles_avg += cycles[lcore_id]; 238 | rate_avg += rate[lcore_id]; 239 | } 240 | cycles_avg /= ncores; 241 | rate_avg /= ncores; 242 | 243 | printf("%s %d cores - %.2f cycles/operation , %.2f M operation/second\n", title, ncores, (double)cycles_avg, (double)rate_avg/1000000.0); 244 | } 245 | 246 | int 247 | main(int argc, char **argv) 248 | { 249 | int ret; 250 | unsigned i; 251 | 252 | /* init EAL */ 253 | ret = rte_eal_init(argc, argv); 254 | if (ret < 0) 255 | rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n"); 256 | argc -= ret; 257 | argv += ret; 258 | 259 | unsigned lcore_id; 260 | 261 | int capacity=0; 262 | float filling_ratio=0; 263 | 264 | if(argc != 3){ 265 | rte_exit(EXIT_FAILURE, "Wrong number of arguments: hash_perf EAL_ARGS -- capacity filling_ratio \n"); 266 | } 267 | sscanf(argv[1],"%d",&capacity); 268 | sscanf(argv[2],"%f",&filling_ratio); 269 | 270 | if(capacity == 0 || filling_ratio == 0){ 271 | rte_exit(EXIT_FAILURE, "Wrong arguments: hash_perf EAL_ARGS -- capacity filling_ratio \n"); 272 | } 273 | 274 | rte_tch_rand_init(&rand_s); 275 | 276 | for(i=0;i 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | 44 | #include 45 | 46 | #include 47 | #include 48 | #include 49 | 50 | 51 | enum rte_tch_hash_variants var[] = {H_BLOOM, H_HORTON/*, H_V1604*/}; 52 | int sizes[] = {/*524288, 4194304, 16777216,*/ 33554432}; 53 | double filling_ratios[] = {0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.75};/*,0.8,0.85,0.875,0.9,0.91,0.92,0.93,0.94,0.95,0.96,0.97,0.975,0.98,0.985,0.99,0.995}; */ 54 | unsigned count=1; 55 | struct rte_tch_rand_state rand_s; 56 | 57 | static void insert(enum rte_tch_hash_variants hvar, struct rte_tch_hash* h, int nops){ 58 | int i; 59 | hash_data_t data; 60 | data.mm = _mm_set1_epi16(16); 61 | hash_key_t key; 62 | int err_count=0; 63 | for(i=0;i 0.92) break; 123 | 124 | double mean_secondary = 0; 125 | for(l=0;l /* defines printf for tests */ 39 | #include /* defines time_t for timings in the test */ 40 | #include /* defines uint32_t etc */ 41 | #include /* attempt to define endianness */ 42 | #ifdef linux 43 | # include /* attempt to define endianness */ 44 | #endif 45 | 46 | /* 47 | * My best guess at if you are big-endian or little-endian. This may 48 | * need adjustment. 49 | */ 50 | #if (defined(__BYTE_ORDER) && defined(__LITTLE_ENDIAN) && \ 51 | __BYTE_ORDER == __LITTLE_ENDIAN) || \ 52 | (defined(i386) || defined(__i386__) || defined(__i486__) || \ 53 | defined(__i586__) || defined(__i686__) || defined(vax) || defined(MIPSEL)) 54 | # define HASH_LITTLE_ENDIAN 1 55 | # define HASH_BIG_ENDIAN 0 56 | #elif (defined(__BYTE_ORDER) && defined(__BIG_ENDIAN) && \ 57 | __BYTE_ORDER == __BIG_ENDIAN) || \ 58 | (defined(sparc) || defined(POWERPC) || defined(mc68000) || defined(sel)) 59 | # define HASH_LITTLE_ENDIAN 0 60 | # define HASH_BIG_ENDIAN 1 61 | #else 62 | # define HASH_LITTLE_ENDIAN 0 63 | # define HASH_BIG_ENDIAN 0 64 | #endif 65 | 66 | #define hashsize(n) ((uint32_t)1<<(n)) 67 | #define hashmask(n) (hashsize(n)-1) 68 | #define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k)))) 69 | 70 | /* 71 | ------------------------------------------------------------------------------- 72 | mix -- mix 3 32-bit values reversibly. 73 | 74 | This is reversible, so any information in (a,b,c) before mix() is 75 | still in (a,b,c) after mix(). 76 | 77 | If four pairs of (a,b,c) inputs are run through mix(), or through 78 | mix() in reverse, there are at least 32 bits of the output that 79 | are sometimes the same for one pair and different for another pair. 80 | This was tested for: 81 | * pairs that differed by one bit, by two bits, in any combination 82 | of top bits of (a,b,c), or in any combination of bottom bits of 83 | (a,b,c). 84 | * "differ" is defined as +, -, ^, or ~^. For + and -, I transformed 85 | the output delta to a Gray code (a^(a>>1)) so a string of 1's (as 86 | is commonly produced by subtraction) look like a single 1-bit 87 | difference. 88 | * the base values were pseudorandom, all zero but one bit set, or 89 | all zero plus a counter that starts at zero. 90 | 91 | Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that 92 | satisfy this are 93 | 4 6 8 16 19 4 94 | 9 15 3 18 27 15 95 | 14 9 3 7 17 3 96 | Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing 97 | for "differ" defined as + with a one-bit base and a two-bit delta. I 98 | used http://burtleburtle.net/bob/hash/avalanche.html to choose 99 | the operations, constants, and arrangements of the variables. 100 | 101 | This does not achieve avalanche. There are input bits of (a,b,c) 102 | that fail to affect some output bits of (a,b,c), especially of a. The 103 | most thoroughly mixed value is c, but it doesn't really even achieve 104 | avalanche in c. 105 | 106 | This allows some parallelism. Read-after-writes are good at doubling 107 | the number of bits affected, so the goal of mixing pulls in the opposite 108 | direction as the goal of parallelism. I did what I could. Rotates 109 | seem to cost as much as shifts on every machine I could lay my hands 110 | on, and rotates are much kinder to the top and bottom bits, so I used 111 | rotates. 112 | ------------------------------------------------------------------------------- 113 | */ 114 | #define mix(a,b,c) \ 115 | { \ 116 | a -= c; a ^= rot(c, 4); c += b; \ 117 | b -= a; b ^= rot(a, 6); a += c; \ 118 | c -= b; c ^= rot(b, 8); b += a; \ 119 | a -= c; a ^= rot(c,16); c += b; \ 120 | b -= a; b ^= rot(a,19); a += c; \ 121 | c -= b; c ^= rot(b, 4); b += a; \ 122 | } 123 | 124 | /* 125 | ------------------------------------------------------------------------------- 126 | final -- final mixing of 3 32-bit values (a,b,c) into c 127 | 128 | Pairs of (a,b,c) values differing in only a few bits will usually 129 | produce values of c that look totally different. This was tested for 130 | * pairs that differed by one bit, by two bits, in any combination 131 | of top bits of (a,b,c), or in any combination of bottom bits of 132 | (a,b,c). 133 | * "differ" is defined as +, -, ^, or ~^. For + and -, I transformed 134 | the output delta to a Gray code (a^(a>>1)) so a string of 1's (as 135 | is commonly produced by subtraction) look like a single 1-bit 136 | difference. 137 | * the base values were pseudorandom, all zero but one bit set, or 138 | all zero plus a counter that starts at zero. 139 | 140 | These constants passed: 141 | 14 11 25 16 4 14 24 142 | 12 14 25 16 4 14 24 143 | and these came close: 144 | 4 8 15 26 3 22 24 145 | 10 8 15 26 3 22 24 146 | 11 8 15 26 3 22 24 147 | ------------------------------------------------------------------------------- 148 | */ 149 | #define final(a,b,c) \ 150 | { \ 151 | c ^= b; c -= rot(b,14); \ 152 | a ^= c; a -= rot(c,11); \ 153 | b ^= a; b -= rot(a,25); \ 154 | c ^= b; c -= rot(b,16); \ 155 | a ^= c; a -= rot(c,4); \ 156 | b ^= a; b -= rot(a,14); \ 157 | c ^= b; c -= rot(b,24); \ 158 | } 159 | 160 | 161 | static uint32_t hash_u32(uint32_t k) 162 | { 163 | uint32_t a,b,c; 164 | 165 | uint32_t pb = 0xdf405ef1; 166 | uint32_t pc = 0x13325cb2; 167 | 168 | /* Set up the internal state */ 169 | a = b = c = 0xdeadbeef + ((uint32_t)(1<<2)) + pc; 170 | c += pb; 171 | 172 | a += k; 173 | 174 | final(a,b,c); 175 | 176 | return pc; 177 | 178 | } 179 | 180 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cmp_x86_v1604.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * Minor modifications from vanilla DPDK 16.04 are licensed under Clear BSD. 3 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 4 | * 5 | * Original code is licensed under: 6 | * 7 | * BSD LICENSE 8 | * 9 | * Copyright(c) 2015 Intel Corporation. All rights reserved. 10 | * All rights reserved. 11 | * 12 | * Redistribution and use in source and binary forms, with or without 13 | * modification, are permitted provided that the following conditions 14 | * are met: 15 | * 16 | * * Redistributions of source code must retain the above copyright 17 | * notice, this list of conditions and the following disclaimer. 18 | * * Redistributions in binary form must reproduce the above copyright 19 | * notice, this list of conditions and the following disclaimer in 20 | * the documentation and/or other materials provided with the 21 | * distribution. 22 | * * Neither the name of Intel Corporation nor the names of its 23 | * contributors may be used to endorse or promote products derived 24 | * from this software without specific prior written permission. 25 | * 26 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | */ 38 | 39 | /* Functions to compare multiple of 16 byte keys (up to 128 bytes) */ 40 | static int 41 | rte_hash_v1604_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused) 42 | { 43 | const __m128i k1 = _mm_loadu_si128((const __m128i *) key1); 44 | const __m128i k2 = _mm_loadu_si128((const __m128i *) key2); 45 | #ifdef RTE_MACHINE_CPUFLAG_SSE4_1 46 | const __m128i x = _mm_xor_si128(k1, k2); 47 | 48 | return !_mm_test_all_zeros(x, x); 49 | #else 50 | const __m128i x = _mm_cmpeq_epi32(k1, k2); 51 | 52 | return _mm_movemask_epi8(x) != 0xffff; 53 | #endif 54 | } 55 | 56 | static int 57 | rte_hash_v1604_k32_cmp_eq(const void *key1, const void *key2, size_t key_len) 58 | { 59 | return rte_hash_v1604_k16_cmp_eq(key1, key2, key_len) || 60 | rte_hash_v1604_k16_cmp_eq((const char *) key1 + 16, 61 | (const char *) key2 + 16, key_len); 62 | } 63 | 64 | static int 65 | rte_hash_v1604_k48_cmp_eq(const void *key1, const void *key2, size_t key_len) 66 | { 67 | return rte_hash_v1604_k16_cmp_eq(key1, key2, key_len) || 68 | rte_hash_v1604_k16_cmp_eq((const char *) key1 + 16, 69 | (const char *) key2 + 16, key_len) || 70 | rte_hash_v1604_k16_cmp_eq((const char *) key1 + 32, 71 | (const char *) key2 + 32, key_len); 72 | } 73 | 74 | static int 75 | rte_hash_v1604_k64_cmp_eq(const void *key1, const void *key2, size_t key_len) 76 | { 77 | return rte_hash_v1604_k32_cmp_eq(key1, key2, key_len) || 78 | rte_hash_v1604_k32_cmp_eq((const char *) key1 + 32, 79 | (const char *) key2 + 32, key_len); 80 | } 81 | 82 | static int 83 | rte_hash_v1604_k80_cmp_eq(const void *key1, const void *key2, size_t key_len) 84 | { 85 | return rte_hash_v1604_k64_cmp_eq(key1, key2, key_len) || 86 | rte_hash_v1604_k16_cmp_eq((const char *) key1 + 64, 87 | (const char *) key2 + 64, key_len); 88 | } 89 | 90 | static int 91 | rte_hash_v1604_k96_cmp_eq(const void *key1, const void *key2, size_t key_len) 92 | { 93 | return rte_hash_v1604_k64_cmp_eq(key1, key2, key_len) || 94 | rte_hash_v1604_k32_cmp_eq((const char *) key1 + 64, 95 | (const char *) key2 + 64, key_len); 96 | } 97 | 98 | static int 99 | rte_hash_v1604_k112_cmp_eq(const void *key1, const void *key2, size_t key_len) 100 | { 101 | return rte_hash_v1604_k64_cmp_eq(key1, key2, key_len) || 102 | rte_hash_v1604_k32_cmp_eq((const char *) key1 + 64, 103 | (const char *) key2 + 64, key_len) || 104 | rte_hash_v1604_k16_cmp_eq((const char *) key1 + 96, 105 | (const char *) key2 + 96, key_len); 106 | } 107 | 108 | static int 109 | rte_hash_v1604_k128_cmp_eq(const void *key1, const void *key2, size_t key_len) 110 | { 111 | return rte_hash_v1604_k64_cmp_eq(key1, key2, key_len) || 112 | rte_hash_v1604_k64_cmp_eq((const char *) key1 + 64, 113 | (const char *) key2 + 64, key_len); 114 | } 115 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cmp_x86_v1702.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * Minor modifications from vanilla DPDK 17.02 are licensed under Clear BSD. 3 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 4 | * 5 | * Original code is licensed under: 6 | * 7 | * BSD LICENSE 8 | * 9 | * Copyright(c) 2015 Intel Corporation. All rights reserved. 10 | * All rights reserved. 11 | * 12 | * Redistribution and use in source and binary forms, with or without 13 | * modification, are permitted provided that the following conditions 14 | * are met: 15 | * 16 | * * Redistributions of source code must retain the above copyright 17 | * notice, this list of conditions and the following disclaimer. 18 | * * Redistributions in binary form must reproduce the above copyright 19 | * notice, this list of conditions and the following disclaimer in 20 | * the documentation and/or other materials provided with the 21 | * distribution. 22 | * * Neither the name of Intel Corporation nor the names of its 23 | * contributors may be used to endorse or promote products derived 24 | * from this software without specific prior written permission. 25 | * 26 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | */ 38 | 39 | /* Functions to compare multiple of 16 byte keys (up to 128 bytes) */ 40 | static int 41 | rte_hash_v1702_k16_cmp_eq(const void *key1, const void *key2, size_t key_len __rte_unused) 42 | { 43 | const __m128i k1 = _mm_loadu_si128((const __m128i *) key1); 44 | const __m128i k2 = _mm_loadu_si128((const __m128i *) key2); 45 | #ifdef RTE_MACHINE_CPUFLAG_SSE4_1 46 | const __m128i x = _mm_xor_si128(k1, k2); 47 | 48 | return !_mm_test_all_zeros(x, x); 49 | #else 50 | const __m128i x = _mm_cmpeq_epi32(k1, k2); 51 | 52 | return _mm_movemask_epi8(x) != 0xffff; 53 | #endif 54 | } 55 | 56 | static int 57 | rte_hash_v1702_k32_cmp_eq(const void *key1, const void *key2, size_t key_len) 58 | { 59 | return rte_hash_v1702_k16_cmp_eq(key1, key2, key_len) || 60 | rte_hash_v1702_k16_cmp_eq((const char *) key1 + 16, 61 | (const char *) key2 + 16, key_len); 62 | } 63 | 64 | static int 65 | rte_hash_v1702_k48_cmp_eq(const void *key1, const void *key2, size_t key_len) 66 | { 67 | return rte_hash_v1702_k16_cmp_eq(key1, key2, key_len) || 68 | rte_hash_v1702_k16_cmp_eq((const char *) key1 + 16, 69 | (const char *) key2 + 16, key_len) || 70 | rte_hash_v1702_k16_cmp_eq((const char *) key1 + 32, 71 | (const char *) key2 + 32, key_len); 72 | } 73 | 74 | static int 75 | rte_hash_v1702_k64_cmp_eq(const void *key1, const void *key2, size_t key_len) 76 | { 77 | return rte_hash_v1702_k32_cmp_eq(key1, key2, key_len) || 78 | rte_hash_v1702_k32_cmp_eq((const char *) key1 + 32, 79 | (const char *) key2 + 32, key_len); 80 | } 81 | 82 | static int 83 | rte_hash_v1702_k80_cmp_eq(const void *key1, const void *key2, size_t key_len) 84 | { 85 | return rte_hash_v1702_k64_cmp_eq(key1, key2, key_len) || 86 | rte_hash_v1702_k16_cmp_eq((const char *) key1 + 64, 87 | (const char *) key2 + 64, key_len); 88 | } 89 | 90 | static int 91 | rte_hash_v1702_k96_cmp_eq(const void *key1, const void *key2, size_t key_len) 92 | { 93 | return rte_hash_v1702_k64_cmp_eq(key1, key2, key_len) || 94 | rte_hash_v1702_k32_cmp_eq((const char *) key1 + 64, 95 | (const char *) key2 + 64, key_len); 96 | } 97 | 98 | static int 99 | rte_hash_v1702_k112_cmp_eq(const void *key1, const void *key2, size_t key_len) 100 | { 101 | return rte_hash_v1702_k64_cmp_eq(key1, key2, key_len) || 102 | rte_hash_v1702_k32_cmp_eq((const char *) key1 + 64, 103 | (const char *) key2 + 64, key_len) || 104 | rte_hash_v1702_k16_cmp_eq((const char *) key1 + 96, 105 | (const char *) key2 + 96, key_len); 106 | } 107 | 108 | static int 109 | rte_hash_v1702_k128_cmp_eq(const void *key1, const void *key2, size_t key_len) 110 | { 111 | return rte_hash_v1702_k64_cmp_eq(key1, key2, key_len) || 112 | rte_hash_v1702_k64_cmp_eq((const char *) key1 + 64, 113 | (const char *) key2 + 64, key_len); 114 | } 115 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cuckoo_hash_bloom.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | 10 | 11 | #include "rte_hash_bloom.h" 12 | 13 | #include "rte_cuckoo_hash_template.c" 14 | 15 | 16 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cuckoo_hash_cond.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | 10 | #include "rte_hash_lazy_cond.h" 11 | 12 | #include "rte_cuckoo_hash_template.c" 13 | 14 | 15 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cuckoo_hash_horton.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #include "rte_hash_horton.h" 10 | 11 | #include "rte_cuckoo_hash_template.c" 12 | 13 | 14 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cuckoo_hash_lazy_bloom.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | 10 | #include "rte_hash_lazy_bloom.h" 11 | 12 | #include "rte_cuckoo_hash_template.c" 13 | 14 | 15 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cuckoo_hash_lazy_cond.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #include "rte_hash_cond.h" 10 | 11 | #include "rte_cuckoo_hash_template.c" 12 | 13 | 14 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cuckoo_hash_lazy_no.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #include "rte_hash_lazy_no.h" 10 | 11 | #include "rte_cuckoo_hash_template.c" 12 | 13 | 14 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cuckoo_hash_lazy_uncond.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | 10 | #include "rte_hash_lazy_uncond.h" 11 | 12 | #include "rte_cuckoo_hash_template.c" 13 | 14 | 15 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cuckoo_hash_uncond.c: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #include "rte_hash_uncond.h" 10 | 11 | #include "rte_cuckoo_hash_template.c" 12 | 13 | 14 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cuckoo_hash_v1702.c: -------------------------------------------------------------------------------- 1 | /*- 2 | * Minor modifications from vanilla DPDK 17.02 are licensed under Clear BSD. 3 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 4 | * 5 | * Original code is licensed under: 6 | * 7 | * BSD LICENSE 8 | * 9 | * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. 10 | * All rights reserved. 11 | * 12 | * Redistribution and use in source and binary forms, with or without 13 | * modification, are permitted provided that the following conditions 14 | * are met: 15 | * 16 | * * Redistributions of source code must retain the above copyright 17 | * notice, this list of conditions and the following disclaimer. 18 | * * Redistributions in binary form must reproduce the above copyright 19 | * notice, this list of conditions and the following disclaimer in 20 | * the documentation and/or other materials provided with the 21 | * distribution. 22 | * * Neither the name of Intel Corporation nor the names of its 23 | * contributors may be used to endorse or promote products derived 24 | * from this software without specific prior written permission. 25 | * 26 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | */ 38 | 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | 46 | #include 47 | #include /* for definition of RTE_CACHE_LINE_SIZE */ 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | #include 60 | #include 61 | #include 62 | #include 63 | #include 64 | #include 65 | 66 | #include "rte_hash_v1702.h" 67 | #include "rte_cuckoo_hash_v1702.h" 68 | 69 | #if defined(RTE_ARCH_X86) 70 | #include "rte_cuckoo_hash_x86_v1702.h" 71 | #endif 72 | 73 | TAILQ_HEAD(rte_hash_v1702_list, rte_tailq_entry); 74 | 75 | static struct rte_tailq_elem rte_hash_v1702_tailq = { 76 | .name = "RTE_HASH_V1702", 77 | }; 78 | EAL_REGISTER_TAILQ(rte_hash_v1702_tailq) 79 | 80 | struct rte_hash_v1702 * 81 | rte_hash_v1702_find_existing(const char *name) 82 | { 83 | struct rte_hash_v1702 *h = NULL; 84 | struct rte_tailq_entry *te; 85 | struct rte_hash_v1702_list *hash_list; 86 | 87 | hash_list = RTE_TAILQ_CAST(rte_hash_v1702_tailq.head, rte_hash_v1702_list); 88 | 89 | rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK); 90 | TAILQ_FOREACH(te, hash_list, next) { 91 | h = (struct rte_hash_v1702 *) te->data; 92 | if (strncmp(name, h->name, RTE_HASH_V1702_NAMESIZE) == 0) 93 | break; 94 | } 95 | rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK); 96 | 97 | if (te == NULL) { 98 | rte_errno = ENOENT; 99 | return NULL; 100 | } 101 | return h; 102 | } 103 | 104 | void rte_hash_v1702_set_cmp_func(struct rte_hash_v1702 *h, rte_hash_v1702_cmp_eq_t func) 105 | { 106 | h->cmp_jump_table_idx = KEY_CUSTOM; 107 | h->rte_hash_v1702_custom_cmp_eq = func; 108 | } 109 | 110 | static inline int 111 | rte_hash_v1702_cmp_eq(const void *key1, const void *key2, const struct rte_hash_v1702 *h) 112 | { 113 | if (h->cmp_jump_table_idx == KEY_CUSTOM) 114 | return h->rte_hash_v1702_custom_cmp_eq(key1, key2, h->key_len); 115 | else 116 | return cmp_jump_table_v1702[h->cmp_jump_table_idx](key1, key2, h->key_len); 117 | } 118 | 119 | struct rte_hash_v1702 * 120 | rte_hash_v1702_create(const struct rte_hash_v1702_parameters *params) 121 | { 122 | struct rte_hash_v1702 *h = NULL; 123 | struct rte_tailq_entry *te = NULL; 124 | struct rte_hash_v1702_list *hash_list; 125 | struct rte_ring *r = NULL; 126 | char hash_name[RTE_HASH_V1702_NAMESIZE]; 127 | void *k = NULL; 128 | void *buckets = NULL; 129 | char ring_name[RTE_RING_NAMESIZE]; 130 | unsigned num_key_slots; 131 | unsigned hw_trans_mem_support = 0; 132 | unsigned i; 133 | 134 | hash_list = RTE_TAILQ_CAST(rte_hash_v1702_tailq.head, rte_hash_v1702_list); 135 | 136 | if (params == NULL) { 137 | RTE_LOG(ERR, HASH, "rte_hash_v1702_create has no parameters\n"); 138 | return NULL; 139 | } 140 | 141 | /* Check for valid parameters */ 142 | if ((params->entries > RTE_HASH_V1702_ENTRIES_MAX) || 143 | (params->entries < RTE_HASH_V1702_BUCKET_ENTRIES) || 144 | !rte_is_power_of_2(RTE_HASH_V1702_BUCKET_ENTRIES) || 145 | (params->key_len == 0)) { 146 | rte_errno = EINVAL; 147 | RTE_LOG(ERR, HASH, "rte_hash_v1702_create has invalid parameters\n"); 148 | return NULL; 149 | } 150 | 151 | /* Check extra flags field to check extra options. */ 152 | if (params->extra_flag & RTE_HASH_V1702_EXTRA_FLAGS_TRANS_MEM_SUPPORT) 153 | hw_trans_mem_support = 1; 154 | 155 | /* Store all keys and leave the first entry as a dummy entry for lookup_bulk */ 156 | if (hw_trans_mem_support) 157 | /* 158 | * Increase number of slots by total number of indices 159 | * that can be stored in the lcore caches 160 | * except for the first cache 161 | */ 162 | num_key_slots = params->entries + (RTE_MAX_LCORE - 1) * 163 | LCORE_CACHE_SIZE + 1; 164 | else 165 | num_key_slots = params->entries + 1; 166 | 167 | snprintf(ring_name, sizeof(ring_name), "HT_%s", params->name); 168 | /* Create ring (Dummy slot index is not enqueued) */ 169 | r = rte_ring_create(ring_name, rte_align32pow2(num_key_slots - 1), 170 | params->socket_id, 0); 171 | if (r == NULL) { 172 | RTE_LOG(ERR, HASH, "memory allocation failed\n"); 173 | goto err; 174 | } 175 | 176 | snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name); 177 | 178 | rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); 179 | 180 | /* guarantee there's no existing: this is normally already checked 181 | * by ring creation above */ 182 | TAILQ_FOREACH(te, hash_list, next) { 183 | h = (struct rte_hash_v1702 *) te->data; 184 | if (strncmp(params->name, h->name, RTE_HASH_V1702_NAMESIZE) == 0) 185 | break; 186 | } 187 | h = NULL; 188 | if (te != NULL) { 189 | rte_errno = EEXIST; 190 | te = NULL; 191 | goto err_unlock; 192 | } 193 | 194 | te = rte_zmalloc("HASH_TAILQ_ENTRY", sizeof(*te), 0); 195 | if (te == NULL) { 196 | RTE_LOG(ERR, HASH, "tailq entry allocation failed\n"); 197 | goto err_unlock; 198 | } 199 | 200 | h = (struct rte_hash_v1702 *)rte_zmalloc_socket(hash_name, sizeof(struct rte_hash_v1702), 201 | RTE_CACHE_LINE_SIZE, params->socket_id); 202 | 203 | if (h == NULL) { 204 | RTE_LOG(ERR, HASH, "memory allocation failed\n"); 205 | goto err_unlock; 206 | } 207 | 208 | const uint32_t num_buckets = rte_align32pow2(params->entries) 209 | / RTE_HASH_V1702_BUCKET_ENTRIES; 210 | 211 | buckets = rte_zmalloc_socket(NULL, 212 | num_buckets * sizeof(struct rte_hash_v1702_bucket), 213 | RTE_CACHE_LINE_SIZE, params->socket_id); 214 | 215 | if (buckets == NULL) { 216 | RTE_LOG(ERR, HASH, "memory allocation failed\n"); 217 | goto err_unlock; 218 | } 219 | 220 | const uint32_t key_entry_size = sizeof(struct rte_hash_v1702_key) + params->key_len; 221 | const uint64_t key_tbl_size = (uint64_t) key_entry_size * num_key_slots; 222 | 223 | k = rte_zmalloc_socket(NULL, key_tbl_size, 224 | RTE_CACHE_LINE_SIZE, params->socket_id); 225 | 226 | if (k == NULL) { 227 | RTE_LOG(ERR, HASH, "memory allocation failed\n"); 228 | goto err_unlock; 229 | } 230 | 231 | /* 232 | * If x86 architecture is used, select appropriate compare function, 233 | * which may use x86 intrinsics, otherwise use memcmp 234 | */ 235 | #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) 236 | /* Select function to compare keys */ 237 | switch (params->key_len) { 238 | case 16: 239 | h->cmp_jump_table_idx = KEY_16_BYTES; 240 | break; 241 | case 32: 242 | h->cmp_jump_table_idx = KEY_32_BYTES; 243 | break; 244 | case 48: 245 | h->cmp_jump_table_idx = KEY_48_BYTES; 246 | break; 247 | case 64: 248 | h->cmp_jump_table_idx = KEY_64_BYTES; 249 | break; 250 | case 80: 251 | h->cmp_jump_table_idx = KEY_80_BYTES; 252 | break; 253 | case 96: 254 | h->cmp_jump_table_idx = KEY_96_BYTES; 255 | break; 256 | case 112: 257 | h->cmp_jump_table_idx = KEY_112_BYTES; 258 | break; 259 | case 128: 260 | h->cmp_jump_table_idx = KEY_128_BYTES; 261 | break; 262 | default: 263 | /* If key is not multiple of 16, use generic memcmp */ 264 | h->cmp_jump_table_idx = KEY_OTHER_BYTES; 265 | } 266 | #else 267 | h->cmp_jump_table_idx = KEY_OTHER_BYTES; 268 | #endif 269 | 270 | if (hw_trans_mem_support) { 271 | h->local_free_slots = rte_zmalloc_socket(NULL, 272 | sizeof(struct lcore_cache) * RTE_MAX_LCORE, 273 | RTE_CACHE_LINE_SIZE, params->socket_id); 274 | } 275 | 276 | /* Setup hash context */ 277 | snprintf(h->name, sizeof(h->name), "%s", params->name); 278 | h->entries = params->entries; 279 | h->key_len = params->key_len; 280 | h->key_entry_size = key_entry_size; 281 | h->hash_func_init_val = params->hash_func_init_val; 282 | 283 | h->num_buckets = num_buckets; 284 | h->bucket_bitmask = h->num_buckets - 1; 285 | h->buckets = buckets; 286 | h->hash_func = (params->hash_func == NULL) ? 287 | DEFAULT_HASH_FUNC : params->hash_func; 288 | h->key_store = k; 289 | h->free_slots = r; 290 | h->hw_trans_mem_support = hw_trans_mem_support; 291 | 292 | #if defined(RTE_ARCH_X86) 293 | if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2)) 294 | h->sig_cmp_fn = RTE_HASH_V1702_COMPARE_AVX2; 295 | else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE2)) 296 | h->sig_cmp_fn = RTE_HASH_V1702_COMPARE_SSE; 297 | else 298 | #endif 299 | h->sig_cmp_fn = RTE_HASH_V1702_COMPARE_SCALAR; 300 | 301 | /* Turn on multi-writer only with explicit flat from user and TM 302 | * support. 303 | */ 304 | if (params->extra_flag & RTE_HASH_V1702_EXTRA_FLAGS_MULTI_WRITER_ADD) { 305 | if (h->hw_trans_mem_support) { 306 | h->add_key = ADD_KEY_MULTIWRITER_TM; 307 | } else { 308 | h->add_key = ADD_KEY_MULTIWRITER; 309 | h->multiwriter_lock = rte_malloc(NULL, 310 | sizeof(rte_spinlock_t), 311 | LCORE_CACHE_SIZE); 312 | rte_spinlock_init(h->multiwriter_lock); 313 | } 314 | } else 315 | h->add_key = ADD_KEY_SINGLEWRITER; 316 | 317 | /* Populate free slots ring. Entry zero is reserved for key misses. */ 318 | for (i = 1; i < params->entries + 1; i++) 319 | rte_ring_sp_enqueue(r, (void *)((uintptr_t) i)); 320 | 321 | te->data = (void *) h; 322 | TAILQ_INSERT_TAIL(hash_list, te, next); 323 | rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); 324 | 325 | return h; 326 | err_unlock: 327 | rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); 328 | err: 329 | rte_ring_free(r); 330 | rte_free(te); 331 | rte_free(h); 332 | rte_free(buckets); 333 | rte_free(k); 334 | return NULL; 335 | } 336 | 337 | void 338 | rte_hash_v1702_free(struct rte_hash_v1702 *h) 339 | { 340 | struct rte_tailq_entry *te; 341 | struct rte_hash_v1702_list *hash_list; 342 | 343 | if (h == NULL) 344 | return; 345 | 346 | hash_list = RTE_TAILQ_CAST(rte_hash_v1702_tailq.head, rte_hash_v1702_list); 347 | 348 | rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); 349 | 350 | /* find out tailq entry */ 351 | TAILQ_FOREACH(te, hash_list, next) { 352 | if (te->data == (void *) h) 353 | break; 354 | } 355 | 356 | if (te == NULL) { 357 | rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); 358 | return; 359 | } 360 | 361 | TAILQ_REMOVE(hash_list, te, next); 362 | 363 | rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); 364 | 365 | if (h->hw_trans_mem_support) 366 | rte_free(h->local_free_slots); 367 | 368 | if (h->add_key == ADD_KEY_MULTIWRITER) 369 | rte_free(h->multiwriter_lock); 370 | rte_ring_free(h->free_slots); 371 | rte_free(h->key_store); 372 | rte_free(h->buckets); 373 | rte_free(h); 374 | rte_free(te); 375 | } 376 | 377 | hash_sig_t 378 | rte_hash_v1702_hash(const struct rte_hash_v1702 *h, const void *key) 379 | { 380 | /* calc hash result by key */ 381 | return h->hash_func(key, h->key_len, h->hash_func_init_val); 382 | } 383 | 384 | /* Calc the secondary hash value from the primary hash value of a given key */ 385 | static inline hash_sig_t 386 | rte_hash_v1702_secondary_hash(const hash_sig_t primary_hash) 387 | { 388 | static const unsigned all_bits_shift = 12; 389 | static const unsigned alt_bits_xor = 0x5bd1e995; 390 | 391 | uint32_t tag = primary_hash >> all_bits_shift; 392 | 393 | return primary_hash ^ ((tag + 1) * alt_bits_xor); 394 | } 395 | 396 | void 397 | rte_hash_v1702_reset(struct rte_hash_v1702 *h) 398 | { 399 | void *ptr; 400 | unsigned i; 401 | 402 | if (h == NULL) 403 | return; 404 | 405 | memset(h->buckets, 0, h->num_buckets * sizeof(struct rte_hash_v1702_bucket)); 406 | memset(h->key_store, 0, h->key_entry_size * (h->entries + 1)); 407 | 408 | /* clear the free ring */ 409 | while (rte_ring_dequeue(h->free_slots, &ptr) == 0) 410 | rte_pause(); 411 | 412 | /* Repopulate the free slots ring. Entry zero is reserved for key misses */ 413 | for (i = 1; i < h->entries + 1; i++) 414 | rte_ring_sp_enqueue(h->free_slots, (void *)((uintptr_t) i)); 415 | 416 | if (h->hw_trans_mem_support) { 417 | /* Reset local caches per lcore */ 418 | for (i = 0; i < RTE_MAX_LCORE; i++) 419 | h->local_free_slots[i].len = 0; 420 | } 421 | } 422 | 423 | /* Search for an entry that can be pushed to its alternative location */ 424 | static inline int 425 | make_space_bucket(const struct rte_hash_v1702 *h, struct rte_hash_v1702_bucket *bkt) 426 | { 427 | static unsigned int nr_pushes; 428 | unsigned i, j; 429 | int ret; 430 | uint32_t next_bucket_idx; 431 | struct rte_hash_v1702_bucket *next_bkt[RTE_HASH_V1702_BUCKET_ENTRIES]; 432 | 433 | /* 434 | * Push existing item (search for bucket with space in 435 | * alternative locations) to its alternative location 436 | */ 437 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) { 438 | /* Search for space in alternative locations */ 439 | next_bucket_idx = bkt->sig_alt[i] & h->bucket_bitmask; 440 | next_bkt[i] = &h->buckets[next_bucket_idx]; 441 | for (j = 0; j < RTE_HASH_V1702_BUCKET_ENTRIES; j++) { 442 | if (next_bkt[i]->key_idx[j] == EMPTY_SLOT) 443 | break; 444 | } 445 | 446 | if (j != RTE_HASH_V1702_BUCKET_ENTRIES) 447 | break; 448 | } 449 | 450 | /* Alternative location has spare room (end of recursive function) */ 451 | if (i != RTE_HASH_V1702_BUCKET_ENTRIES) { 452 | next_bkt[i]->sig_alt[j] = bkt->sig_current[i]; 453 | next_bkt[i]->sig_current[j] = bkt->sig_alt[i]; 454 | next_bkt[i]->key_idx[j] = bkt->key_idx[i]; 455 | return i; 456 | } 457 | 458 | /* Pick entry that has not been pushed yet */ 459 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) 460 | if (bkt->flag[i] == 0) 461 | break; 462 | 463 | /* All entries have been pushed, so entry cannot be added */ 464 | if (i == RTE_HASH_V1702_BUCKET_ENTRIES || nr_pushes > RTE_HASH_V1702_MAX_PUSHES) 465 | return -ENOSPC; 466 | 467 | /* Set flag to indicate that this entry is going to be pushed */ 468 | bkt->flag[i] = 1; 469 | 470 | nr_pushes++; 471 | /* Need room in alternative bucket to insert the pushed entry */ 472 | ret = make_space_bucket(h, next_bkt[i]); 473 | /* 474 | * After recursive function. 475 | * Clear flags and insert the pushed entry 476 | * in its alternative location if successful, 477 | * or return error 478 | */ 479 | bkt->flag[i] = 0; 480 | nr_pushes = 0; 481 | if (ret >= 0) { 482 | next_bkt[i]->sig_alt[ret] = bkt->sig_current[i]; 483 | next_bkt[i]->sig_current[ret] = bkt->sig_alt[i]; 484 | next_bkt[i]->key_idx[ret] = bkt->key_idx[i]; 485 | return i; 486 | } else 487 | return ret; 488 | 489 | } 490 | 491 | /* 492 | * Function called to enqueue back an index in the cache/ring, 493 | * as slot has not being used and it can be used in the 494 | * next addition attempt. 495 | */ 496 | static inline void 497 | enqueue_slot_back(const struct rte_hash_v1702 *h, 498 | struct lcore_cache *cached_free_slots, 499 | void *slot_id) 500 | { 501 | if (h->hw_trans_mem_support) { 502 | cached_free_slots->objs[cached_free_slots->len] = slot_id; 503 | cached_free_slots->len++; 504 | } else 505 | rte_ring_sp_enqueue(h->free_slots, slot_id); 506 | } 507 | 508 | static inline int32_t 509 | __rte_hash_v1702_add_key_with_hash(const struct rte_hash_v1702 *h, const void *key, 510 | hash_sig_t sig, hash_data_t* data) 511 | { 512 | hash_sig_t alt_hash; 513 | uint32_t prim_bucket_idx, sec_bucket_idx; 514 | unsigned i; 515 | struct rte_hash_v1702_bucket *prim_bkt, *sec_bkt; 516 | struct rte_hash_v1702_key *new_k, *k, *keys = h->key_store; 517 | void *slot_id = NULL; 518 | uint32_t new_idx; 519 | int ret; 520 | unsigned n_slots; 521 | unsigned lcore_id; 522 | struct lcore_cache *cached_free_slots = NULL; 523 | 524 | if (h->add_key == ADD_KEY_MULTIWRITER) 525 | rte_spinlock_lock(h->multiwriter_lock); 526 | 527 | prim_bucket_idx = sig & h->bucket_bitmask; 528 | prim_bkt = &h->buckets[prim_bucket_idx]; 529 | rte_prefetch0(prim_bkt); 530 | 531 | alt_hash = rte_hash_v1702_secondary_hash(sig); 532 | sec_bucket_idx = alt_hash & h->bucket_bitmask; 533 | sec_bkt = &h->buckets[sec_bucket_idx]; 534 | rte_prefetch0(sec_bkt); 535 | 536 | /* Get a new slot for storing the new key */ 537 | if (h->hw_trans_mem_support) { 538 | lcore_id = rte_lcore_id(); 539 | cached_free_slots = &h->local_free_slots[lcore_id]; 540 | /* Try to get a free slot from the local cache */ 541 | if (cached_free_slots->len == 0) { 542 | /* Need to get another burst of free slots from global ring */ 543 | n_slots = rte_ring_mc_dequeue_burst(h->free_slots, 544 | cached_free_slots->objs, LCORE_CACHE_SIZE, NULL); 545 | if (n_slots == 0) 546 | return -ENOSPC; 547 | 548 | cached_free_slots->len += n_slots; 549 | } 550 | 551 | /* Get a free slot from the local cache */ 552 | cached_free_slots->len--; 553 | slot_id = cached_free_slots->objs[cached_free_slots->len]; 554 | } else { 555 | if (rte_ring_sc_dequeue(h->free_slots, &slot_id) != 0) 556 | return -ENOSPC; 557 | } 558 | 559 | new_k = RTE_PTR_ADD(keys, (uintptr_t)slot_id * h->key_entry_size); 560 | rte_prefetch0(new_k); 561 | new_idx = (uint32_t)((uintptr_t) slot_id); 562 | 563 | /* Check if key is already inserted in primary location */ 564 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) { 565 | if (prim_bkt->sig_current[i] == sig && 566 | prim_bkt->sig_alt[i] == alt_hash) { 567 | k = (struct rte_hash_v1702_key *) ((char *)keys + 568 | prim_bkt->key_idx[i] * h->key_entry_size); 569 | if (rte_hash_v1702_cmp_eq(key, k->key, h) == 0) { 570 | /* Enqueue index of free slot back in the ring. */ 571 | enqueue_slot_back(h, cached_free_slots, slot_id); 572 | /* Update data */ 573 | k->data = *data; 574 | /* 575 | * Return index where key is stored, 576 | * substracting the first dummy index 577 | */ 578 | return prim_bkt->key_idx[i] - 1; 579 | } 580 | } 581 | } 582 | 583 | /* Check if key is already inserted in secondary location */ 584 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) { 585 | if (sec_bkt->sig_alt[i] == sig && 586 | sec_bkt->sig_current[i] == alt_hash) { 587 | k = (struct rte_hash_v1702_key *) ((char *)keys + 588 | sec_bkt->key_idx[i] * h->key_entry_size); 589 | if (rte_hash_v1702_cmp_eq(key, k->key, h) == 0) { 590 | /* Enqueue index of free slot back in the ring. */ 591 | enqueue_slot_back(h, cached_free_slots, slot_id); 592 | /* Update data */ 593 | k->data = *data; 594 | /* 595 | * Return index where key is stored, 596 | * substracting the first dummy index 597 | */ 598 | return sec_bkt->key_idx[i] - 1; 599 | } 600 | } 601 | } 602 | 603 | /* Copy key */ 604 | rte_memcpy(new_k->key, key, h->key_len); 605 | new_k->data = *data; 606 | 607 | #if defined(RTE_ARCH_X86) /* currently only x86 support HTM */ 608 | if (h->add_key == ADD_KEY_MULTIWRITER_TM) { 609 | ret = rte_hash_v1702_cuckoo_insert_mw_tm(prim_bkt, 610 | sig, alt_hash, new_idx); 611 | if (ret >= 0) 612 | return new_idx - 1; 613 | 614 | /* Primary bucket full, need to make space for new entry */ 615 | ret = rte_hash_v1702_cuckoo_make_space_mw_tm(h, prim_bkt, sig, 616 | alt_hash, new_idx); 617 | 618 | if (ret >= 0) 619 | return new_idx - 1; 620 | 621 | /* Also search secondary bucket to get better occupancy */ 622 | ret = rte_hash_v1702_cuckoo_make_space_mw_tm(h, sec_bkt, sig, 623 | alt_hash, new_idx); 624 | 625 | if (ret >= 0) 626 | return new_idx - 1; 627 | } else { 628 | #endif 629 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) { 630 | /* Check if slot is available */ 631 | if (likely(prim_bkt->key_idx[i] == EMPTY_SLOT)) { 632 | prim_bkt->sig_current[i] = sig; 633 | prim_bkt->sig_alt[i] = alt_hash; 634 | prim_bkt->key_idx[i] = new_idx; 635 | break; 636 | } 637 | } 638 | 639 | if (i != RTE_HASH_V1702_BUCKET_ENTRIES) { 640 | if (h->add_key == ADD_KEY_MULTIWRITER) 641 | rte_spinlock_unlock(h->multiwriter_lock); 642 | return new_idx - 1; 643 | } 644 | 645 | /* Primary bucket full, need to make space for new entry 646 | * After recursive function. 647 | * Insert the new entry in the position of the pushed entry 648 | * if successful or return error and 649 | * store the new slot back in the ring 650 | */ 651 | ret = make_space_bucket(h, prim_bkt); 652 | if (ret >= 0) { 653 | prim_bkt->sig_current[ret] = sig; 654 | prim_bkt->sig_alt[ret] = alt_hash; 655 | prim_bkt->key_idx[ret] = new_idx; 656 | if (h->add_key == ADD_KEY_MULTIWRITER) 657 | rte_spinlock_unlock(h->multiwriter_lock); 658 | return new_idx - 1; 659 | } 660 | #if defined(RTE_ARCH_X86) 661 | } 662 | #endif 663 | /* Error in addition, store new slot back in the ring and return error */ 664 | enqueue_slot_back(h, cached_free_slots, (void *)((uintptr_t) new_idx)); 665 | 666 | if (h->add_key == ADD_KEY_MULTIWRITER) 667 | rte_spinlock_unlock(h->multiwriter_lock); 668 | return ret; 669 | } 670 | 671 | int32_t 672 | rte_hash_v1702_add_key_with_hash(const struct rte_hash_v1702 *h, 673 | const void *key, hash_sig_t sig) 674 | { 675 | RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL); 676 | return __rte_hash_v1702_add_key_with_hash(h, key, sig, NULL); 677 | } 678 | 679 | int32_t 680 | rte_hash_v1702_add_key(const struct rte_hash_v1702 *h, const void *key) 681 | { 682 | RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL); 683 | return __rte_hash_v1702_add_key_with_hash(h, key, rte_hash_v1702_hash(h, key), NULL); 684 | } 685 | 686 | int 687 | rte_hash_v1702_add_key_with_hash_data(const struct rte_hash_v1702 *h, 688 | const void *key, hash_sig_t sig, hash_data_t data) 689 | { 690 | int ret; 691 | 692 | RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL); 693 | ret = __rte_hash_v1702_add_key_with_hash(h, key, sig, &data); 694 | if (ret >= 0) 695 | return 0; 696 | else 697 | return ret; 698 | } 699 | 700 | int 701 | rte_hash_v1702_add_key_data(const struct rte_hash_v1702 *h, const void *key, hash_data_t data) 702 | { 703 | int ret; 704 | 705 | RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL); 706 | 707 | ret = __rte_hash_v1702_add_key_with_hash(h, key, rte_hash_v1702_hash(h, key), &data); 708 | if (ret >= 0) 709 | return 0; 710 | else 711 | return ret; 712 | } 713 | static inline int32_t 714 | __rte_hash_v1702_lookup_with_hash(const struct rte_hash_v1702 *h, const void *key, 715 | hash_sig_t sig, hash_data_t *data) 716 | { 717 | uint32_t bucket_idx; 718 | hash_sig_t alt_hash; 719 | unsigned i; 720 | struct rte_hash_v1702_bucket *bkt; 721 | struct rte_hash_v1702_key *k, *keys = h->key_store; 722 | 723 | bucket_idx = sig & h->bucket_bitmask; 724 | bkt = &h->buckets[bucket_idx]; 725 | 726 | /* Check if key is in primary location */ 727 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) { 728 | if (bkt->sig_current[i] == sig && 729 | bkt->key_idx[i] != EMPTY_SLOT) { 730 | k = (struct rte_hash_v1702_key *) ((char *)keys + 731 | bkt->key_idx[i] * h->key_entry_size); 732 | if (rte_hash_v1702_cmp_eq(key, k->key, h) == 0) { 733 | if (data != NULL) 734 | *data = k->data; 735 | /* 736 | * Return index where key is stored, 737 | * substracting the first dummy index 738 | */ 739 | return bkt->key_idx[i] - 1; 740 | } 741 | } 742 | } 743 | 744 | /* Calculate secondary hash */ 745 | alt_hash = rte_hash_v1702_secondary_hash(sig); 746 | bucket_idx = alt_hash & h->bucket_bitmask; 747 | bkt = &h->buckets[bucket_idx]; 748 | 749 | /* Check if key is in secondary location */ 750 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) { 751 | if (bkt->sig_current[i] == alt_hash && 752 | bkt->sig_alt[i] == sig) { 753 | k = (struct rte_hash_v1702_key *) ((char *)keys + 754 | bkt->key_idx[i] * h->key_entry_size); 755 | if (rte_hash_v1702_cmp_eq(key, k->key, h) == 0) { 756 | if (data != NULL) 757 | *data = k->data; 758 | /* 759 | * Return index where key is stored, 760 | * substracting the first dummy index 761 | */ 762 | return bkt->key_idx[i] - 1; 763 | } 764 | } 765 | } 766 | 767 | return -ENOENT; 768 | } 769 | 770 | int32_t 771 | rte_hash_v1702_lookup_with_hash(const struct rte_hash_v1702 *h, 772 | const void *key, hash_sig_t sig) 773 | { 774 | RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL); 775 | return __rte_hash_v1702_lookup_with_hash(h, key, sig, NULL); 776 | } 777 | 778 | int32_t 779 | rte_hash_v1702_lookup(const struct rte_hash_v1702 *h, const void *key) 780 | { 781 | RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL); 782 | return __rte_hash_v1702_lookup_with_hash(h, key, rte_hash_v1702_hash(h, key), NULL); 783 | } 784 | 785 | int 786 | rte_hash_v1702_lookup_with_hash_data(const struct rte_hash_v1702 *h, 787 | const void *key, hash_sig_t sig, hash_data_t *data) 788 | { 789 | RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL); 790 | return __rte_hash_v1702_lookup_with_hash(h, key, sig, data); 791 | } 792 | 793 | int 794 | rte_hash_v1702_lookup_data(const struct rte_hash_v1702 *h, const void *key, hash_data_t *data) 795 | { 796 | RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL); 797 | return __rte_hash_v1702_lookup_with_hash(h, key, rte_hash_v1702_hash(h, key), data); 798 | } 799 | 800 | static inline void 801 | remove_entry(const struct rte_hash_v1702 *h, struct rte_hash_v1702_bucket *bkt, unsigned i) 802 | { 803 | unsigned lcore_id, n_slots; 804 | struct lcore_cache *cached_free_slots; 805 | 806 | bkt->sig_current[i] = NULL_SIGNATURE; 807 | bkt->sig_alt[i] = NULL_SIGNATURE; 808 | if (h->hw_trans_mem_support) { 809 | lcore_id = rte_lcore_id(); 810 | cached_free_slots = &h->local_free_slots[lcore_id]; 811 | /* Cache full, need to free it. */ 812 | if (cached_free_slots->len == LCORE_CACHE_SIZE) { 813 | /* Need to enqueue the free slots in global ring. */ 814 | n_slots = rte_ring_mp_enqueue_burst(h->free_slots, 815 | cached_free_slots->objs, 816 | LCORE_CACHE_SIZE,NULL); 817 | cached_free_slots->len -= n_slots; 818 | } 819 | /* Put index of new free slot in cache. */ 820 | cached_free_slots->objs[cached_free_slots->len] = 821 | (void *)((uintptr_t)bkt->key_idx[i]); 822 | cached_free_slots->len++; 823 | } else { 824 | rte_ring_sp_enqueue(h->free_slots, 825 | (void *)((uintptr_t)bkt->key_idx[i])); 826 | } 827 | } 828 | 829 | static inline int32_t 830 | __rte_hash_v1702_del_key_with_hash(const struct rte_hash_v1702 *h, const void *key, 831 | hash_sig_t sig) 832 | { 833 | uint32_t bucket_idx; 834 | hash_sig_t alt_hash; 835 | unsigned i; 836 | struct rte_hash_v1702_bucket *bkt; 837 | struct rte_hash_v1702_key *k, *keys = h->key_store; 838 | int32_t ret; 839 | 840 | bucket_idx = sig & h->bucket_bitmask; 841 | bkt = &h->buckets[bucket_idx]; 842 | 843 | /* Check if key is in primary location */ 844 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) { 845 | if (bkt->sig_current[i] == sig && 846 | bkt->key_idx[i] != EMPTY_SLOT) { 847 | k = (struct rte_hash_v1702_key *) ((char *)keys + 848 | bkt->key_idx[i] * h->key_entry_size); 849 | if (rte_hash_v1702_cmp_eq(key, k->key, h) == 0) { 850 | remove_entry(h, bkt, i); 851 | 852 | /* 853 | * Return index where key is stored, 854 | * substracting the first dummy index 855 | */ 856 | ret = bkt->key_idx[i] - 1; 857 | bkt->key_idx[i] = EMPTY_SLOT; 858 | return ret; 859 | } 860 | } 861 | } 862 | 863 | /* Calculate secondary hash */ 864 | alt_hash = rte_hash_v1702_secondary_hash(sig); 865 | bucket_idx = alt_hash & h->bucket_bitmask; 866 | bkt = &h->buckets[bucket_idx]; 867 | 868 | /* Check if key is in secondary location */ 869 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) { 870 | if (bkt->sig_current[i] == alt_hash && 871 | bkt->key_idx[i] != EMPTY_SLOT) { 872 | k = (struct rte_hash_v1702_key *) ((char *)keys + 873 | bkt->key_idx[i] * h->key_entry_size); 874 | if (rte_hash_v1702_cmp_eq(key, k->key, h) == 0) { 875 | remove_entry(h, bkt, i); 876 | 877 | /* 878 | * Return index where key is stored, 879 | * substracting the first dummy index 880 | */ 881 | ret = bkt->key_idx[i] - 1; 882 | bkt->key_idx[i] = EMPTY_SLOT; 883 | return ret; 884 | } 885 | } 886 | } 887 | 888 | return -ENOENT; 889 | } 890 | 891 | int32_t 892 | rte_hash_v1702_del_key_with_hash(const struct rte_hash_v1702 *h, 893 | const void *key, hash_sig_t sig) 894 | { 895 | RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL); 896 | return __rte_hash_v1702_del_key_with_hash(h, key, sig); 897 | } 898 | 899 | int32_t 900 | rte_hash_v1702_del_key(const struct rte_hash_v1702 *h, const void *key) 901 | { 902 | RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL); 903 | return __rte_hash_v1702_del_key_with_hash(h, key, rte_hash_v1702_hash(h, key)); 904 | } 905 | 906 | int 907 | rte_hash_v1702_get_key_with_position(const struct rte_hash_v1702 *h, const int32_t position, 908 | void **key) 909 | { 910 | RETURN_IF_TRUE(((h == NULL) || (key == NULL)), -EINVAL); 911 | 912 | struct rte_hash_v1702_key *k, *keys = h->key_store; 913 | k = (struct rte_hash_v1702_key *) ((char *) keys + (position + 1) * 914 | h->key_entry_size); 915 | *key = k->key; 916 | 917 | if (position != 918 | __rte_hash_v1702_lookup_with_hash(h, *key, rte_hash_v1702_hash(h, *key), 919 | NULL)) { 920 | return -ENOENT; 921 | } 922 | 923 | return 0; 924 | } 925 | 926 | static inline void 927 | compare_signatures(uint32_t *prim_hash_matches, uint32_t *sec_hash_matches, 928 | const struct rte_hash_v1702_bucket *prim_bkt, 929 | const struct rte_hash_v1702_bucket *sec_bkt, 930 | hash_sig_t prim_hash, hash_sig_t sec_hash, 931 | enum rte_hash_v1702_sig_compare_function sig_cmp_fn) 932 | { 933 | unsigned int i; 934 | 935 | switch (sig_cmp_fn) { 936 | #ifdef RTE_MACHINE_CPUFLAG_AVX2 937 | case RTE_HASH_V1702_COMPARE_AVX2: 938 | *prim_hash_matches = _mm256_movemask_ps((__m256)_mm256_cmpeq_epi32( 939 | _mm256_load_si256( 940 | (__m256i const *)prim_bkt->sig_current), 941 | _mm256_set1_epi32(prim_hash))); 942 | *sec_hash_matches = _mm256_movemask_ps((__m256)_mm256_cmpeq_epi32( 943 | _mm256_load_si256( 944 | (__m256i const *)sec_bkt->sig_current), 945 | _mm256_set1_epi32(sec_hash))); 946 | break; 947 | #endif 948 | #ifdef RTE_MACHINE_CPUFLAG_SSE2 949 | case RTE_HASH_V1702_COMPARE_SSE: 950 | /* Compare the first 4 signatures in the bucket */ 951 | *prim_hash_matches = _mm_movemask_ps((__m128)_mm_cmpeq_epi16( 952 | _mm_load_si128( 953 | (__m128i const *)prim_bkt->sig_current), 954 | _mm_set1_epi32(prim_hash))); 955 | *prim_hash_matches |= (_mm_movemask_ps((__m128)_mm_cmpeq_epi16( 956 | _mm_load_si128( 957 | (__m128i const *)&prim_bkt->sig_current[4]), 958 | _mm_set1_epi32(prim_hash)))) << 4; 959 | /* Compare the first 4 signatures in the bucket */ 960 | *sec_hash_matches = _mm_movemask_ps((__m128)_mm_cmpeq_epi16( 961 | _mm_load_si128( 962 | (__m128i const *)sec_bkt->sig_current), 963 | _mm_set1_epi32(sec_hash))); 964 | *sec_hash_matches |= (_mm_movemask_ps((__m128)_mm_cmpeq_epi16( 965 | _mm_load_si128( 966 | (__m128i const *)&sec_bkt->sig_current[4]), 967 | _mm_set1_epi32(sec_hash)))) << 4; 968 | break; 969 | #endif 970 | default: 971 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) { 972 | *prim_hash_matches |= 973 | ((prim_hash == prim_bkt->sig_current[i]) << i); 974 | *sec_hash_matches |= 975 | ((sec_hash == sec_bkt->sig_current[i]) << i); 976 | } 977 | } 978 | 979 | } 980 | 981 | #define PREFETCH_OFFSET 4 982 | static inline void 983 | __rte_hash_v1702_lookup_bulk(const struct rte_hash_v1702 *h, const void **keys, 984 | int32_t num_keys, int32_t *positions, 985 | uint64_t *hit_mask, hash_data_t data[]) 986 | { 987 | uint64_t hits = 0; 988 | int32_t i; 989 | uint32_t prim_hash[RTE_HASH_V1702_LOOKUP_BULK_MAX]; 990 | uint32_t sec_hash[RTE_HASH_V1702_LOOKUP_BULK_MAX]; 991 | const struct rte_hash_v1702_bucket *primary_bkt[RTE_HASH_V1702_LOOKUP_BULK_MAX]; 992 | const struct rte_hash_v1702_bucket *secondary_bkt[RTE_HASH_V1702_LOOKUP_BULK_MAX]; 993 | uint32_t prim_hitmask[RTE_HASH_V1702_LOOKUP_BULK_MAX] = {0}; 994 | uint32_t sec_hitmask[RTE_HASH_V1702_LOOKUP_BULK_MAX] = {0}; 995 | 996 | /* Prefetch first keys */ 997 | for (i = 0; i < PREFETCH_OFFSET && i < num_keys; i++) 998 | rte_prefetch0(keys[i]); 999 | 1000 | /* 1001 | * Prefetch rest of the keys, calculate primary and 1002 | * secondary bucket and prefetch them 1003 | */ 1004 | for (i = 0; i < (num_keys - PREFETCH_OFFSET); i++) { 1005 | rte_prefetch0(keys[i + PREFETCH_OFFSET]); 1006 | 1007 | prim_hash[i] = rte_hash_v1702_hash(h, keys[i]); 1008 | sec_hash[i] = rte_hash_v1702_secondary_hash(prim_hash[i]); 1009 | 1010 | primary_bkt[i] = &h->buckets[prim_hash[i] & h->bucket_bitmask]; 1011 | secondary_bkt[i] = &h->buckets[sec_hash[i] & h->bucket_bitmask]; 1012 | 1013 | rte_prefetch0(primary_bkt[i]); 1014 | rte_prefetch0(secondary_bkt[i]); 1015 | } 1016 | 1017 | /* Calculate and prefetch rest of the buckets */ 1018 | for (; i < num_keys; i++) { 1019 | prim_hash[i] = rte_hash_v1702_hash(h, keys[i]); 1020 | sec_hash[i] = rte_hash_v1702_secondary_hash(prim_hash[i]); 1021 | 1022 | primary_bkt[i] = &h->buckets[prim_hash[i] & h->bucket_bitmask]; 1023 | secondary_bkt[i] = &h->buckets[sec_hash[i] & h->bucket_bitmask]; 1024 | 1025 | rte_prefetch0(primary_bkt[i]); 1026 | rte_prefetch0(secondary_bkt[i]); 1027 | } 1028 | 1029 | /* Compare signatures and prefetch key slot of first hit */ 1030 | for (i = 0; i < num_keys; i++) { 1031 | compare_signatures(&prim_hitmask[i], &sec_hitmask[i], 1032 | primary_bkt[i], secondary_bkt[i], 1033 | prim_hash[i], sec_hash[i], h->sig_cmp_fn); 1034 | 1035 | if (prim_hitmask[i]) { 1036 | uint32_t first_hit = __builtin_ctzl(prim_hitmask[i]); 1037 | uint32_t key_idx = primary_bkt[i]->key_idx[first_hit]; 1038 | const struct rte_hash_v1702_key *key_slot = 1039 | (const struct rte_hash_v1702_key *)( 1040 | (const char *)h->key_store + 1041 | key_idx * h->key_entry_size); 1042 | rte_prefetch0(key_slot); 1043 | continue; 1044 | } 1045 | 1046 | if (sec_hitmask[i]) { 1047 | uint32_t first_hit = __builtin_ctzl(sec_hitmask[i]); 1048 | uint32_t key_idx = secondary_bkt[i]->key_idx[first_hit]; 1049 | const struct rte_hash_v1702_key *key_slot = 1050 | (const struct rte_hash_v1702_key *)( 1051 | (const char *)h->key_store + 1052 | key_idx * h->key_entry_size); 1053 | rte_prefetch0(key_slot); 1054 | } 1055 | } 1056 | 1057 | /* Compare keys, first hits in primary first */ 1058 | for (i = 0; i < num_keys; i++) { 1059 | positions[i] = -ENOENT; 1060 | while (prim_hitmask[i]) { 1061 | uint32_t hit_index = __builtin_ctzl(prim_hitmask[i]); 1062 | 1063 | uint32_t key_idx = primary_bkt[i]->key_idx[hit_index]; 1064 | const struct rte_hash_v1702_key *key_slot = 1065 | (const struct rte_hash_v1702_key *)( 1066 | (const char *)h->key_store + 1067 | key_idx * h->key_entry_size); 1068 | /* 1069 | * If key index is 0, do not compare key, 1070 | * as it is checking the dummy slot 1071 | */ 1072 | if (!!key_idx & !rte_hash_v1702_cmp_eq(key_slot->key, keys[i], h)) { 1073 | if (data != NULL) 1074 | data[i] = key_slot->data; 1075 | 1076 | hits |= 1ULL << i; 1077 | positions[i] = key_idx - 1; 1078 | goto next_key; 1079 | } 1080 | prim_hitmask[i] &= ~(1 << (hit_index)); 1081 | } 1082 | 1083 | while (sec_hitmask[i]) { 1084 | uint32_t hit_index = __builtin_ctzl(sec_hitmask[i]); 1085 | 1086 | uint32_t key_idx = secondary_bkt[i]->key_idx[hit_index]; 1087 | const struct rte_hash_v1702_key *key_slot = 1088 | (const struct rte_hash_v1702_key *)( 1089 | (const char *)h->key_store + 1090 | key_idx * h->key_entry_size); 1091 | /* 1092 | * If key index is 0, do not compare key, 1093 | * as it is checking the dummy slot 1094 | */ 1095 | 1096 | if (!!key_idx & !rte_hash_v1702_cmp_eq(key_slot->key, keys[i], h)) { 1097 | if (data != NULL) 1098 | data[i] = key_slot->data; 1099 | 1100 | hits |= 1ULL << i; 1101 | positions[i] = key_idx - 1; 1102 | goto next_key; 1103 | } 1104 | sec_hitmask[i] &= ~(1 << (hit_index)); 1105 | } 1106 | 1107 | next_key: 1108 | continue; 1109 | } 1110 | 1111 | if (hit_mask != NULL) 1112 | *hit_mask = hits; 1113 | } 1114 | 1115 | int 1116 | rte_hash_v1702_lookup_bulk(const struct rte_hash_v1702 *h, const void **keys, 1117 | uint32_t num_keys, int32_t *positions) 1118 | { 1119 | RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) || 1120 | (num_keys > RTE_HASH_V1702_LOOKUP_BULK_MAX) || 1121 | (positions == NULL)), -EINVAL); 1122 | 1123 | __rte_hash_v1702_lookup_bulk(h, keys, num_keys, positions, NULL, NULL); 1124 | return 0; 1125 | } 1126 | 1127 | int 1128 | rte_hash_v1702_lookup_bulk_data(const struct rte_hash_v1702 *h, const void **keys, 1129 | uint32_t num_keys, uint64_t *hit_mask, hash_data_t data[]) 1130 | { 1131 | RETURN_IF_TRUE(((h == NULL) || (keys == NULL) || (num_keys == 0) || 1132 | (num_keys > RTE_HASH_V1702_LOOKUP_BULK_MAX) || 1133 | (hit_mask == NULL)), -EINVAL); 1134 | 1135 | int32_t positions[num_keys]; 1136 | 1137 | __rte_hash_v1702_lookup_bulk(h, keys, num_keys, positions, hit_mask, data); 1138 | 1139 | /* Return number of hits */ 1140 | return __builtin_popcountl(*hit_mask); 1141 | } 1142 | 1143 | 1144 | 1145 | int32_t 1146 | rte_hash_v1702_iterate(const struct rte_hash_v1702 *h, const void **key, hash_data_t *data, uint32_t *next) 1147 | { 1148 | uint32_t bucket_idx, idx, position; 1149 | struct rte_hash_v1702_key *next_key; 1150 | 1151 | RETURN_IF_TRUE(((h == NULL) || (next == NULL)), -EINVAL); 1152 | 1153 | const uint32_t total_entries = h->num_buckets * RTE_HASH_V1702_BUCKET_ENTRIES; 1154 | /* Out of bounds */ 1155 | if (*next >= total_entries) 1156 | return -ENOENT; 1157 | 1158 | /* Calculate bucket and index of current iterator */ 1159 | bucket_idx = *next / RTE_HASH_V1702_BUCKET_ENTRIES; 1160 | idx = *next % RTE_HASH_V1702_BUCKET_ENTRIES; 1161 | 1162 | /* If current position is empty, go to the next one */ 1163 | while (h->buckets[bucket_idx].key_idx[idx] == EMPTY_SLOT) { 1164 | (*next)++; 1165 | /* End of table */ 1166 | if (*next == total_entries) 1167 | return -ENOENT; 1168 | bucket_idx = *next / RTE_HASH_V1702_BUCKET_ENTRIES; 1169 | idx = *next % RTE_HASH_V1702_BUCKET_ENTRIES; 1170 | } 1171 | 1172 | /* Get position of entry in key table */ 1173 | position = h->buckets[bucket_idx].key_idx[idx]; 1174 | next_key = (struct rte_hash_v1702_key *) ((char *)h->key_store + 1175 | position * h->key_entry_size); 1176 | /* Return key and data */ 1177 | *key = next_key->key; 1178 | *data = next_key->data; 1179 | 1180 | /* Increment iterator */ 1181 | (*next)++; 1182 | 1183 | return position - 1; 1184 | } 1185 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cuckoo_hash_v1702.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * Minor modifications from vanilla DPDK 17.02 are licensed under Clear BSD. 3 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 4 | * 5 | * Original code is licensed under: 6 | * 7 | * BSD LICENSE 8 | * 9 | * Copyright(c) 2016 Intel Corporation. All rights reserved. 10 | * All rights reserved. 11 | * 12 | * Redistribution and use in source and binary forms, with or without 13 | * modification, are permitted provided that the following conditions 14 | * are met: 15 | * 16 | * * Redistributions of source code must retain the above copyright 17 | * notice, this list of conditions and the following disclaimer. 18 | * * Redistributions in binary form must reproduce the above copyright 19 | * notice, this list of conditions and the following disclaimer in 20 | * the documentation and/or other materials provided with the 21 | * distribution. 22 | * * Neither the name of Intel Corporation nor the names of its 23 | * contributors may be used to endorse or promote products derived 24 | * from this software without specific prior written permission. 25 | * 26 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | */ 38 | 39 | /* rte_cuckoo_hash.h 40 | * This file hold Cuckoo Hash private data structures to allows include from 41 | * platform specific files like rte_cuckoo_hash_x86.h 42 | */ 43 | 44 | #ifndef _RTE_CUCKOO_HASH_V1702_H_ 45 | #define _RTE_CUCKOO_HASH_V1702_H_ 46 | 47 | #include "rte_tchh_structs.h" 48 | 49 | #if defined(RTE_ARCH_X86) 50 | #include "rte_cmp_x86_v1702.h" 51 | #endif 52 | 53 | #if defined(RTE_ARCH_ARM64) 54 | #include "rte_cmp_arm64.h" 55 | #endif 56 | 57 | /* Macro to enable/disable run-time checking of function parameters */ 58 | #if defined(RTE_LIBRTE_HASH_V1702_DEBUG) 59 | #define RETURN_IF_TRUE(cond, retval) do { \ 60 | if (cond) \ 61 | return retval; \ 62 | } while (0) 63 | #else 64 | #define RETURN_IF_TRUE(cond, retval) 65 | #endif 66 | 67 | /* Hash function used if none is specified */ 68 | #if defined(RTE_MACHINE_CPUFLAG_SSE4_2) || defined(RTE_MACHINE_CPUFLAG_CRC32) 69 | #include 70 | #define DEFAULT_HASH_FUNC rte_hash_crc 71 | #else 72 | #include 73 | #define DEFAULT_HASH_FUNC rte_jhash 74 | #endif 75 | 76 | #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) 77 | /* 78 | * All different options to select a key compare function, 79 | * based on the key size and custom function. 80 | */ 81 | enum cmp_jump_table_case { 82 | KEY_CUSTOM = 0, 83 | KEY_16_BYTES, 84 | KEY_32_BYTES, 85 | KEY_48_BYTES, 86 | KEY_64_BYTES, 87 | KEY_80_BYTES, 88 | KEY_96_BYTES, 89 | KEY_112_BYTES, 90 | KEY_128_BYTES, 91 | KEY_OTHER_BYTES, 92 | NUM_KEY_CMP_CASES, 93 | }; 94 | 95 | /* 96 | * Table storing all different key compare functions 97 | * (multi-process supported) 98 | */ 99 | const rte_hash_v1702_cmp_eq_t cmp_jump_table_v1702[NUM_KEY_CMP_CASES] = { 100 | NULL, 101 | rte_hash_v1702_k16_cmp_eq, 102 | rte_hash_v1702_k32_cmp_eq, 103 | rte_hash_v1702_k48_cmp_eq, 104 | rte_hash_v1702_k64_cmp_eq, 105 | rte_hash_v1702_k80_cmp_eq, 106 | rte_hash_v1702_k96_cmp_eq, 107 | rte_hash_v1702_k112_cmp_eq, 108 | rte_hash_v1702_k128_cmp_eq, 109 | memcmp 110 | }; 111 | #else 112 | /* 113 | * All different options to select a key compare function, 114 | * based on the key size and custom function. 115 | */ 116 | enum cmp_jump_table_case { 117 | KEY_CUSTOM = 0, 118 | KEY_OTHER_BYTES, 119 | NUM_KEY_CMP_CASES, 120 | }; 121 | 122 | /* 123 | * Table storing all different key compare functions 124 | * (multi-process supported) 125 | */ 126 | const rte_hash_v1702_cmp_eq_t cmp_jump_table[NUM_KEY_CMP_CASES] = { 127 | NULL, 128 | memcmp 129 | }; 130 | 131 | #endif 132 | 133 | enum add_key_case { 134 | ADD_KEY_SINGLEWRITER = 0, 135 | ADD_KEY_MULTIWRITER, 136 | ADD_KEY_MULTIWRITER_TM, 137 | }; 138 | 139 | /** Number of items per bucket. */ 140 | #define RTE_HASH_V1702_BUCKET_ENTRIES 8 141 | 142 | #define NULL_SIGNATURE 0 143 | 144 | #define EMPTY_SLOT 0 145 | 146 | #define KEY_ALIGNMENT 16 147 | 148 | #define LCORE_CACHE_SIZE 64 149 | 150 | #define RTE_HASH_V1702_MAX_PUSHES 100 151 | 152 | #define RTE_HASH_V1702_BFS_QUEUE_MAX_LEN 1000 153 | 154 | #define RTE_XABORT_CUCKOO_PATH_INVALIDED 0x4 155 | 156 | #define RTE_HASH_V1702_TSX_MAX_RETRY 10 157 | 158 | struct lcore_cache { 159 | unsigned len; /**< Cache len */ 160 | void *objs[LCORE_CACHE_SIZE]; /**< Cache objects */ 161 | } __rte_cache_aligned; 162 | 163 | /* Structure that stores key-value pair */ 164 | struct rte_hash_v1702_key { 165 | struct rte_tch_data data; 166 | /* Variable key size */ 167 | char key[0]; 168 | } __attribute__((aligned(KEY_ALIGNMENT))); 169 | 170 | /* All different signature compare functions */ 171 | enum rte_hash_v1702_sig_compare_function { 172 | RTE_HASH_V1702_COMPARE_SCALAR = 0, 173 | RTE_HASH_V1702_COMPARE_SSE, 174 | RTE_HASH_V1702_COMPARE_AVX2, 175 | RTE_HASH_V1702_COMPARE_NUM 176 | }; 177 | 178 | /** Bucket structure */ 179 | struct rte_hash_v1702_bucket { 180 | hash_sig_t sig_current[RTE_HASH_V1702_BUCKET_ENTRIES]; 181 | 182 | uint32_t key_idx[RTE_HASH_V1702_BUCKET_ENTRIES]; 183 | 184 | hash_sig_t sig_alt[RTE_HASH_V1702_BUCKET_ENTRIES]; 185 | 186 | uint8_t flag[RTE_HASH_V1702_BUCKET_ENTRIES]; 187 | } __rte_cache_aligned; 188 | 189 | /** A hash table structure. */ 190 | struct rte_hash_v1702 { 191 | char name[RTE_HASH_V1702_NAMESIZE]; /**< Name of the hash. */ 192 | uint32_t entries; /**< Total table entries. */ 193 | uint32_t num_buckets; /**< Number of buckets in table. */ 194 | 195 | struct rte_ring *free_slots; 196 | /**< Ring that stores all indexes of the free slots in the key table */ 197 | uint8_t hw_trans_mem_support; 198 | /**< Hardware transactional memory support */ 199 | struct lcore_cache *local_free_slots; 200 | /**< Local cache per lcore, storing some indexes of the free slots */ 201 | enum add_key_case add_key; /**< Multi-writer hash add behavior */ 202 | 203 | rte_spinlock_t *multiwriter_lock; /**< Multi-writer spinlock for w/o TM */ 204 | 205 | /* Fields used in lookup */ 206 | 207 | uint32_t key_len __rte_cache_aligned; 208 | /**< Length of hash key. */ 209 | rte_hash_v1702_function hash_func; /**< Function used to calculate hash. */ 210 | uint32_t hash_func_init_val; /**< Init value used by hash_func. */ 211 | rte_hash_v1702_cmp_eq_t rte_hash_v1702_custom_cmp_eq; 212 | /**< Custom function used to compare keys. */ 213 | enum cmp_jump_table_case cmp_jump_table_idx; 214 | /**< Indicates which compare function to use. */ 215 | enum rte_hash_v1702_sig_compare_function sig_cmp_fn; 216 | /**< Indicates which signature compare function to use. */ 217 | uint32_t bucket_bitmask; 218 | /**< Bitmask for getting bucket index from hash signature. */ 219 | uint32_t key_entry_size; /**< Size of each key entry. */ 220 | 221 | void *key_store; /**< Table storing all keys and data */ 222 | struct rte_hash_v1702_bucket *buckets; 223 | /**< Table with buckets storing all the hash values and key indexes 224 | * to the key table. 225 | */ 226 | } __rte_cache_aligned; 227 | 228 | struct queue_node { 229 | struct rte_hash_v1702_bucket *bkt; /* Current bucket on the bfs search */ 230 | 231 | struct queue_node *prev; /* Parent(bucket) in search path */ 232 | int prev_slot; /* Parent(slot) in search path */ 233 | }; 234 | 235 | #endif 236 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_cuckoo_hash_x86_v1702.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * Minor modifications from vanilla DPDK 17.02 are licensed under Clear BSD. 3 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 4 | * 5 | * Original code is licensed under: 6 | * 7 | * BSD LICENSE 8 | * 9 | * Copyright(c) 2016 Intel Corporation. All rights reserved. 10 | * All rights reserved. 11 | * 12 | * Redistribution and use in source and binary forms, with or without 13 | * modification, are permitted provided that the following conditions 14 | * are met: 15 | * 16 | * * Redistributions of source code must retain the above copyright 17 | * notice, this list of conditions and the following disclaimer. 18 | * * Redistributions in binary form must reproduce the above copyright 19 | * notice, this list of conditions and the following disclaimer in 20 | * the documentation and/or other materials provided with the 21 | * distribution. 22 | * * Neither the name of Intel Corporation nor the names of its 23 | * contributors may be used to endorse or promote products derived 24 | * from this software without specific prior written permission. 25 | * 26 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | */ 38 | 39 | /* rte_cuckoo_hash_x86.h 40 | * This file holds all x86 specific Cuckoo Hash functions 41 | */ 42 | 43 | /* Only tries to insert at one bucket (@prim_bkt) without trying to push 44 | * buckets around 45 | */ 46 | static inline unsigned 47 | rte_hash_v1702_cuckoo_insert_mw_tm(struct rte_hash_v1702_bucket *prim_bkt, 48 | hash_sig_t sig, hash_sig_t alt_hash, uint32_t new_idx) 49 | { 50 | unsigned i, status; 51 | unsigned try = 0; 52 | 53 | while (try < RTE_HASH_V1702_TSX_MAX_RETRY) { 54 | status = rte_xbegin(); 55 | if (likely(status == RTE_XBEGIN_STARTED)) { 56 | /* Insert new entry if there is room in the primary 57 | * bucket. 58 | */ 59 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) { 60 | /* Check if slot is available */ 61 | if (likely(prim_bkt->key_idx[i] == EMPTY_SLOT)) { 62 | prim_bkt->sig_current[i] = sig; 63 | prim_bkt->sig_alt[i] = alt_hash; 64 | prim_bkt->key_idx[i] = new_idx; 65 | break; 66 | } 67 | } 68 | rte_xend(); 69 | 70 | if (i != RTE_HASH_V1702_BUCKET_ENTRIES) 71 | return 0; 72 | 73 | break; /* break off try loop if transaction commits */ 74 | } else { 75 | /* If we abort we give up this cuckoo path. */ 76 | try++; 77 | rte_pause(); 78 | } 79 | } 80 | 81 | return -1; 82 | } 83 | 84 | /* Shift buckets along provided cuckoo_path (@leaf and @leaf_slot) and fill 85 | * the path head with new entry (sig, alt_hash, new_idx) 86 | */ 87 | static inline int 88 | rte_hash_v1702_cuckoo_move_insert_mw_tm(const struct rte_hash_v1702 *h, 89 | struct queue_node *leaf, uint32_t leaf_slot, 90 | hash_sig_t sig, hash_sig_t alt_hash, uint32_t new_idx) 91 | { 92 | unsigned try = 0; 93 | unsigned status; 94 | uint32_t prev_alt_bkt_idx; 95 | 96 | struct queue_node *prev_node, *curr_node = leaf; 97 | struct rte_hash_v1702_bucket *prev_bkt, *curr_bkt = leaf->bkt; 98 | uint32_t prev_slot, curr_slot = leaf_slot; 99 | 100 | while (try < RTE_HASH_V1702_TSX_MAX_RETRY) { 101 | status = rte_xbegin(); 102 | if (likely(status == RTE_XBEGIN_STARTED)) { 103 | while (likely(curr_node->prev != NULL)) { 104 | prev_node = curr_node->prev; 105 | prev_bkt = prev_node->bkt; 106 | prev_slot = curr_node->prev_slot; 107 | 108 | prev_alt_bkt_idx 109 | = prev_bkt->sig_alt[prev_slot] 110 | & h->bucket_bitmask; 111 | 112 | if (unlikely(&h->buckets[prev_alt_bkt_idx] 113 | != curr_bkt)) { 114 | rte_xabort(RTE_XABORT_CUCKOO_PATH_INVALIDED); 115 | } 116 | 117 | /* Need to swap current/alt sig to allow later 118 | * Cuckoo insert to move elements back to its 119 | * primary bucket if available 120 | */ 121 | curr_bkt->sig_alt[curr_slot] = 122 | prev_bkt->sig_current[prev_slot]; 123 | curr_bkt->sig_current[curr_slot] = 124 | prev_bkt->sig_alt[prev_slot]; 125 | curr_bkt->key_idx[curr_slot] 126 | = prev_bkt->key_idx[prev_slot]; 127 | 128 | curr_slot = prev_slot; 129 | curr_node = prev_node; 130 | curr_bkt = curr_node->bkt; 131 | } 132 | 133 | curr_bkt->sig_current[curr_slot] = sig; 134 | curr_bkt->sig_alt[curr_slot] = alt_hash; 135 | curr_bkt->key_idx[curr_slot] = new_idx; 136 | 137 | rte_xend(); 138 | 139 | return 0; 140 | } 141 | 142 | /* If we abort we give up this cuckoo path, since most likely it's 143 | * no longer valid as TSX detected data conflict 144 | */ 145 | try++; 146 | rte_pause(); 147 | } 148 | 149 | return -1; 150 | } 151 | 152 | /* 153 | * Make space for new key, using bfs Cuckoo Search and Multi-Writer safe 154 | * Cuckoo 155 | */ 156 | static inline int 157 | rte_hash_v1702_cuckoo_make_space_mw_tm(const struct rte_hash_v1702 *h, 158 | struct rte_hash_v1702_bucket *bkt, 159 | hash_sig_t sig, hash_sig_t alt_hash, 160 | uint32_t new_idx) 161 | { 162 | unsigned i; 163 | struct queue_node queue[RTE_HASH_V1702_BFS_QUEUE_MAX_LEN]; 164 | struct queue_node *tail, *head; 165 | struct rte_hash_v1702_bucket *curr_bkt, *alt_bkt; 166 | 167 | tail = queue; 168 | head = queue + 1; 169 | tail->bkt = bkt; 170 | tail->prev = NULL; 171 | tail->prev_slot = -1; 172 | 173 | /* Cuckoo bfs Search */ 174 | while (likely(tail != head && head < 175 | queue + RTE_HASH_V1702_BFS_QUEUE_MAX_LEN - 176 | RTE_HASH_V1702_BUCKET_ENTRIES)) { 177 | curr_bkt = tail->bkt; 178 | for (i = 0; i < RTE_HASH_V1702_BUCKET_ENTRIES; i++) { 179 | if (curr_bkt->key_idx[i] == EMPTY_SLOT) { 180 | if (likely(rte_hash_v1702_cuckoo_move_insert_mw_tm(h, 181 | tail, i, sig, 182 | alt_hash, new_idx) == 0)) 183 | return 0; 184 | } 185 | 186 | /* Enqueue new node and keep prev node info */ 187 | alt_bkt = &(h->buckets[curr_bkt->sig_alt[i] 188 | & h->bucket_bitmask]); 189 | head->bkt = alt_bkt; 190 | head->prev = tail; 191 | head->prev_slot = i; 192 | head++; 193 | } 194 | tail++; 195 | } 196 | 197 | return -ENOSPC; 198 | } 199 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash64.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | 10 | #ifndef RTE_HASH64_H 11 | #define RTE_HASH64_H 12 | 13 | #include "x86intrin.h" 14 | #include "immintrin.h" 15 | #include "rte_tchh_structs.h" 16 | 17 | 18 | /** 19 | * Hash function based on two CRC32 with different seeds (instead of one in DPDK) 20 | */ 21 | static inline uint64_t dcrc_hash_m128(const hash_key_t k) { 22 | uint64_t a = k.a; 23 | uint64_t b = k.b; 24 | uint64_t crc00 = _mm_crc32_u64(0,a); 25 | uint64_t crc01 = _mm_crc32_u64(crc00,b); 26 | uint64_t crc10 = _mm_crc32_u64(0x5bd1e995,b); 27 | uint64_t crc11 = _mm_crc32_u64(crc10,a); 28 | uint64_t crc64 = (crc11 << 32)| crc01 ; 29 | return crc64; 30 | } 31 | 32 | 33 | 34 | 35 | 36 | /** 37 | * This is the function that should be called in all places so that using another hash function only 38 | * requires to change this function. 39 | */ 40 | static inline uint32_t rte_tch_hash_function(const void *key, __rte_unused uint32_t key_len, 41 | __rte_unused uint32_t init_val){ 42 | return dcrc_hash_m128(*(const hash_key_t*)key); 43 | } 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_bloom.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #ifndef LIBRTE_TCH_HASH_RTE_HASH_BLOOM_H_ 10 | #define LIBRTE_TCH_HASH_RTE_HASH_BLOOM_H_ 11 | 12 | #ifdef H 13 | #undef H 14 | #undef BLOOM 15 | #undef HORTON 16 | #undef UNCONDITIONAL_PREFETCH 17 | #undef CONDITIONAL_PREFETCH 18 | #undef NO_PREFETCH 19 | #undef TIMER 20 | #endif 21 | 22 | 23 | #define H(x,y) x##_bloom_##y 24 | 25 | /* Configuration Flags */ 26 | /* A. When to prefetch bucket -- Should be set to BLOOM*/ 27 | #define BLOOM 1 28 | #define HORTON 0 29 | #define UNCONDITIONAL_PREFETCH 0 30 | #define CONDITIONAL_PREFETCH 0 31 | #define NO_PREFETCH 0 32 | /* B. Enable time management or not -- Should be enabled*/ 33 | #define TIMER 0 34 | 35 | 36 | 37 | #include "rte_hash_template.h" 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_commons.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #ifndef RTE_HASH_COMMONS 10 | #define RTE_HASH_COMMONS 11 | /** 12 | * @file 13 | * 14 | * RTE Hash Table 15 | */ 16 | 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | #ifdef __cplusplus 23 | extern "C" { 24 | #endif 25 | 26 | 27 | /** 28 | * Max exiration period 29 | */ 30 | #define RTE_HASH_HVARIANT_MAX_EXPIRATION_PERIOD ((uint32_t)1024U) 31 | 32 | /** Maximum size of hash table that can be created. */ 33 | #define RTE_HASH_HVARIANT_ENTRIES_MAX (1 << 30) 34 | 35 | /** Maximum number of characters in hash name.*/ 36 | #define RTE_HASH_HVARIANT_NAMESIZE 32 37 | 38 | /** Maximum number of keys that can be searched for using rte_hash_hvariant_lookup_bulk. */ 39 | #define RTE_HASH_HVARIANT_LOOKUP_BULK_MAX 64 40 | #define RTE_HASH_HVARIANT_LOOKUP_MULTI_MAX RTE_HASH_HVARIANT_LOOKUP_BULK_MAX 41 | 42 | /** Constants for returning results of operation */ 43 | #define RHL_NOT_FOUND -ENOENT 44 | #define RHL_NOT_ADDED -ENOSPC 45 | #define RHL_FOUND_UPDATED 1 46 | #define RHL_FOUND_NOTUPDATED 2 47 | 48 | /** Signature of key that is stored internally. */ 49 | typedef uint32_t hash_sig32_t; 50 | 51 | /** 52 | * Parameters used when creating the hash table. 53 | */ 54 | struct rte_hash_hvariant_parameters { 55 | const char *name; /**< Name of the hash. */ 56 | uint32_t entries; /**< Total hash table entries. */ 57 | int socket_id; /**< NUMA Socket ID for memory. */ 58 | }; 59 | 60 | /** @internal A hash table structure. */ 61 | struct rte_hash_hvariant; 62 | 63 | 64 | #ifdef __cplusplus 65 | } 66 | #endif 67 | #endif 68 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_cond.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | #ifndef LIBRTE_TCH_HASH_RTE_HASH_COND_H_ 9 | #define LIBRTE_TCH_HASH_RTE_HASH_COND_H_ 10 | 11 | #ifdef H 12 | #undef H 13 | #undef BLOOM 14 | #undef HORTON 15 | #undef UNCONDITIONAL_PREFETCH 16 | #undef CONDITIONAL_PREFETCH 17 | #undef NO_PREFETCH 18 | #undef TIMER 19 | #endif 20 | 21 | 22 | #define H(x,y) x##_cond_##y 23 | 24 | /* Configuration Flags */ 25 | /* A. When to prefetch bucket */ 26 | #define BLOOM 0 27 | #define HORTON 0 28 | #define UNCONDITIONAL_PREFETCH 0 29 | #define CONDITIONAL_PREFETCH 1 30 | #define NO_PREFETCH 0 31 | /* B. Enable time management or not*/ 32 | #define TIMER 0 33 | 34 | 35 | 36 | #include "rte_hash_template.h" 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_horton.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #ifndef LIBRTE_TCH_HASH_RTE_HASH_HORTON_H_ 10 | #define LIBRTE_TCH_HASH_RTE_HASH_HORTON_H_ 11 | 12 | #ifdef H 13 | #undef H 14 | #undef BLOOM 15 | #undef HORTON 16 | #undef UNCONDITIONAL_PREFETCH 17 | #undef CONDITIONAL_PREFETCH 18 | #undef NO_PREFETCH 19 | #undef TIMER 20 | #endif 21 | 22 | #define H(x,y) x##_horton_##y 23 | 24 | /* Configuration Flags */ 25 | /* A. When to prefetch bucket */ 26 | #define BLOOM 0 27 | #define HORTON 1 28 | #define UNCONDITIONAL_PREFETCH 0 29 | #define CONDITIONAL_PREFETCH 0 30 | #define NO_PREFETCH 0 31 | /* B. Enable time management or not */ 32 | #define TIMER 0 33 | 34 | 35 | 36 | #include "rte_hash_template.h" 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_lazy_bloom.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | #ifndef LIBRTE_TCH_HASH_RTE_HASH_LAZY_BLOOM_H_ 9 | #define LIBRTE_TCH_HASH_RTE_HASH_LAZY_BLOOM_H_ 10 | 11 | #ifdef H 12 | #undef H 13 | #undef BLOOM 14 | #undef HORTON 15 | #undef UNCONDITIONAL_PREFETCH 16 | #undef CONDITIONAL_PREFETCH 17 | #undef NO_PREFETCH 18 | #undef TIMER 19 | #endif 20 | 21 | 22 | #define H(x,y) x##_lazy_bloom_##y 23 | 24 | /* Configuration Flags */ 25 | /* A. When to prefetch bucket */ 26 | #define BLOOM 1 27 | #define HORTON 0 28 | #define UNCONDITIONAL_PREFETCH 0 29 | #define CONDITIONAL_PREFETCH 0 30 | #define NO_PREFETCH 0 31 | /* B. Enable time management or not */ 32 | #define TIMER 1 33 | 34 | 35 | 36 | #include "rte_hash_template.h" 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_lazy_cond.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | #ifndef LIBRTE_TCH_HASH_RTE_HASH_LAZY_COND_H_ 9 | #define LIBRTE_TCH_HASH_RTE_HASH_LAZY_COND_H_ 10 | 11 | #ifdef H 12 | #undef H 13 | #undef BLOOM 14 | #undef HORTON 15 | #undef UNCONDITIONAL_PREFETCH 16 | #undef CONDITIONAL_PREFETCH 17 | #undef NO_PREFETCH 18 | #undef TIMER 19 | #endif 20 | 21 | #define H(x,y) x##_lazy_cond_##y 22 | 23 | /* Configuration Flags */ 24 | #define BLOOM 0 25 | #define HORTON 0 26 | #define UNCONDITIONAL_PREFETCH 0 27 | #define CONDITIONAL_PREFETCH 1 28 | #define NO_PREFETCH 0 29 | /* B. Enable time management or not */ 30 | #define TIMER 1 31 | 32 | 33 | 34 | #include "rte_hash_template.h" 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_lazy_no.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | #ifndef LIBRTE_TCH_HASH_RTE_HASH_LAZY_NO_H_ 9 | #define LIBRTE_TCH_HASH_RTE_HASH_LAZY_NO_H_ 10 | 11 | #ifdef H 12 | #undef H 13 | #undef BLOOM 14 | #undef HORTON 15 | #undef UNCONDITIONAL_PREFETCH 16 | #undef CONDITIONAL_PREFETCH 17 | #undef NO_PREFETCH 18 | #undef TIMER 19 | #endif 20 | 21 | 22 | #define H(x,y) x##_lazy_no_##y 23 | 24 | /* Configuration Flags */ 25 | /* A. When to prefetch bucket*/ 26 | #define BLOOM 0 27 | #define HORTON 0 28 | #define UNCONDITIONAL_PREFETCH 0 29 | #define CONDITIONAL_PREFETCH 0 30 | #define NO_PREFETCH 1 31 | /* B. Enable time management or not*/ 32 | #define TIMER 1 33 | 34 | 35 | 36 | #include "rte_hash_template.h" 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_lazy_uncond.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | #ifndef LIBRTE_TCH_HASH_RTE_HASH_LAZY_UNCOND_H_ 9 | #define LIBRTE_TCH_HASH_RTE_HASH_LAZY_UNCOND_H_ 10 | 11 | #ifdef H 12 | #undef H 13 | #undef BLOOM 14 | #undef HORTON 15 | #undef UNCONDITIONAL_PREFETCH 16 | #undef CONDITIONAL_PREFETCH 17 | #undef NO_PREFETCH 18 | #undef TIMER 19 | #endif 20 | 21 | 22 | #define H(x,y) x##_lazy_uncond_##y 23 | 24 | 25 | /* Configuration Flags */ 26 | /* A. When to prefetch bucket -- Should be set to BLOOM*/ 27 | #define BLOOM 0 28 | #define HORTON 0 29 | #define UNCONDITIONAL_PREFETCH 1 30 | #define CONDITIONAL_PREFETCH 0 31 | #define NO_PREFETCH 0 32 | /* B. Enable time management or not -- Should be enabled*/ 33 | #define TIMER 1 34 | 35 | 36 | #include "rte_hash_template.h" 37 | 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_template.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | 10 | #include 11 | 12 | #ifdef __cplusplus 13 | extern "C" { 14 | #endif 15 | /** 16 | * Create a new hash table. 17 | * 18 | * @param params 19 | * Parameters used to create and initialise the hash table. 20 | * @return 21 | * Pointer to hash table structure that is used in future hash table 22 | * operations, or NULL on error, with error code set in rte_errno. 23 | * Possible rte_errno errors include: 24 | * - ENOENT - missing entry 25 | * - EINVAL - invalid parameter passed to function 26 | * - ENOSPC - the maximum number of memzones has already been allocated 27 | * - EEXIST - a memzone with the same name already exists 28 | * - ENOMEM - no appropriate memory area found in which to create memzone 29 | */ 30 | struct rte_hash_hvariant * 31 | H(rte_hash,create)(const struct rte_hash_hvariant_parameters *params); 32 | 33 | 34 | /** 35 | * De-allocate all memory used by hash table. 36 | * @param h 37 | * Hash table to free 38 | */ 39 | void 40 | H(rte_hash,free)(struct rte_hash_hvariant *h); 41 | 42 | /** 43 | * Reset all hash structure, by zeroing all entries 44 | * @param h 45 | * Hash table to reset 46 | */ 47 | void 48 | H(rte_hash,reset)(struct rte_hash_hvariant *h); 49 | 50 | /** 51 | * Print stats to stdout 52 | * @param h 53 | * Hash table to reset 54 | * @param currentTime 55 | * current time to consider 56 | */ 57 | void H(rte_hash,print_stats)(struct rte_hash_hvariant *h, uint16_t currentTime); 58 | 59 | /** 60 | * Get the size of the hash table 61 | * @param h 62 | * Hash table to reset 63 | * @param currentTime 64 | * current time to consider 65 | */ 66 | uint32_t H(rte_hash,size)(struct rte_hash_hvariant *h, uint16_t currentTime); 67 | 68 | /** 69 | * Get the capacity of the hash table 70 | * @param h 71 | * Hash table to reset 72 | */ 73 | uint32_t H(rte_hash,capacity)(struct rte_hash_hvariant *h); 74 | 75 | /** 76 | * Add a key-value pair to an existing hash table. 77 | * This operation is not multi-thread safe 78 | * and should only be called from one thread. 79 | * 80 | * @param h 81 | * Hash table to add the key to. 82 | * @param key 83 | * Key to add to the hash table. 84 | * @param data 85 | * Data to add to the hash table. 86 | * @param expirationTime 87 | * Timeunit at which is the inserted entry should be expired 88 | * @param currentTime 89 | * Current time unit 90 | * @return 91 | * - RHL_FOUND_UPDATED if the key was added 92 | * - -EINVAL if the parameters are invalid. 93 | * - -ENOSPC if there is no space in the hash for this key. 94 | */ 95 | int 96 | H(rte_hash,add_key_data)(struct rte_hash_hvariant *h, const hash_key_t key, hash_data_t data, uint16_t expirationTime, uint16_t currentTime); 97 | 98 | /** 99 | * Add a key-value pair with a pre-computed hash value 100 | * to an existing hash table. 101 | * This operation is not multi-thread safe 102 | * and should only be called from one thread. 103 | * 104 | * @param h 105 | * Hash table to add the key to. 106 | * @param key 107 | * Key to add to the hash table. 108 | * @param sig 109 | * Precomputed hash value for 'key' 110 | * @param data 111 | * Data to add to the hash table. 112 | * @param expirationTime 113 | * Timeunit at which is the inserted entry should be expired 114 | * @param currentTime 115 | * Current time unit 116 | * @return 117 | * - RHL_FOUND_UPDATED if the key was added 118 | * - -EINVAL if the parameters are invalid. 119 | * - -ENOSPC if there is no space in the hash for this key. 120 | */ 121 | int32_t 122 | H(rte_hash,add_key_with_hash_data)(struct rte_hash_hvariant *h, const hash_key_t key, 123 | hash_sig64_t sig, hash_data_t data, uint16_t expirationTime, uint16_t currentTime); 124 | 125 | /** 126 | * Add a key to an existing hash table. This operation is not multi-thread safe 127 | * and should only be called from one thread. 128 | * 129 | * @param h 130 | * Hash table to add the key to. 131 | * @param key 132 | * Key to add to the hash table. 133 | * @param expirationTime 134 | * Timeunit at which is the inserted entry should be expired 135 | * @param currentTime 136 | * Current time unit 137 | * @return 138 | * - -EINVAL if the parameters are invalid. 139 | * - -ENOSPC if there is no space in the hash for this key. 140 | * - RHL_FOUND_UPDATED if the key was added 141 | */ 142 | int32_t 143 | H(rte_hash,add_key)(struct rte_hash_hvariant *h, const hash_key_t key , uint16_t expirationTime, uint16_t currentTime); 144 | 145 | /** 146 | * Add a key to an existing hash table. 147 | * This operation is not multi-thread safe 148 | * and should only be called from one thread. 149 | * 150 | * @param h 151 | * Hash table to add the key to. 152 | * @param key 153 | * Key to add to the hash table. 154 | * @param sig 155 | * Precomputed hash value for 'key'. 156 | * @param expirationTime 157 | * Timeunit at which is the inserted entry should be expired 158 | * @param currentTime 159 | * Current time unit 160 | * @return 161 | * - -EINVAL if the parameters are invalid. 162 | * - -ENOSPC if there is no space in the hash for this key. 163 | * - RHL_FOUND_UPDATED if the key was added 164 | */ 165 | int32_t 166 | H(rte_hash,add_key_with_hash)(struct rte_hash_hvariant *h, const hash_key_t key, hash_sig64_t sig, uint16_t expirationTime, uint16_t currentTime); 167 | 168 | /** 169 | * Remove a key from an existing hash table. 170 | * This operation is not multi-thread safe 171 | * and should only be called from one thread. 172 | * 173 | * @param h 174 | * Hash table to remove the key from. 175 | * @param key 176 | * Key to remove from the hash table. 177 | * @param currentTime 178 | * Current time unit 179 | * @return 180 | * - -EINVAL if the parameters are invalid. 181 | * - -ENOENT if the key is not found. 182 | * - RHL_FOUND_UPDATED if the key was deleted 183 | */ 184 | int32_t 185 | H(rte_hash,del_key)(struct rte_hash_hvariant *h, const hash_key_t key, uint16_t currentTime); 186 | 187 | /** 188 | * Remove a key from an existing hash table. 189 | * This operation is not multi-thread safe 190 | * and should only be called from one thread. 191 | * 192 | * @param h 193 | * Hash table to remove the key from. 194 | * @param key 195 | * Key to remove from the hash table. 196 | * @param sig 197 | * Precomputed hash value for 'key'. 198 | * @param currentTime 199 | * Current time unit 200 | * @return 201 | * - -EINVAL if the parameters are invalid. 202 | * - -ENOENT if the key is not found. 203 | * - RHL_FOUND_UPDATED if the key was deleted 204 | */ 205 | int32_t 206 | H(rte_hash,del_key_with_hash)(struct rte_hash_hvariant *h, const hash_key_t key, hash_sig64_t sig, uint16_t currentTime); 207 | 208 | 209 | /** 210 | * Find a key-value pair in the hash table. 211 | * This operation is multi-thread safe. 212 | * 213 | * @param h 214 | * Hash table to look in. 215 | * @param key 216 | * Key to find. 217 | * @param data 218 | * Output with pointer to data returned from the hash table. 219 | * @param currentTime 220 | * Current time unit 221 | * @return 222 | * - RHL_FOUND_NOT_UPDATED if the key was found 223 | * - EINVAL if the parameters are invalid. 224 | * - ENOENT if the key is not found. 225 | */ 226 | int 227 | H(rte_hash,lookup_data)(struct rte_hash_hvariant *h, const hash_key_t key, hash_data_t *data, uint16_t currentTime); 228 | 229 | /** 230 | * Find a key-value pair with a pre-computed hash value 231 | * to an existing hash table. 232 | * This operation is multi-thread safe. 233 | * 234 | * @param h 235 | * Hash table to look in. 236 | * @param key 237 | * Key to find. 238 | * @param sig 239 | * Precomputed hash value for 'key' 240 | * @param data 241 | * Output with pointer to data returned from the hash table. 242 | * @param currentTime 243 | * Current time unit 244 | * @return 245 | * - RHL_FOUND_NOT_UPDATED if the key was found 246 | * - EINVAL if the parameters are invalid. 247 | * - ENOENT if the key is not found. 248 | */ 249 | int 250 | H(rte_hash,lookup_with_hash_data)(struct rte_hash_hvariant *h, const hash_key_t key, 251 | hash_sig64_t sig, hash_data_t *data, uint16_t currentTime); 252 | 253 | /** 254 | * Find a key in the hash table. 255 | * This operation is multi-thread safe. 256 | * 257 | * @param h 258 | * Hash table to look in. 259 | * @param key 260 | * Key to find. 261 | * @param currentTime 262 | * Current time unit 263 | * @return 264 | * - -EINVAL if the parameters are invalid. 265 | * - -ENOENT if the key is not found. 266 | * - RHL_FOUND_NOT_UPDATED if the key was found 267 | */ 268 | int32_t 269 | H(rte_hash,lookup)(struct rte_hash_hvariant *h, const hash_key_t key, uint16_t currentTime); 270 | 271 | /** 272 | * Find a key in the hash table. 273 | * This operation is multi-thread safe. 274 | * 275 | * @param h 276 | * Hash table to look in. 277 | * @param key 278 | * Key to find. 279 | * @param sig 280 | * Hash value to remove from the hash table. 281 | * @param currentTime 282 | * Current time unit 283 | * @return 284 | * - -EINVAL if the parameters are invalid. 285 | * - -ENOENT if the key is not found. 286 | * - RHL_FOUND_NOT_UPDATED if the key was found 287 | */ 288 | int32_t 289 | H(rte_hash,lookup_with_hash)(struct rte_hash_hvariant *h, 290 | const hash_key_t key, hash_sig64_t sig, uint16_t currentTime); 291 | 292 | 293 | /** 294 | * Find a key-value pair in the hash table and update expiration time. 295 | * This operation is multi-thread safe. 296 | * 297 | * @param h 298 | * Hash table to look in. 299 | * @param key 300 | * Key to find. 301 | * @param currentTime 302 | * Current time unit 303 | * @param expieration time 304 | * New expiration time 305 | * @return 306 | * - RHL_FOUND_UPDATED if the key was found and expiration time has been updated 307 | * - RHL_FOUND_NOT_UPDATED if the key was found 308 | * - EINVAL if the parameters are invalid. 309 | * - ENOENT if the key is not found. 310 | */ 311 | int32_t H(rte_hash,lookup_update)(struct rte_hash_hvariant *h, const hash_key_t key, uint16_t expirationTime, uint16_t currentTime); 312 | 313 | 314 | 315 | /** 316 | * Find a key-value pair in the hash table and update expiration time. 317 | * This operation is multi-thread safe. 318 | * 319 | * @param h 320 | * Hash table to look in. 321 | * @param key 322 | * Key to find. 323 | * @param currentTime 324 | * Current time unit 325 | * @param sig 326 | * Hash value to remove from the hash table. 327 | * @param expiration time 328 | * New expiration time 329 | * @return 330 | * - RHL_FOUND_UPDATED if the key was found and expiration time has been updated 331 | * - RHL_FOUND_NOT_UPDATED if the key was found 332 | * - EINVAL if the parameters are invalid. 333 | * - ENOENT if the key is not found. 334 | */ 335 | int32_t H(rte_hash,lookup_update_with_hash)(struct rte_hash_hvariant *h, const hash_key_t key, hash_sig64_t sig, uint16_t expirationTime, uint16_t currentTime); 336 | 337 | /** 338 | * Find a key-value pair in the hash table and update expiration time. 339 | * This operation is multi-thread safe. 340 | * 341 | * @param h 342 | * Hash table to look in. 343 | * @param key 344 | * Key to find. 345 | * @param data 346 | * Output with pointer to data returned from the hash table. 347 | * @param currentTime 348 | * Current time unit 349 | * @param sig 350 | * Hash value to remove from the hash table. 351 | * @param expiration time 352 | * New expiration time 353 | * @return 354 | * - RHL_FOUND_UPDATED if the key was found and expiration time has been updated 355 | * - RHL_FOUND_NOT_UPDATED if the key was found 356 | * - EINVAL if the parameters are invalid. 357 | * - ENOENT if the key is not found. 358 | */ 359 | int H(rte_hash,lookup_update_with_hash_data)(struct rte_hash_hvariant *h,const hash_key_t key, hash_sig64_t sig, hash_data_t *data, uint16_t expirationTime, uint16_t currentTime); 360 | 361 | /** 362 | * Find a key-value pair in the hash table and update expiration time. 363 | * This operation is multi-thread safe. 364 | * 365 | * @param h 366 | * Hash table to look in. 367 | * @param key 368 | * Key to find. 369 | * @param data 370 | * Output with pointer to data returned from the hash table. 371 | * @param currentTime 372 | * Current time unit 373 | * @param expieration time 374 | * New expiration time 375 | * @return 376 | * - RHL_FOUND_UPDATED if the key was found and expiration time has been updated 377 | * - RHL_FOUND_NOT_UPDATED if the key was found 378 | * - EINVAL if the parameters are invalid. 379 | * - ENOENT if the key is not found. 380 | */ 381 | int H(rte_hash,lookup_update_data)(struct rte_hash_hvariant *h, const hash_key_t key, hash_data_t *data, uint16_t expirationTime, uint16_t currentTime); 382 | 383 | 384 | /** 385 | * Find multiple keys in the hash table. 386 | * This operation is multi-thread safe. 387 | * 388 | * @param h 389 | * Hash table to look in. 390 | * @param keys 391 | * A pointer to a list of keys to look for. 392 | * @param num_keys 393 | * How many keys are in the keys list (less than RTE_HASH_LOOKUP_BULK_MAX). 394 | * @param hit_mask 395 | * Output containing a bitmask with all successful lookups. 396 | * @param data 397 | * Output containing array of data returned from all the successful lookups. 398 | * @param currentTime 399 | * Current time unit 400 | * @return 401 | * -EINVAL if there's an error, otherwise number of successful lookups. 402 | */ 403 | int H(rte_hash,lookup_bulk_data)(struct rte_hash_hvariant *h, const hash_key_t *keys, uint32_t num_keys, uint64_t *hit_mask, hash_data_t data[], uint16_t currentTime); 404 | 405 | /** 406 | * Find multiple keys in the hash table. 407 | * This operation is multi-thread safe. 408 | * 409 | * @param h 410 | * Hash table to look in. 411 | * @param keys 412 | * A pointer to a list of keys to look for. 413 | * @param num_keys 414 | * How many keys are in the keys list (less than RTE_HASH_LOOKUP_BULK_MAX). 415 | * @param positions 416 | * Output containing a list of values, corresponding to the list of keys that 417 | * can be used by the caller as an offset into an array of user data. These 418 | * values are unique for each key, and are the same values that were returned 419 | * when each key was added. If a key in the list was not found, then -ENOENT 420 | * will be the value. 421 | * @param currentTime 422 | * Current time unit 423 | * @return 424 | * -EINVAL if there's an error, otherwise 0. 425 | */ 426 | int H(rte_hash,lookup_bulk)(struct rte_hash_hvariant *h, const hash_key_t *keys, uint32_t num_keys, int32_t *positions, uint16_t currentTime); 427 | 428 | /** 429 | * Find multiple keys in the hash table. 430 | * This operation is multi-thread safe. 431 | * 432 | * @param h 433 | * Hash table to look in. 434 | * @param keys 435 | * A pointer to a list of keys to look for (up to 64 keys, according to lookup_mask). 436 | * @param lookup_mask 437 | * bitmask of keys to lookup 438 | * @param hit_mask 439 | * Output containing a bitmask with all successful lookups. 440 | * @param data 441 | * Output containing array of data returned from all the successful lookups. 442 | * @param currentTime 443 | * Current time unit 444 | * @return 445 | * -EINVAL if there's an error, otherwise number of successful lookups. 446 | */ 447 | int H(rte_hash,lookup_bulk_data_mask)(struct rte_hash_hvariant *h, const hash_key_t *keys, uint64_t lookup_mask, uint64_t *hit_mask, hash_data_t data[], uint16_t currentTime); 448 | 449 | /** 450 | * Find multiple keys in the hash table. 451 | * This operation is multi-thread safe. 452 | * 453 | * @param h 454 | * Hash table to look in. 455 | * @param keys 456 | * A pointer to a list of keys to look for (up to 64 keys, according to lookup_mask). 457 | * @param lookup_mask 458 | * bitmask of keys to lookup 459 | * @param hit_mask 460 | * Output containing a bitmask with all successful lookups. 461 | * @param updated_mask 462 | * Output containing a bitmask with all updates expiration times. 463 | * @param data 464 | * Output containing array of data returned from all the successful lookups. 465 | * @param currentTime 466 | * Current time unit 467 | * @param expirationTime 468 | * A pointer to a list of expiration times for each keys (up to 64 expiration times, according to lookup_mask). 469 | * @return 470 | * -EINVAL if there's an error, otherwise number of successful lookups. 471 | */ 472 | int H(rte_hash,lookup_update_bulk_data_mask)(struct rte_hash_hvariant *h, const hash_key_t *keys, uint64_t lookup_mask, uint64_t *hit_mask, uint64_t *updated_mask, hash_data_t data[], uint16_t * newExpirationTime, uint16_t currentTime); 473 | 474 | 475 | /** 476 | * Reset iterator (only a single iterator can be active at a time on the hashtable) 477 | * 478 | * @param h 479 | * Hash table to iterate 480 | */ 481 | void H(rte_hash,iterator_reset)(struct rte_hash_hvariant *h); 482 | 483 | /** 484 | * Iterate through the hash table, returning key-value pairs. 485 | * This version supports modifications to the hash table in between calls to iterate. 486 | * Reset iterator must be called before starting iteration on the hash table. 487 | * 488 | * @param h 489 | * Hash table to iterate 490 | * @param key 491 | * Output containing the key where current iterator was pointing at 492 | * @param data 493 | * Output containing the data associated with key. 494 | * Returns NULL if data was not stored. 495 | * @param currentTime 496 | * Current time unit 497 | * @param remainingTime 498 | * Output containing remaining time before expiration (relative to currentTime) 499 | * @return 500 | * - 0 if a key/data is returned 501 | * - -EINVAL if the parameters are invalid. 502 | * - -ENOENT if end of the hash table. 503 | * - -EBUSY if the iteration was suspended (and should be continued by a new call). This allows to limit the duration of unpreemptable calls to this function. 504 | */ 505 | int32_t 506 | H(rte_hash,iterate)(struct rte_hash_hvariant *h, hash_key_t *key, hash_data_t *data, uint16_t * remaining_time, uint16_t currentTime); 507 | 508 | /** 509 | * Iterate through the hash table, returning key-value pairs. Unsafe version, multiple iterators allowed, but entries may be missed if modification are concurrent. 510 | * 511 | * @param h 512 | * Hash table to iterate 513 | * @param pos 514 | * Placeholder for iterator data (must be set to 0 for starting an iteration) 515 | * @param key 516 | * Output containing the key where current iterator 517 | * was pointing at 518 | * @param data 519 | * Output containing the data associated with key. 520 | * Returns NULL if data was not stored. 521 | * @param currentTime 522 | * Current time unit 523 | * @param remainingTime 524 | * Output containing remaining time before expiration (relative to currentTime) 525 | * @return 526 | * - 0 if a key/data is returned 527 | * - -EINVAL if the parameters are invalid. 528 | * - -ENOENT if end of the hash table. 529 | */ 530 | int32_t 531 | H(rte_hash,unsafe_iterate)(struct rte_hash_hvariant *h, uint64_t * pos, hash_key_t *key, hash_data_t *data, uint16_t * remaining_time, uint16_t currentTime); 532 | 533 | /** 534 | * Check the integrity of the structure. This function is meant to be used during development or testing. 535 | * 536 | * @param h 537 | * Hash table to iterate 538 | * @param currentTime 539 | * Current time unit 540 | * @return 541 | * Nothing. Errors are printed to standard output. 542 | */ 543 | void H(rte_hash,check_integrity)(struct rte_hash_hvariant *h, uint16_t currentTime); 544 | 545 | /** 546 | * Check the integrity of the structure 547 | * 548 | * @param h 549 | * Hash table to iterate 550 | * @param currentTime 551 | * Current time unit 552 | * @return 553 | * Stasticis on fraction of entries stored in secondary bucket. 554 | */ 555 | double H(rte_hash,stats_secondary)(struct rte_hash_hvariant *h, uint16_t currentTime); 556 | 557 | /** 558 | * Returns the number of slots per bucket 559 | * 560 | * @return 561 | * the number of slots per bucket 562 | */ 563 | int H(rte_hash,slots_per_bucket)(void); 564 | 565 | 566 | #ifdef __cplusplus 567 | } 568 | #endif 569 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_uncond.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #ifndef LIBRTE_TCH_HASH_RTE_HASH_UNCOND_H_ 10 | #define LIBRTE_TCH_HASH_RTE_HASH_UNCOND_H_ 11 | 12 | #ifdef H 13 | #undef H 14 | #undef BLOOM 15 | #undef HORTON 16 | #undef UNCONDITIONAL_PREFETCH 17 | #undef CONDITIONAL_PREFETCH 18 | #undef NO_PREFETCH 19 | #undef TIMER 20 | #endif 21 | 22 | #define H(x,y) x##_uncond_##y 23 | 24 | 25 | /* Configuration Flags */ 26 | /* A. When to prefetch bucket */ 27 | #define BLOOM 0 28 | #define HORTON 0 29 | #define UNCONDITIONAL_PREFETCH 1 30 | #define CONDITIONAL_PREFETCH 0 31 | #define NO_PREFETCH 0 32 | /* B. Enable time management or not */ 33 | #define TIMER 0 34 | 35 | 36 | #include "rte_hash_template.h" 37 | 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_v1604.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * Minor modifications from vanilla DPDK 16.04 are licensed under Clear BSD. 3 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 4 | * 5 | * Original code is licensed under: 6 | * 7 | * BSD LICENSE 8 | * 9 | * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. 10 | * All rights reserved. 11 | * 12 | * Redistribution and use in source and binary forms, with or without 13 | * modification, are permitted provided that the following conditions 14 | * are met: 15 | * 16 | * * Redistributions of source code must retain the above copyright 17 | * notice, this list of conditions and the following disclaimer. 18 | * * Redistributions in binary form must reproduce the above copyright 19 | * notice, this list of conditions and the following disclaimer in 20 | * the documentation and/or other materials provided with the 21 | * distribution. 22 | * * Neither the name of Intel Corporation nor the names of its 23 | * contributors may be used to endorse or promote products derived 24 | * from this software without specific prior written permission. 25 | * 26 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | */ 38 | 39 | #ifndef _RTE_HASH_V1604_H_ 40 | #define _RTE_HASH_V1604_H_ 41 | 42 | 43 | #include "rte_tchh_structs.h" 44 | 45 | /** 46 | ash_ * @file 47 | * 48 | * RTE Hash Table 49 | */ 50 | 51 | #include 52 | #include 53 | 54 | #ifdef __cplusplus 55 | extern "C" { 56 | #endif 57 | 58 | /** Maximum size of hash table that can be created. */ 59 | #define RTE_HASH_V1604_ENTRIES_MAX (1 << 30) 60 | 61 | /** Maximum number of characters in hash name.*/ 62 | #define RTE_HASH_V1604_NAMESIZE 32 63 | 64 | /** Maximum number of keys that can be searched for using rte_hash_v1604_lookup_bulk. */ 65 | #define RTE_HASH_V1604_LOOKUP_BULK_MAX 64 66 | #define RTE_HASH_V1604_LOOKUP_MULTI_MAX RTE_HASH_V1604_LOOKUP_BULK_MAX 67 | 68 | /** Enable Hardware transactional memory support. */ 69 | #define RTE_HASH_V1604_EXTRA_FLAGS_TRANS_MEM_SUPPORT 0x01 70 | 71 | /** Signature of key that is stored internally. */ 72 | typedef uint32_t hash_sig_t; 73 | 74 | /** Type of function that can be used for calculating the hash value. */ 75 | typedef uint32_t (*rte_hash_v1604_function)(const void *key, uint32_t key_len, 76 | uint32_t init_val); 77 | 78 | /** Type of function used to compare the hash key. */ 79 | typedef int (*rte_hash_v1604_cmp_eq_t)(const void *key1, const void *key2, size_t key_len); 80 | 81 | /** 82 | * Parameters used when creating the hash table. 83 | */ 84 | struct rte_hash_v1604_parameters { 85 | const char *name; /**< Name of the hash. */ 86 | uint32_t entries; /**< Total hash table entries. */ 87 | uint32_t reserved; /**< Unused field. Should be set to 0 */ 88 | uint32_t key_len; /**< Length of hash key. */ 89 | rte_hash_v1604_function hash_func; /**< Primary Hash function used to calculate hash. */ 90 | uint32_t hash_func_init_val; /**< Init value used by hash_func. */ 91 | int socket_id; /**< NUMA Socket ID for memory. */ 92 | uint8_t extra_flag; /**< Indicate if additional parameters are present. */ 93 | }; 94 | 95 | /** @internal A hash table structure. */ 96 | struct rte_hash_v1604; 97 | 98 | /** 99 | * Create a new hash table. 100 | * 101 | * @param params 102 | * Parameters used to create and initialise the hash table. 103 | * @return 104 | * Pointer to hash table structure that is used in future hash table 105 | * operations, or NULL on error, with error code set in rte_errno. 106 | * Possible rte_errno errors include: 107 | * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure 108 | * - E_RTE_SECONDARY - function was called from a secondary process instance 109 | * - ENOENT - missing entry 110 | * - EINVAL - invalid parameter passed to function 111 | * - ENOSPC - the maximum number of memzones has already been allocated 112 | * - EEXIST - a memzone with the same name already exists 113 | * - ENOMEM - no appropriate memory area found in which to create memzone 114 | */ 115 | struct rte_hash_v1604 * 116 | rte_hash_v1604_create(const struct rte_hash_v1604_parameters *params); 117 | 118 | /** 119 | * Set a new hash compare function other than the default one. 120 | * 121 | * @note Function pointer does not work with multi-process, so do not use it 122 | * in multi-process mode. 123 | * 124 | * @param h 125 | * Hash table for which the function is to be changed 126 | * @param func 127 | * New compare function 128 | */ 129 | void rte_hash_v1604_set_cmp_func(struct rte_hash_v1604 *h, rte_hash_v1604_cmp_eq_t func); 130 | 131 | /** 132 | * Find an existing hash table object and return a pointer to it. 133 | * 134 | * @param name 135 | * Name of the hash table as passed to rte_hash_v1604_create() 136 | * @return 137 | * Pointer to hash table or NULL if object not found 138 | * with rte_errno set appropriately. Possible rte_errno values include: 139 | * - ENOENT - value not available for return 140 | */ 141 | struct rte_hash_v1604 * 142 | rte_hash_v1604_find_existing(const char *name); 143 | 144 | /** 145 | * De-allocate all memory used by hash table. 146 | * @param h 147 | * Hash table to free 148 | */ 149 | void 150 | rte_hash_v1604_free(struct rte_hash_v1604 *h); 151 | 152 | /** 153 | * Reset all hash structure, by zeroing all entries 154 | * @param h 155 | * Hash table to reset 156 | */ 157 | void 158 | rte_hash_v1604_reset(struct rte_hash_v1604 *h); 159 | 160 | /** 161 | * Add a key-value pair to an existing hash table. 162 | * This operation is not multi-thread safe 163 | * and should only be called from one thread. 164 | * 165 | * @param h 166 | * Hash table to add the key to. 167 | * @param key 168 | * Key to add to the hash table. 169 | * @param data 170 | * Data to add to the hash table. 171 | * @return 172 | * - 0 if added successfully 173 | * - -EINVAL if the parameters are invalid. 174 | * - -ENOSPC if there is no space in the hash for this key. 175 | */ 176 | int 177 | rte_hash_v1604_add_key_data(const struct rte_hash_v1604 *h, const void *key, hash_data_t data); 178 | 179 | /** 180 | * Add a key-value pair with a pre-computed hash value 181 | * to an existing hash table. 182 | * This operation is not multi-thread safe 183 | * and should only be called from one thread. 184 | * 185 | * @param h 186 | * Hash table to add the key to. 187 | * @param key 188 | * Key to add to the hash table. 189 | * @param sig 190 | * Precomputed hash value for 'key' 191 | * @param data 192 | * Data to add to the hash table. 193 | * @return 194 | * - 0 if added successfully 195 | * - -EINVAL if the parameters are invalid. 196 | * - -ENOSPC if there is no space in the hash for this key. 197 | */ 198 | int32_t 199 | rte_hash_v1604_add_key_with_hash_data(const struct rte_hash_v1604 *h, const void *key, 200 | hash_sig_t sig, hash_data_t data); 201 | 202 | /** 203 | * Add a key to an existing hash table. This operation is not multi-thread safe 204 | * and should only be called from one thread. 205 | * 206 | * @param h 207 | * Hash table to add the key to. 208 | * @param key 209 | * Key to add to the hash table. 210 | * @return 211 | * - -EINVAL if the parameters are invalid. 212 | * - -ENOSPC if there is no space in the hash for this key. 213 | * - A positive value that can be used by the caller as an offset into an 214 | * array of user data. This value is unique for this key. 215 | */ 216 | int32_t 217 | rte_hash_v1604_add_key(const struct rte_hash_v1604 *h, const void *key); 218 | 219 | /** 220 | * Add a key to an existing hash table. 221 | * This operation is not multi-thread safe 222 | * and should only be called from one thread. 223 | * 224 | * @param h 225 | * Hash table to add the key to. 226 | * @param key 227 | * Key to add to the hash table. 228 | * @param sig 229 | * Precomputed hash value for 'key'. 230 | * @return 231 | * - -EINVAL if the parameters are invalid. 232 | * - -ENOSPC if there is no space in the hash for this key. 233 | * - A positive value that can be used by the caller as an offset into an 234 | * array of user data. This value is unique for this key. 235 | */ 236 | int32_t 237 | rte_hash_v1604_add_key_with_hash(const struct rte_hash_v1604 *h, const void *key, hash_sig_t sig); 238 | 239 | /** 240 | * Remove a key from an existing hash table. 241 | * This operation is not multi-thread safe 242 | * and should only be called from one thread. 243 | * 244 | * @param h 245 | * Hash table to remove the key from. 246 | * @param key 247 | * Key to remove from the hash table. 248 | * @return 249 | * - -EINVAL if the parameters are invalid. 250 | * - -ENOENT if the key is not found. 251 | * - A positive value that can be used by the caller as an offset into an 252 | * array of user data. This value is unique for this key, and is the same 253 | * value that was returned when the key was added. 254 | */ 255 | int32_t 256 | rte_hash_v1604_del_key(const struct rte_hash_v1604 *h, const void *key); 257 | 258 | /** 259 | * Remove a key from an existing hash table. 260 | * This operation is not multi-thread safe 261 | * and should only be called from one thread. 262 | * 263 | * @param h 264 | * Hash table to remove the key from. 265 | * @param key 266 | * Key to remove from the hash table. 267 | * @param sig 268 | * Precomputed hash value for 'key'. 269 | * @return 270 | * - -EINVAL if the parameters are invalid. 271 | * - -ENOENT if the key is not found. 272 | * - A positive value that can be used by the caller as an offset into an 273 | * array of user data. This value is unique for this key, and is the same 274 | * value that was returned when the key was added. 275 | */ 276 | int32_t 277 | rte_hash_v1604_del_key_with_hash(const struct rte_hash_v1604 *h, const void *key, hash_sig_t sig); 278 | 279 | 280 | /** 281 | * Find a key-value pair in the hash table. 282 | * This operation is multi-thread safe. 283 | * 284 | * @param h 285 | * Hash table to look in. 286 | * @param key 287 | * Key to find. 288 | * @param data 289 | * Output with pointer to data returned from the hash table. 290 | * @return 291 | * 0 if successful lookup 292 | * - EINVAL if the parameters are invalid. 293 | * - ENOENT if the key is not found. 294 | */ 295 | int 296 | rte_hash_v1604_lookup_data(const struct rte_hash_v1604 *h, const void *key, hash_data_t *data); 297 | 298 | /** 299 | * Find a key-value pair with a pre-computed hash value 300 | * to an existing hash table. 301 | * This operation is multi-thread safe. 302 | * 303 | * @param h 304 | * Hash table to look in. 305 | * @param key 306 | * Key to find. 307 | * @param sig 308 | * Precomputed hash value for 'key' 309 | * @param data 310 | * Output with pointer to data returned from the hash table. 311 | * @return 312 | * 0 if successful lookup 313 | * - EINVAL if the parameters are invalid. 314 | * - ENOENT if the key is not found. 315 | */ 316 | int 317 | rte_hash_v1604_lookup_with_hash_data(const struct rte_hash_v1604 *h, const void *key, 318 | hash_sig_t sig, hash_data_t *data); 319 | 320 | /** 321 | * Find a key in the hash table. 322 | * This operation is multi-thread safe. 323 | * 324 | * @param h 325 | * Hash table to look in. 326 | * @param key 327 | * Key to find. 328 | * @return 329 | * - -EINVAL if the parameters are invalid. 330 | * - -ENOENT if the key is not found. 331 | * - A positive value that can be used by the caller as an offset into an 332 | * array of user data. This value is unique for this key, and is the same 333 | * value that was returned when the key was added. 334 | */ 335 | int32_t 336 | rte_hash_v1604_lookup(const struct rte_hash_v1604 *h, const void *key); 337 | 338 | /** 339 | * Find a key in the hash table. 340 | * This operation is multi-thread safe. 341 | * 342 | * @param h 343 | * Hash table to look in. 344 | * @param key 345 | * Key to find. 346 | * @param sig 347 | * Hash value to remove from the hash table. 348 | * @return 349 | * - -EINVAL if the parameters are invalid. 350 | * - -ENOENT if the key is not found. 351 | * - A positive value that can be used by the caller as an offset into an 352 | * array of user data. This value is unique for this key, and is the same 353 | * value that was returned when the key was added. 354 | */ 355 | int32_t 356 | rte_hash_v1604_lookup_with_hash(const struct rte_hash_v1604 *h, 357 | const void *key, hash_sig_t sig); 358 | 359 | /** 360 | * Calc a hash value by key. 361 | * This operation is not multi-thread safe. 362 | * 363 | * @param h 364 | * Hash table to look in. 365 | * @param key 366 | * Key to find. 367 | * @return 368 | * - hash value 369 | */ 370 | hash_sig_t 371 | rte_hash_v1604_hash(const struct rte_hash_v1604 *h, const void *key); 372 | 373 | #define rte_hash_v1604_lookup_multi rte_hash_v1604_lookup_bulk 374 | #define rte_hash_v1604_lookup_multi_data rte_hash_v1604_lookup_bulk_data 375 | /** 376 | * Find multiple keys in the hash table. 377 | * This operation is multi-thread safe. 378 | * 379 | * @param h 380 | * Hash table to look in. 381 | * @param keys 382 | * A pointer to a list of keys to look for. 383 | * @param num_keys 384 | * How many keys are in the keys list (less than RTE_HASH_V1604_LOOKUP_BULK_MAX). 385 | * @param hit_mask 386 | * Output containing a bitmask with all successful lookups. 387 | * @param data 388 | * Output containing array of data returned from all the successful lookups. 389 | * @return 390 | * -EINVAL if there's an error, otherwise number of successful lookups. 391 | */ 392 | int 393 | rte_hash_v1604_lookup_bulk_data(const struct rte_hash_v1604 *h, const void **keys, 394 | uint32_t num_keys, uint64_t *hit_mask, hash_data_t data[]); 395 | 396 | /** 397 | * Find multiple keys in the hash table. 398 | * This operation is multi-thread safe. 399 | * 400 | * @param h 401 | * Hash table to look in. 402 | * @param keys 403 | * A pointer to a list of keys to look for. 404 | * @param num_keys 405 | * How many keys are in the keys list (less than RTE_HASH_V1604_LOOKUP_BULK_MAX). 406 | * @param positions 407 | * Output containing a list of values, corresponding to the list of keys that 408 | * can be used by the caller as an offset into an array of user data. These 409 | * values are unique for each key, and are the same values that were returned 410 | * when each key was added. If a key in the list was not found, then -ENOENT 411 | * will be the value. 412 | * @return 413 | * -EINVAL if there's an error, otherwise 0. 414 | */ 415 | int 416 | rte_hash_v1604_lookup_bulk(const struct rte_hash_v1604 *h, const void **keys, 417 | uint32_t num_keys, int32_t *positions); 418 | 419 | /** 420 | * Iterate through the hash table, returning key-value pairs. 421 | * 422 | * @param h 423 | * Hash table to iterate 424 | * @param key 425 | * Output containing the key where current iterator 426 | * was pointing at 427 | * @param data 428 | * Output containing the data associated with key. 429 | * Returns NULL if data was not stored. 430 | * @param next 431 | * Pointer to iterator. Should be 0 to start iterating the hash table. 432 | * Iterator is incremented after each call of this function. 433 | * @return 434 | * Position where key was stored, if successful. 435 | * - -EINVAL if the parameters are invalid. 436 | * - -ENOENT if end of the hash table. 437 | */ 438 | int32_t 439 | rte_hash_v1604_iterate(const struct rte_hash_v1604 *h, const void **key, hash_data_t *data, uint32_t *next); 440 | 441 | double rte_hash_v1604_stats_secondary(const struct rte_hash_v1604 *h); 442 | #ifdef __cplusplus 443 | } 444 | #endif 445 | 446 | #endif /* _RTE_HASH_V1604_H_ */ 447 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_hash_v1702.h: -------------------------------------------------------------------------------- 1 | /*- 2 | * Minor modifications from vanilla DPDK 17.02 are licensed under Clear BSD. 3 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 4 | * 5 | * Original code is licensed under: 6 | * 7 | * BSD LICENSE 8 | * 9 | * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. 10 | * All rights reserved. 11 | * 12 | * Redistribution and use in source and binary forms, with or without 13 | * modification, are permitted provided that the following conditions 14 | * are met: 15 | * 16 | * * Redistributions of source code must retain the above copyright 17 | * notice, this list of conditions and the following disclaimer. 18 | * * Redistributions in binary form must reproduce the above copyright 19 | * notice, this list of conditions and the following disclaimer in 20 | * the documentation and/or other materials provided with the 21 | * distribution. 22 | * * Neither the name of Intel Corporation nor the names of its 23 | * contributors may be used to endorse or promote products derived 24 | * from this software without specific prior written permission. 25 | * 26 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 29 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 30 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 31 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 32 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 36 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | */ 38 | 39 | #ifndef _RTE_HASH_V1702_H_ 40 | #define _RTE_HASH_V1702_H_ 41 | 42 | #include "rte_tchh_structs.h" 43 | 44 | /** 45 | * @file 46 | * 47 | * RTE Hash Table 48 | */ 49 | 50 | #include 51 | #include 52 | 53 | #ifdef __cplusplus 54 | extern "C" { 55 | #endif 56 | 57 | /** Maximum size of hash table that can be created. */ 58 | #define RTE_HASH_V1702_ENTRIES_MAX (1 << 30) 59 | 60 | /** Maximum number of characters in hash name.*/ 61 | #define RTE_HASH_V1702_NAMESIZE 32 62 | 63 | /** Maximum number of keys that can be searched for using rte_hash_v1702_lookup_bulk. */ 64 | #define RTE_HASH_V1702_LOOKUP_BULK_MAX 64 65 | #define RTE_HASH_V1702_LOOKUP_MULTI_MAX RTE_HASH_V1702_LOOKUP_BULK_MAX 66 | 67 | /** Enable Hardware transactional memory support. */ 68 | #define RTE_HASH_V1702_EXTRA_FLAGS_TRANS_MEM_SUPPORT 0x01 69 | 70 | /** Default behavior of insertion, single writer/multi writer */ 71 | #define RTE_HASH_V1702_EXTRA_FLAGS_MULTI_WRITER_ADD 0x02 72 | 73 | /** Signature of key that is stored internally. */ 74 | typedef uint32_t hash_sig_t; 75 | 76 | /** Type of function that can be used for calculating the hash value. */ 77 | typedef uint32_t (*rte_hash_v1702_function)(const void *key, uint32_t key_len, 78 | uint32_t init_val); 79 | 80 | /** Type of function used to compare the hash key. */ 81 | typedef int (*rte_hash_v1702_cmp_eq_t)(const void *key1, const void *key2, size_t key_len); 82 | 83 | /** 84 | * Parameters used when creating the hash table. 85 | */ 86 | struct rte_hash_v1702_parameters { 87 | const char *name; /**< Name of the hash. */ 88 | uint32_t entries; /**< Total hash table entries. */ 89 | uint32_t reserved; /**< Unused field. Should be set to 0 */ 90 | uint32_t key_len; /**< Length of hash key. */ 91 | rte_hash_v1702_function hash_func; /**< Primary Hash function used to calculate hash. */ 92 | uint32_t hash_func_init_val; /**< Init value used by hash_func. */ 93 | int socket_id; /**< NUMA Socket ID for memory. */ 94 | uint8_t extra_flag; /**< Indicate if additional parameters are present. */ 95 | }; 96 | 97 | /** @internal A hash table structure. */ 98 | struct rte_hash_v1702; 99 | 100 | /** 101 | * Create a new hash table. 102 | * 103 | * @param params 104 | * Parameters used to create and initialise the hash table. 105 | * @return 106 | * Pointer to hash table structure that is used in future hash table 107 | * operations, or NULL on error, with error code set in rte_errno. 108 | * Possible rte_errno errors include: 109 | * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure 110 | * - E_RTE_SECONDARY - function was called from a secondary process instance 111 | * - ENOENT - missing entry 112 | * - EINVAL - invalid parameter passed to function 113 | * - ENOSPC - the maximum number of memzones has already been allocated 114 | * - EEXIST - a memzone with the same name already exists 115 | * - ENOMEM - no appropriate memory area found in which to create memzone 116 | */ 117 | struct rte_hash_v1702 * 118 | rte_hash_v1702_create(const struct rte_hash_v1702_parameters *params); 119 | 120 | /** 121 | * Set a new hash compare function other than the default one. 122 | * 123 | * @note Function pointer does not work with multi-process, so do not use it 124 | * in multi-process mode. 125 | * 126 | * @param h 127 | * Hash table for which the function is to be changed 128 | * @param func 129 | * New compare function 130 | */ 131 | void rte_hash_v1702_set_cmp_func(struct rte_hash_v1702 *h, rte_hash_v1702_cmp_eq_t func); 132 | 133 | /** 134 | * Find an existing hash table object and return a pointer to it. 135 | * 136 | * @param name 137 | * Name of the hash table as passed to rte_hash_v1702_create() 138 | * @return 139 | * Pointer to hash table or NULL if object not found 140 | * with rte_errno set appropriately. Possible rte_errno values include: 141 | * - ENOENT - value not available for return 142 | */ 143 | struct rte_hash_v1702 * 144 | rte_hash_v1702_find_existing(const char *name); 145 | 146 | /** 147 | * De-allocate all memory used by hash table. 148 | * @param h 149 | * Hash table to free 150 | */ 151 | void 152 | rte_hash_v1702_free(struct rte_hash_v1702 *h); 153 | 154 | /** 155 | * Reset all hash structure, by zeroing all entries 156 | * @param h 157 | * Hash table to reset 158 | */ 159 | void 160 | rte_hash_v1702_reset(struct rte_hash_v1702 *h); 161 | 162 | /** 163 | * Add a key-value pair to an existing hash table. 164 | * This operation is not multi-thread safe 165 | * and should only be called from one thread. 166 | * 167 | * @param h 168 | * Hash table to add the key to. 169 | * @param key 170 | * Key to add to the hash table. 171 | * @param data 172 | * Data to add to the hash table. 173 | * @return 174 | * - 0 if added successfully 175 | * - -EINVAL if the parameters are invalid. 176 | * - -ENOSPC if there is no space in the hash for this key. 177 | */ 178 | int 179 | rte_hash_v1702_add_key_data(const struct rte_hash_v1702 *h, const void *key, hash_data_t data); 180 | 181 | /** 182 | * Add a key-value pair with a pre-computed hash value 183 | * to an existing hash table. 184 | * This operation is not multi-thread safe 185 | * and should only be called from one thread. 186 | * 187 | * @param h 188 | * Hash table to add the key to. 189 | * @param key 190 | * Key to add to the hash table. 191 | * @param sig 192 | * Precomputed hash value for 'key' 193 | * @param data 194 | * Data to add to the hash table. 195 | * @return 196 | * - 0 if added successfully 197 | * - -EINVAL if the parameters are invalid. 198 | * - -ENOSPC if there is no space in the hash for this key. 199 | */ 200 | int32_t 201 | rte_hash_v1702_add_key_with_hash_data(const struct rte_hash_v1702 *h, const void *key, 202 | hash_sig_t sig, hash_data_t data); 203 | 204 | /** 205 | * Add a key to an existing hash table. This operation is not multi-thread safe 206 | * and should only be called from one thread. 207 | * 208 | * @param h 209 | * Hash table to add the key to. 210 | * @param key 211 | * Key to add to the hash table. 212 | * @return 213 | * - -EINVAL if the parameters are invalid. 214 | * - -ENOSPC if there is no space in the hash for this key. 215 | * - A positive value that can be used by the caller as an offset into an 216 | * array of user data. This value is unique for this key. 217 | */ 218 | int32_t 219 | rte_hash_v1702_add_key(const struct rte_hash_v1702 *h, const void *key); 220 | 221 | /** 222 | * Add a key to an existing hash table. 223 | * This operation is not multi-thread safe 224 | * and should only be called from one thread. 225 | * 226 | * @param h 227 | * Hash table to add the key to. 228 | * @param key 229 | * Key to add to the hash table. 230 | * @param sig 231 | * Precomputed hash value for 'key'. 232 | * @return 233 | * - -EINVAL if the parameters are invalid. 234 | * - -ENOSPC if there is no space in the hash for this key. 235 | * - A positive value that can be used by the caller as an offset into an 236 | * array of user data. This value is unique for this key. 237 | */ 238 | int32_t 239 | rte_hash_v1702_add_key_with_hash(const struct rte_hash_v1702 *h, const void *key, hash_sig_t sig); 240 | 241 | /** 242 | * Remove a key from an existing hash table. 243 | * This operation is not multi-thread safe 244 | * and should only be called from one thread. 245 | * 246 | * @param h 247 | * Hash table to remove the key from. 248 | * @param key 249 | * Key to remove from the hash table. 250 | * @return 251 | * - -EINVAL if the parameters are invalid. 252 | * - -ENOENT if the key is not found. 253 | * - A positive value that can be used by the caller as an offset into an 254 | * array of user data. This value is unique for this key, and is the same 255 | * value that was returned when the key was added. 256 | */ 257 | int32_t 258 | rte_hash_v1702_del_key(const struct rte_hash_v1702 *h, const void *key); 259 | 260 | /** 261 | * Remove a key from an existing hash table. 262 | * This operation is not multi-thread safe 263 | * and should only be called from one thread. 264 | * 265 | * @param h 266 | * Hash table to remove the key from. 267 | * @param key 268 | * Key to remove from the hash table. 269 | * @param sig 270 | * Precomputed hash value for 'key'. 271 | * @return 272 | * - -EINVAL if the parameters are invalid. 273 | * - -ENOENT if the key is not found. 274 | * - A positive value that can be used by the caller as an offset into an 275 | * array of user data. This value is unique for this key, and is the same 276 | * value that was returned when the key was added. 277 | */ 278 | int32_t 279 | rte_hash_v1702_del_key_with_hash(const struct rte_hash_v1702 *h, const void *key, hash_sig_t sig); 280 | 281 | /** 282 | * Find a key in the hash table given the position. 283 | * This operation is multi-thread safe. 284 | * 285 | * @param h 286 | * Hash table to get the key from. 287 | * @param position 288 | * Position returned when the key was inserted. 289 | * @param key 290 | * Output containing a pointer to the key 291 | * @return 292 | * - 0 if retrieved successfully 293 | * - EINVAL if the parameters are invalid. 294 | * - ENOENT if no valid key is found in the given position. 295 | */ 296 | int 297 | rte_hash_v1702_get_key_with_position(const struct rte_hash_v1702 *h, const int32_t position, 298 | void **key); 299 | 300 | /** 301 | * Find a key-value pair in the hash table. 302 | * This operation is multi-thread safe. 303 | * 304 | * @param h 305 | * Hash table to look in. 306 | * @param key 307 | * Key to find. 308 | * @param data 309 | * Output with pointer to data returned from the hash table. 310 | * @return 311 | * 0 if successful lookup 312 | * - EINVAL if the parameters are invalid. 313 | * - ENOENT if the key is not found. 314 | */ 315 | int 316 | rte_hash_v1702_lookup_data(const struct rte_hash_v1702 *h, const void *key, hash_data_t *data); 317 | 318 | /** 319 | * Find a key-value pair with a pre-computed hash value 320 | * to an existing hash table. 321 | * This operation is multi-thread safe. 322 | * 323 | * @param h 324 | * Hash table to look in. 325 | * @param key 326 | * Key to find. 327 | * @param sig 328 | * Precomputed hash value for 'key' 329 | * @param data 330 | * Output with pointer to data returned from the hash table. 331 | * @return 332 | * 0 if successful lookup 333 | * - EINVAL if the parameters are invalid. 334 | * - ENOENT if the key is not found. 335 | */ 336 | int 337 | rte_hash_v1702_lookup_with_hash_data(const struct rte_hash_v1702 *h, const void *key, 338 | hash_sig_t sig, hash_data_t *data); 339 | 340 | /** 341 | * Find a key in the hash table. 342 | * This operation is multi-thread safe. 343 | * 344 | * @param h 345 | * Hash table to look in. 346 | * @param key 347 | * Key to find. 348 | * @return 349 | * - -EINVAL if the parameters are invalid. 350 | * - -ENOENT if the key is not found. 351 | * - A positive value that can be used by the caller as an offset into an 352 | * array of user data. This value is unique for this key, and is the same 353 | * value that was returned when the key was added. 354 | */ 355 | int32_t 356 | rte_hash_v1702_lookup(const struct rte_hash_v1702 *h, const void *key); 357 | 358 | /** 359 | * Find a key in the hash table. 360 | * This operation is multi-thread safe. 361 | * 362 | * @param h 363 | * Hash table to look in. 364 | * @param key 365 | * Key to find. 366 | * @param sig 367 | * Hash value to remove from the hash table. 368 | * @return 369 | * - -EINVAL if the parameters are invalid. 370 | * - -ENOENT if the key is not found. 371 | * - A positive value that can be used by the caller as an offset into an 372 | * array of user data. This value is unique for this key, and is the same 373 | * value that was returned when the key was added. 374 | */ 375 | int32_t 376 | rte_hash_v1702_lookup_with_hash(const struct rte_hash_v1702 *h, 377 | const void *key, hash_sig_t sig); 378 | 379 | /** 380 | * Calc a hash value by key. 381 | * This operation is not multi-thread safe. 382 | * 383 | * @param h 384 | * Hash table to look in. 385 | * @param key 386 | * Key to find. 387 | * @return 388 | * - hash value 389 | */ 390 | hash_sig_t 391 | rte_hash_v1702_hash(const struct rte_hash_v1702 *h, const void *key); 392 | 393 | /** 394 | * Find multiple keys in the hash table. 395 | * This operation is multi-thread safe. 396 | * 397 | * @param h 398 | * Hash table to look in. 399 | * @param keys 400 | * A pointer to a list of keys to look for. 401 | * @param num_keys 402 | * How many keys are in the keys list (less than RTE_HASH_V1702_LOOKUP_BULK_MAX). 403 | * @param hit_mask 404 | * Output containing a bitmask with all successful lookups. 405 | * @param data 406 | * Output containing array of data returned from all the successful lookups. 407 | * @return 408 | * -EINVAL if there's an error, otherwise number of successful lookups. 409 | */ 410 | int 411 | rte_hash_v1702_lookup_bulk_data(const struct rte_hash_v1702 *h, const void **keys, 412 | uint32_t num_keys, uint64_t *hit_mask, hash_data_t data[]); 413 | 414 | /** 415 | * Find multiple keys in the hash table. 416 | * This operation is multi-thread safe. 417 | * 418 | * @param h 419 | * Hash table to look in. 420 | * @param keys 421 | * A pointer to a list of keys to look for. 422 | * @param num_keys 423 | * How many keys are in the keys list (less than RTE_HASH_V1702_LOOKUP_BULK_MAX). 424 | * @param positions 425 | * Output containing a list of values, corresponding to the list of keys that 426 | * can be used by the caller as an offset into an array of user data. These 427 | * values are unique for each key, and are the same values that were returned 428 | * when each key was added. If a key in the list was not found, then -ENOENT 429 | * will be the value. 430 | * @return 431 | * -EINVAL if there's an error, otherwise 0. 432 | */ 433 | int 434 | rte_hash_v1702_lookup_bulk(const struct rte_hash_v1702 *h, const void **keys, 435 | uint32_t num_keys, int32_t *positions); 436 | 437 | /** 438 | * Iterate through the hash table, returning key-value pairs. 439 | * 440 | * @param h 441 | * Hash table to iterate 442 | * @param key 443 | * Output containing the key where current iterator 444 | * was pointing at 445 | * @param data 446 | * Output containing the data associated with key. 447 | * Returns NULL if data was not stored. 448 | * @param next 449 | * Pointer to iterator. Should be 0 to start iterating the hash table. 450 | * Iterator is incremented after each call of this function. 451 | * @return 452 | * Position where key was stored, if successful. 453 | * - -EINVAL if the parameters are invalid. 454 | * - -ENOENT if end of the hash table. 455 | */ 456 | int32_t 457 | rte_hash_v1702_iterate(const struct rte_hash_v1702 *h, const void **key, hash_data_t *data, uint32_t *next); 458 | #ifdef __cplusplus 459 | } 460 | #endif 461 | 462 | #endif /* _RTE_HASH_V1702_H_ */ 463 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_tch_hash.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #ifndef LIBRTE_TCH_HASH_RTE_TCH_HASH_H_ 10 | #define LIBRTE_TCH_HASH_RTE_TCH_HASH_H_ 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #include 33 | #include 34 | 35 | enum rte_tch_hash_variants { 36 | H_V1604 = 0, 37 | H_V1702, 38 | H_LAZY_BLOOM, 39 | H_LAZY_COND, 40 | H_LAZY_UNCOND, 41 | H_LAZY_NO, 42 | H_HORTON, 43 | H_BLOOM, 44 | H_COND, 45 | H_UNCOND 46 | }; 47 | 48 | const char * variants_names[] = { 49 | "DPDK_1604", 50 | "DPDK_1702", 51 | "LAZY_BLOOM", 52 | "LAZY_COND", 53 | "LAZY_UNCOND", 54 | "LAZY_NO", 55 | "HORTON", 56 | "BLOOM", 57 | "COND", 58 | "UNCOND" 59 | }; 60 | 61 | /** @internal A hash table structure. */ 62 | struct rte_tch_hash { 63 | union { 64 | struct rte_hash_v1604 * h_dpdk1604; 65 | struct rte_hash_v1702 * h_dpdk1702; 66 | struct rte_hash_hvariant * h_tch; // Non-defined type (polymorphic) 67 | }; 68 | }; 69 | 70 | /** 71 | * Parameters used when creating the hash table. 72 | */ 73 | struct rte_tch_hash_parameters { 74 | uint32_t entries; /**< Total hash table entries. */ 75 | int socket_id; /**< NUMA Socket ID for memory. */ 76 | }; 77 | 78 | 79 | #define EXPAND(F) \ 80 | if(v == H_HORTON){ F(horton)} \ 81 | if(v == H_LAZY_BLOOM){ F(lazy_bloom)} \ 82 | if(v == H_BLOOM){ F(bloom)} \ 83 | if(v == H_LAZY_COND){ F(lazy_cond)} \ 84 | if(v == H_COND){ F(cond)} \ 85 | if(v == H_LAZY_UNCOND){ F(lazy_uncond)} \ 86 | if(v == H_UNCOND){ F(uncond)} \ 87 | if(v == H_LAZY_NO){ F(lazy_no)} 88 | 89 | 90 | 91 | 92 | /** 93 | * Reset all hash structure, by zeroing all entries 94 | * @param h 95 | * Hash table to reset 96 | */ 97 | #define RESET(x) return rte_hash_##x##_reset(h->h_tch); 98 | static inline void rte_tch_hash_reset(enum rte_tch_hash_variants v,struct rte_tch_hash *h){ 99 | if(v == H_V1604) return rte_hash_v1604_reset(h->h_dpdk1604); 100 | if(v == H_V1702) return rte_hash_v1702_reset(h->h_dpdk1702); 101 | EXPAND(RESET) 102 | } 103 | 104 | /** 105 | * Print stats to stdout 106 | */ 107 | #define PRINT(x) return rte_hash_##x##_print_stats(h->h_tch,currentTime); 108 | static inline void rte_tch_hash_print_stats(enum rte_tch_hash_variants v,struct rte_tch_hash *h, uint16_t currentTime){ 109 | if(v == H_V1604) return ; 110 | if(v == H_V1702) return ; 111 | EXPAND(PRINT) 112 | } 113 | 114 | /** 115 | * Create a new hash table. 116 | * 117 | * @param params 118 | * Parameters used to create and initialise the hash table. 119 | * @return 120 | * Pointer to hash table structure that is used in future hash table 121 | * operations, or NULL on error, with error code set in rte_errno. 122 | * Possible rte_errno errors include: 123 | * - E_RTE_NO_CONFIG - function could not get pointer to rte_config structure 124 | * - E_RTE_SECONDARY - function was called from a secondary process instance 125 | * - ENOENT - missing entry 126 | * - EINVAL - invalid parameter passed to function 127 | * - ENOSPC - the maximum number of memzones has already been allocated 128 | * - EEXIST - a memzone with the same name already exists 129 | * - ENOMEM - no appropriate memory area found in which to create memzone 130 | */ 131 | #define CREATE(x) h->h_tch = rte_hash_##x##_create(&p); 132 | static inline struct rte_tch_hash * 133 | rte_tch_hash_create(enum rte_tch_hash_variants v, const struct rte_tch_hash_parameters *params){ 134 | char buf[L_tmpnam]; 135 | char * name = tmpnam(buf); 136 | struct rte_tch_hash * h = (struct rte_tch_hash *) rte_zmalloc(NULL,sizeof(struct rte_tch_hash),64); 137 | struct rte_hash_hvariant_parameters p; 138 | p.entries=params->entries; 139 | p.socket_id=params->socket_id; 140 | p.name=name; 141 | 142 | if(name == NULL) rte_exit(EXIT_FAILURE, "Failed to generate temporary name for hash table\n"); 143 | if(v == H_V1604){ 144 | struct rte_hash_v1604_parameters p; 145 | p.entries=params->entries; 146 | p.key_len=16; 147 | p.socket_id=params->socket_id; 148 | p.name=name; 149 | p.extra_flag = 0; 150 | p.reserved = 0; 151 | p.hash_func_init_val = 0xffeeffee; 152 | p.hash_func = rte_tch_hash_function; 153 | h->h_dpdk1604 = rte_hash_v1604_create(&p); 154 | } 155 | if(v == H_V1702){ 156 | struct rte_hash_v1702_parameters p; 157 | p.entries=params->entries; 158 | p.key_len=16; 159 | p.socket_id=params->socket_id; 160 | p.name=name; 161 | p.extra_flag = 0; 162 | p.reserved = 0; 163 | p.hash_func_init_val = 0xffeeffee; 164 | p.hash_func = rte_tch_hash_function; 165 | h->h_dpdk1702 = rte_hash_v1702_create(&p); 166 | } 167 | EXPAND(CREATE) 168 | return h; 169 | } 170 | 171 | 172 | 173 | /** 174 | * Add a key-value pair to an existing hash table. 175 | * This operation is not multi-thread safe 176 | * and should only be called from one thread. 177 | * 178 | * @param h 179 | * Hash table to add the key to. 180 | * @param key 181 | * Key to add to the hash table. 182 | * @param data 183 | * Data to add to the hash table. 184 | * @param expirationTime 185 | * Timeunit at which is the inserted entry should be expired 186 | * @param currentTime 187 | * Current time unit 188 | * @return 189 | * - RHL_FOUND_UPDATED if the key was added 190 | * - -EINVAL if the parameters are invalid. 191 | * - -ENOSPC if there is>= 0 ? RHL_FOUND_NOTUPDATED : -1 no space in the hash for this key. 192 | */ 193 | #define ADDKD(x) return rte_hash_##x##_add_key_data(h->h_tch,key,data,expirationTime,currentTime); 194 | static inline int rte_tch_hash_add_key_data(enum rte_tch_hash_variants v, struct rte_tch_hash *h, const hash_key_t key, hash_data_t data, uint16_t expirationTime, uint16_t currentTime){ 195 | if(v == H_V1604) return rte_hash_v1604_add_key_data(h->h_dpdk1604,&key,data) >= 0 ? RHL_FOUND_UPDATED : -1 ; 196 | if(v == H_V1702) return rte_hash_v1702_add_key_data(h->h_dpdk1702,&key,data) >= 0 ? RHL_FOUND_UPDATED : -1 ; 197 | EXPAND(ADDKD); 198 | return -1; 199 | } 200 | 201 | #define DELK(x) return rte_hash_##x##_del_key(h->h_tch,key,currentTime); 202 | static inline int rte_tch_hash_del_key(enum rte_tch_hash_variants v, struct rte_tch_hash *h, const hash_key_t key, uint16_t currentTime){ 203 | if(v == H_V1604) return rte_hash_v1604_del_key(h->h_dpdk1604,&key) >= 0 ? RHL_FOUND_UPDATED : -1 ; 204 | if(v == H_V1702)return rte_hash_v1702_del_key(h->h_dpdk1702,&key) >= 0 ? RHL_FOUND_UPDATED : -1 ; 205 | EXPAND(DELK); 206 | return -1; 207 | } 208 | 209 | 210 | /** 211 | * Find a key-value pair in the hash table. 212 | * This operation is multi-thread safe. 213 | * 214 | * @param h 215 | * Hash table to look in. 216 | * @param key 217 | * Key to find. 218 | * @param data 219 | * Output with pointer to data returned from the hash table. 220 | * @param currentTime 221 | * Current time unit 222 | * @return 223 | * - RHL_FOUND_NOT_UPDATED if the key was found 224 | * - EINVAL if the parameters are invalid. 225 | * - ENOENT if the key is not found. 226 | */ 227 | #define LOOKD(x) return rte_hash_##x##_lookup_data(h->h_tch,key,data,currentTime); 228 | static inline int rte_tch_hash_lookup_data(enum rte_tch_hash_variants v, struct rte_tch_hash *h, const hash_key_t key, hash_data_t *data, uint16_t currentTime){ 229 | if(v == H_V1604)return rte_hash_v1604_lookup_data(h->h_dpdk1604,&key,data) >= 0 ? RHL_FOUND_NOTUPDATED : -1 ; 230 | if(v == H_V1702) return rte_hash_v1702_lookup_data(h->h_dpdk1702,&key,data) >= 0 ? RHL_FOUND_NOTUPDATED : -1 ; 231 | EXPAND(LOOKD); 232 | return -1; 233 | } 234 | 235 | /** 236 | * Find a key-value pair with a pre-computed hash value 237 | * to an existing hash table. 238 | * This operation is multi-thread safe. 239 | * 240 | * @param h 241 | * Hash table to look in. 242 | * @param key 243 | * Key to find. 244 | * @param sig 245 | * Precomputed hash value for 'key' 246 | * @param data 247 | * Output with pointer to data returned from the hash table. 248 | * @param currentTime 249 | * Current time unit 250 | * @return 251 | * - RHL_FOUND_NOT_UPDATED if the key was found 252 | * - EINVAL if the parameters are invalid. 253 | * - ENOENT if the key is not found. 254 | */ 255 | #define LOOKDH(x) return rte_hash_##x##_lookup_with_hash_data(h->h_tch,key,sig,data,currentTime); 256 | static inline int rte_tch_hash_lookup_with_hash_data(enum rte_tch_hash_variants v, struct rte_tch_hash *h, const hash_key_t key, 257 | hash_sig64_t sig, hash_data_t *data, uint16_t currentTime){ 258 | if(v == H_V1604){ 259 | return rte_hash_v1604_lookup_with_hash_data(h->h_dpdk1604,&key,sig,data) >= 0 ? RHL_FOUND_NOTUPDATED : -1 ; 260 | } 261 | if(v == H_V1702){ 262 | return rte_hash_v1702_lookup_with_hash_data(h->h_dpdk1702,&key,sig,data) >= 0 ? RHL_FOUND_NOTUPDATED : -1; 263 | } 264 | EXPAND(LOOKDH) 265 | return -1; 266 | } 267 | 268 | #define LOOKH(x) return rte_hash_##x##_lookup_with_hash(h->h_tch,key,sig,currentTime); 269 | static inline int rte_tch_hash_lookup_with_hash(enum rte_tch_hash_variants v, struct rte_tch_hash *h, const hash_key_t key, 270 | hash_sig64_t sig, uint16_t currentTime){ 271 | if(v == H_V1604){ 272 | return rte_hash_v1604_lookup_with_hash(h->h_dpdk1604,&key,sig) >= 0 ? RHL_FOUND_NOTUPDATED : -1 ; 273 | } 274 | if(v == H_V1702){ 275 | return rte_hash_v1702_lookup_with_hash(h->h_dpdk1702,&key,sig) >= 0 ? RHL_FOUND_NOTUPDATED : -1; 276 | } 277 | EXPAND(LOOKH) 278 | return -1; 279 | } 280 | 281 | 282 | #define LOOKDUH(x) return rte_hash_##x##_lookup_update_with_hash_data(h->h_tch,key,sig,data,expirationTime,currentTime); 283 | static inline int rte_tch_hash_lookup_update_with_hash_data(enum rte_tch_hash_variants v, struct rte_tch_hash *h,const hash_key_t key, hash_sig64_t sig, hash_data_t *data, uint16_t expirationTime, uint16_t currentTime){ 284 | if(v == H_V1604){ 285 | return rte_hash_v1604_lookup_with_hash_data(h->h_dpdk1604,&key,sig,data) >= 0 ? RHL_FOUND_NOTUPDATED : -1; 286 | } 287 | if(v == H_V1702){ 288 | return rte_hash_v1702_lookup_with_hash_data(h->h_dpdk1702,&key,sig,data) >= 0 ? RHL_FOUND_NOTUPDATED : -1; 289 | } 290 | EXPAND(LOOKDUH) 291 | return -1; 292 | } 293 | 294 | #define LOOKDU(x) return rte_hash_##x##_lookup_update_data(h->h_tch,key,data,expirationTime,currentTime); 295 | static inline int rte_tch_hash_lookup_update_data(enum rte_tch_hash_variants v, struct rte_tch_hash *h, const hash_key_t key, hash_data_t *data, uint16_t expirationTime, uint16_t currentTime){ 296 | if(v == H_V1604){ 297 | return rte_hash_v1604_lookup_data(h->h_dpdk1604,&key,data) >= 0 ? RHL_FOUND_NOTUPDATED : -1; 298 | } 299 | if(v == H_V1702){ 300 | return rte_hash_v1702_lookup_data(h->h_dpdk1702,&key,data) >= 0 ? RHL_FOUND_NOTUPDATED : -1; 301 | } 302 | EXPAND(LOOKDU) 303 | return -1; 304 | } 305 | 306 | 307 | /** 308 | * Add a key-value pair with a pre-computed hash value 309 | * to an existing hash table. 310 | * This operation is not multi-thread safe 311 | * and should only be called from one thread. 312 | * 313 | * @param h 314 | * Hash table to add the key to. 315 | * @param key 316 | * Key to add to the hash table. 317 | * @param sig 318 | * Precomputed hash value for 'key' 319 | * @param data 320 | * Data to add to the hash table. 321 | * @param expirationTime 322 | * Timeunit at which is the inserted entry should be expired 323 | * @param currentTime 324 | * Current time unit 325 | * @return 326 | * - RHL_FOUND_UPDATED if the key was added 327 | * - -EINVAL if the parameters are invalid. 328 | * - -ENOSPC if there is no space in the hash for this key. 329 | */ 330 | #define ADDKHD(x) return rte_hash_##x##_add_key_with_hash_data(h->h_tch,key,sig,data,expirationTime,currentTime); 331 | 332 | static inline int32_t 333 | rte_tch_hash_add_key_with_hash_data(enum rte_tch_hash_variants v, struct rte_tch_hash *h, const hash_key_t key, 334 | hash_sig64_t sig, hash_data_t data, uint16_t expirationTime, uint16_t currentTime){ 335 | if(v == H_V1604){ 336 | return rte_hash_v1604_add_key_with_hash_data(h->h_dpdk1604,&key,sig,data) >= 0 ? RHL_FOUND_UPDATED: -ENOSPC ; 337 | } 338 | if(v == H_V1702){ 339 | return rte_hash_v1702_add_key_with_hash_data(h->h_dpdk1702,&key,sig,data) >= 0 ? RHL_FOUND_UPDATED: -ENOSPC; 340 | } 341 | EXPAND(ADDKHD) 342 | return -1; 343 | } 344 | 345 | static inline void keys_to_ptr(const void ** keys_ptr, const hash_key_t * keys, uint32_t num_keys){ 346 | assert(num_keys <= 64); 347 | uint32_t i; 348 | for(i=0;ih_tch,keys, num_keys,hit_mask,data,currentTime); 354 | static inline int rte_tch_hash_lookup_bulk_data(enum rte_tch_hash_variants v, struct rte_tch_hash *h, const hash_key_t *keys, uint32_t num_keys, uint64_t *hit_mask, hash_data_t data[], uint16_t currentTime){ 355 | if(v == H_V1604){ 356 | const void * keys_ptr[64]; 357 | keys_to_ptr(keys_ptr,keys,num_keys); 358 | return rte_hash_v1604_lookup_bulk_data(h->h_dpdk1604,keys_ptr, num_keys,hit_mask,data); 359 | } 360 | if(v == H_V1702){ 361 | const void * keys_ptr[64]; 362 | keys_to_ptr(keys_ptr,keys,num_keys); 363 | return rte_hash_v1702_lookup_bulk_data(h->h_dpdk1702,keys_ptr, num_keys,hit_mask,data); 364 | } 365 | EXPAND(BULKD) 366 | return -1; 367 | } 368 | 369 | /** 370 | * Find multiple keys in the hash table. 371 | * This operation is multi-thread safe. 372 | * 373 | * @param h 374 | * Hash table to look in. 375 | * @param keys 376 | * A pointer to a list of keys to look for. 377 | * @param num_keys 378 | * How many keys are in the keys list (less than RTE_HASH_LOOKUP_BULK_MAX). 379 | * @param positions 380 | * Output containing a list of values, corresponding to the list of keys that 381 | * can be used by the caller as an offset into an array of user data. These 382 | * values are unique for each key, and are the same values that were returned 383 | * when each key was added. If a key in the list was not found, then -ENOENT 384 | * will be the value. 385 | * @param currentTime 386 | * Current time unit 387 | * @return 388 | * -EINVAL if there's an error, otherwise 0. 389 | */ 390 | //static inline int rte_tch_hash_lookup_bulk(enum rte_tch_hash_variants v, struct rte_hash_lazy *h, const hash_key_t *keys, uint32_t num_keys, int32_t *positions, uint16_t currentTime); 391 | 392 | #define FOREACH_IN_MASK64(i,m,mn) for(mn=m,i=__builtin_ctzl(mn); mn != 0; mn &= ~(1llu << i),i=__builtin_ctzl(mn)) 393 | 394 | #define BULKDM(x) return rte_hash_##x##_lookup_bulk_data_mask(h->h_tch,keys, lookup_mask,hit_mask,data,currentTime); 395 | static inline int rte_tch_hash_lookup_bulk_data_mask(enum rte_tch_hash_variants v, struct rte_tch_hash *h, const hash_key_t *keys, uint64_t lookup_mask, uint64_t *hit_mask, hash_data_t data[], uint16_t currentTime){ 396 | if(v == H_V1604){ 397 | return -1; 398 | }else if(v == H_V1702){ 399 | int i; 400 | int j=0; 401 | uint64_t tmp_mask; 402 | (*hit_mask) =0; 403 | const void * keys_compact[64]; 404 | hash_data_t data_compact[64]; 405 | uint64_t hit_mask_compact=0; 406 | FOREACH_IN_MASK64(i,lookup_mask,tmp_mask){ 407 | keys_compact[j] = &keys[i]; 408 | j++; 409 | } 410 | int r=rte_hash_v1702_lookup_bulk_data(h->h_dpdk1702,keys_compact,j,&hit_mask_compact,data_compact); 411 | FOREACH_IN_MASK64(i,lookup_mask,tmp_mask){ 412 | int match = (hit_mask_compact >> j) & 1; 413 | if(match){ 414 | data[i] = data_compact[j]; 415 | (*hit_mask) |= (1 << i); 416 | } 417 | j++; 418 | } 419 | return r; 420 | } 421 | EXPAND(BULKDM) 422 | return -1; 423 | } 424 | 425 | #define BULKDUM(x) return rte_hash_##x##_lookup_update_bulk_data_mask(h->h_tch,keys, lookup_mask,hit_mask,updated_mask,data,newExpirationTime,currentTime); 426 | static inline int rte_tch_hash_lookup_update_bulk_data_mask(enum rte_tch_hash_variants v, struct rte_tch_hash *h, const hash_key_t *keys, uint64_t lookup_mask, uint64_t *hit_mask, uint64_t *updated_mask, hash_data_t data[], uint16_t * newExpirationTime, uint16_t currentTime){ 427 | if(v == H_V1604){ 428 | int r = rte_tch_hash_lookup_bulk_data_mask(v,h,keys,lookup_mask,hit_mask,data,currentTime); 429 | *updated_mask=*hit_mask; 430 | return r; 431 | } 432 | if(v == H_V1702){ 433 | int r = rte_tch_hash_lookup_bulk_data_mask(v,h,keys,lookup_mask,hit_mask,data,currentTime); 434 | *updated_mask=*hit_mask; 435 | return r; 436 | } 437 | EXPAND(BULKDUM) 438 | return -1; 439 | } 440 | 441 | 442 | /** 443 | * Reset iterator (only a single iterator can be active at a time on the hashtable) 444 | * 445 | * @param h 446 | * Hash table to iterate 447 | */ 448 | #define RESET_ITERATOR(x) return rte_hash_##x##_iterator_reset(h->h_tch); 449 | static inline void rte_tch_hash_iterator_reset(__rte_unused enum rte_tch_hash_variants v, __rte_unused struct rte_tch_hash *h){ 450 | if(v == H_V1604){ 451 | assert(0); 452 | } 453 | if(v == H_V1702){ 454 | assert(0); 455 | } 456 | EXPAND(RESET_ITERATOR); 457 | return; 458 | 459 | } 460 | 461 | 462 | /** 463 | * Iterate through the hash table, returning key-value pairs. 464 | * 465 | * @param h 466 | * Hash table to iterate 467 | * @param key 468 | * Output containing the key where current iterator 469 | * was pointing at 470 | * @param data 471 | * Output containing the data associated with key. 472 | * Returns NULL if data was not stored. 473 | * @param currentTime 474 | * Current time unit 475 | * @return 476 | * Position where key was stored, if successful. 477 | * - -EINVAL if the parameters are invalid. 478 | * - -ENOENT if end of the hash table. 479 | */ 480 | #define ITERATE(x) return rte_hash_##x##_iterate(h->h_tch,key,data,remaining_time,currentTime); 481 | static inline int32_t 482 | rte_tch_hash_iterate(enum rte_tch_hash_variants v, struct rte_tch_hash *h, hash_key_t *key, hash_data_t *data, uint16_t * remaining_time, uint16_t currentTime){ 483 | if(v == H_V1604){ 484 | key->a = 0; 485 | key->b = 0; 486 | data->a = 0; 487 | data->b = 0; 488 | } 489 | if(v == H_V1702){ 490 | key->a = 0; 491 | key->b = 0; 492 | data->a = 0; 493 | data->b = 0; 494 | } 495 | EXPAND(ITERATE) 496 | return -1; 497 | } 498 | 499 | /** 500 | * Iterate through the hash table, returning key-value pairs -- unsafe version (for debugging/statistics only!!!). 501 | * Multiple unsafe iterator can be run on the hashtable. 502 | * 503 | * @param h 504 | * Hash table to iterate 505 | * @param pos 506 | * Position in the hash table 507 | * @param key 508 | * Output containing the key where current iterator 509 | * was pointing at 510 | * @param data 511 | * Output containing the data associated with key. 512 | * Returns NULL if data was not stored. 513 | * @param currentTime 514 | * Current time unit 515 | * @return 516 | * Position where key was stored, if successful. 517 | * - -EINVAL if the parameters are invalid. 518 | * - -ENOENT if end of the hash table. 519 | */ 520 | #define ITERATEUN(x) return rte_hash_##x##_unsafe_iterate(h->h_tch,pos,key,data,remaining_time,currentTime); 521 | static inline int32_t 522 | rte_tch_hash_unsafe_iterate(enum rte_tch_hash_variants v, struct rte_tch_hash *h, uint64_t * pos, hash_key_t *key, hash_data_t *data, uint16_t * remaining_time, uint16_t currentTime){ 523 | if(v == H_V1604){ 524 | key->a = 0; 525 | key->b = 0; 526 | data->a = 0; 527 | data->b = 0; 528 | } 529 | if(v == H_V1702){ 530 | key->a = 0; 531 | key->b = 0; 532 | data->a = 0; 533 | data->b = 0; 534 | } 535 | EXPAND(ITERATEUN) 536 | return -1; 537 | } 538 | 539 | /** 540 | * Check the integrity of the structure 541 | * 542 | * @param h 543 | * Hash table to iterate 544 | * @param currentTime 545 | * Current time unit 546 | */ 547 | #define INTEGRITY(x) rte_hash_##x##_check_integrity(h->h_tch,currentTime); 548 | static inline void rte_tch_hash_check_integrity(enum rte_tch_hash_variants v, struct rte_tch_hash *h, uint16_t currentTime){ 549 | if(v == H_V1604){ 550 | return ; 551 | }else if(v == H_V1702){ 552 | return ; 553 | } 554 | EXPAND(INTEGRITY) 555 | } 556 | 557 | #define FREE(x) rte_hash_##x##_free(h->h_tch); 558 | static inline void rte_tch_hash_free(enum rte_tch_hash_variants v, struct rte_tch_hash *h){ 559 | if(v == H_V1604){ 560 | return rte_hash_v1604_free(h->h_dpdk1604); 561 | }else if(v == H_V1702){ 562 | return rte_hash_v1702_free(h->h_dpdk1702); 563 | } 564 | EXPAND(FREE) 565 | rte_free(h); 566 | } 567 | 568 | #define SIZE(x) return rte_hash_##x##_size(h->h_tch,currentTime); 569 | static inline int rte_tch_hash_size(enum rte_tch_hash_variants v, struct rte_tch_hash *h, uint16_t currentTime){ 570 | if(v == H_V1604){ 571 | return -1; 572 | }else if(v == H_V1702){ 573 | return -1; 574 | } 575 | EXPAND(SIZE) 576 | return -1; 577 | } 578 | 579 | #define CAPACITY(x) return rte_hash_##x##_capacity(h->h_tch); 580 | static inline uint32_t rte_tch_hash_capacity(enum rte_tch_hash_variants v, struct rte_tch_hash *h){ 581 | if(v == H_V1604){ 582 | return -1; 583 | }else if(v == H_V1702){ 584 | return -1; 585 | } 586 | EXPAND(CAPACITY) 587 | return -1; 588 | } 589 | 590 | 591 | #define STATS(x) return rte_hash_##x##_stats_secondary(h->h_tch, currentTime); 592 | static inline double rte_tch_hash_stats_secondary(enum rte_tch_hash_variants v, struct rte_tch_hash *h, uint16_t currentTime){ 593 | if(v == H_V1604){ 594 | return rte_hash_v1604_stats_secondary(h->h_dpdk1604); 595 | }else if(v == H_V1702){ 596 | return nan(""); 597 | } 598 | EXPAND(STATS) 599 | return nan(""); 600 | } 601 | 602 | #define BUCKETS(x) return rte_hash_##x##_slots_per_bucket(); 603 | static inline int rte_tch_hash_slots_per_bucket(enum rte_tch_hash_variants v){ 604 | if(v == H_V1604){ 605 | return 4; 606 | }else if(v == H_V1702){ 607 | return 8; 608 | } 609 | EXPAND(BUCKETS) 610 | return 0; 611 | } 612 | 613 | static inline const char * rte_tch_hash_str(enum rte_tch_hash_variants v){ 614 | if(v < sizeof(variants_names)){ 615 | return variants_names[v]; 616 | }else{ 617 | return "UNKNOWN"; 618 | } 619 | } 620 | 621 | #endif /* LIBRTE_TCH_HASH_RTE_TCH_HASH_H_ */ 622 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_tch_utils.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #ifndef _RTE_TCH_UTILS_H_ 10 | #define _RTE_TCH_UTILS_H_ 11 | #include 12 | #include 13 | 14 | struct rte_tch_rand_state{ 15 | uint64_t s0; 16 | uint64_t s1; 17 | }; 18 | 19 | /* The state must be seeded so that it is not everywhere zero. */ 20 | static inline uint64_t xorshift128plus(struct rte_tch_rand_state *r) { 21 | uint64_t x = r->s0; 22 | uint64_t const y = r->s1; 23 | r->s0 = y; 24 | x ^= x << 23; // a 25 | r->s1 = x ^ y ^ (x >> 17) ^ (y >> 26); // b, c 26 | return r->s1 + y; 27 | } 28 | 29 | 30 | 31 | static inline uint32_t random_max(uint32_t max, struct rte_tch_rand_state *r){ 32 | uint64_t rd = xorshift128plus(r) & 0xffffffffULL; 33 | uint64_t res = ( rd * (uint64_t)max ) >> 32ULL; // This is more or less the same as a modulo max, but much faster to execute. 34 | return res; 35 | } 36 | 37 | 38 | static inline void rte_tch_rand_init(struct rte_tch_rand_state *r){ 39 | r->s0 = rte_rand(); 40 | r->s1 = rte_rand(); 41 | } 42 | /** 43 | * Force alignment to TWO cache lines to avoid side effects of the adjacent cache line prefetcher. 44 | */ 45 | #define __rte_tch_twocache_aligned __rte_aligned(2*RTE_CACHE_LINE_SIZE) 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /lib/librte_tch_hash/rte_tchh_structs.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2018 - Present – Thomson Licensing, SAS 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the Clear BSD license found in the 6 | * LICENSE.md file in the root directory of this source tree. 7 | */ 8 | 9 | #ifndef RTE_TCHH_STRUCTS_H 10 | #define RTE_TCHH_STRUCTS_H 11 | 12 | #include "x86intrin.h" 13 | #include "immintrin.h" 14 | 15 | struct rte_tch_key { 16 | union{ 17 | struct { 18 | uint64_t a; 19 | uint64_t b; 20 | }; 21 | __m128i mm; 22 | }; 23 | }; 24 | 25 | typedef struct rte_tch_key hash_key_t; 26 | 27 | struct rte_tch_data { 28 | union{ 29 | struct { 30 | uint64_t a; 31 | uint64_t b; 32 | }; 33 | __m128i mm; 34 | }; 35 | }; 36 | 37 | typedef struct rte_tch_data hash_data_t; 38 | typedef uint64_t hash_sig64_t; 39 | 40 | #endif 41 | --------------------------------------------------------------------------------