├── .clang-format ├── .gitignore ├── LICENSE.txt ├── Makefile.am ├── README.md ├── automake.sh ├── configure.ac ├── docs ├── README.md ├── capqs.md ├── dicts.md ├── hatstack.md ├── migration.md ├── queue.md ├── ring.md └── sets.md ├── examples ├── README.md ├── array.c ├── basic.c ├── hashable.c ├── logringex.c ├── old_queue_ex.c ├── qperf.c ├── qtest.c ├── ring.c └── set1.c ├── include ├── hatrack.h ├── hatrack │ ├── ballcap.h │ ├── capq.h │ ├── counters.h │ ├── crown.h │ ├── debug.h │ ├── dict.h │ ├── duncecap.h │ ├── flexarray.h │ ├── gate.h │ ├── hash.h │ ├── hatomic.h │ ├── hatrack_common.h │ ├── hatrack_config.h │ ├── hatring.h │ ├── hatvtable.h │ ├── helpmanager.h │ ├── hihat.h │ ├── hq.h │ ├── llstack.h │ ├── logring.h │ ├── lohat-a.h │ ├── lohat.h │ ├── lohat_common.h │ ├── mmm.h │ ├── newshat.h │ ├── oldhat.h │ ├── q64.h │ ├── queue.h │ ├── refhat.h │ ├── set.h │ ├── stack.h │ ├── swimcap.h │ ├── tiara.h │ ├── tophat.h │ ├── vector.h │ ├── witchhat.h │ ├── woolhat.h │ └── xxhash.h └── testhat.h ├── scripts ├── config-debug └── config-opt ├── src ├── array │ ├── flexarray.c │ └── vector.c ├── hash │ ├── README.md │ ├── ballcap.c │ ├── crown.c │ ├── dict.c │ ├── duncecap.c │ ├── hihat-a.c │ ├── hihat.c │ ├── lohat-a.c │ ├── lohat.c │ ├── newshat.c │ ├── oldhat.c │ ├── refhat.c │ ├── set.c │ ├── swimcap.c │ ├── tiara.c │ ├── tophat.c │ ├── witchhat.c │ ├── woolhat.c │ └── xxhash.c ├── queue │ ├── capq.c │ ├── debug.c │ ├── hatring.c │ ├── hq.c │ ├── llstack.c │ ├── logring.c │ ├── q64.c │ ├── queue.c │ └── stack.c └── support │ ├── counters.c │ ├── hatrack_common.c │ ├── helpmanager.c │ └── mmm.c └── tests ├── README.md ├── config.c ├── default.c ├── functional.c ├── performance.c ├── rand.c ├── test.c └── testhat.c /.clang-format: -------------------------------------------------------------------------------- 1 | AlignAfterOpenBracket: Align 2 | #13 AlignArrayOfStructures: Left 3 | AlignConsecutiveAssignments: AcrossComments 4 | AlignConsecutiveBitFields: AcrossComments 5 | AlignConsecutiveDeclarations: AcrossComments 6 | AlignConsecutiveMacros: AcrossComments 7 | AlignEscapedNewlines: Right 8 | AlignOperands: AlignAfterOperator 9 | AlignTrailingComments: true 10 | AllowAllArgumentsOnNextLine: false 11 | AllowAllParametersOfDeclarationOnNextLine: false 12 | AllowShortBlocksOnASingleLine: Empty 13 | AllowShortCaseLabelsOnASingleLine: false 14 | AllowShortEnumsOnASingleLine: false 15 | AllowShortFunctionsOnASingleLine: Empty 16 | AllowShortIfStatementsOnASingleLine: Never 17 | AllowShortLoopsOnASingleLine: false 18 | AlwaysBreakAfterReturnType: TopLevelDefinitions 19 | AlwaysBreakBeforeMultilineStrings: true 20 | 21 | BinPackArguments: false 22 | BinPackParameters: false 23 | BitFieldColonSpacing: Both 24 | BreakBeforeBinaryOperators: All 25 | BreakBeforeBraces: Custom 26 | BraceWrapping: 27 | AfterCaseLabel: false 28 | AfterClass: false 29 | AfterControlStatement: Never 30 | AfterEnum: false 31 | AfterFunction: true 32 | AfterNamespace: false 33 | AfterStruct: false 34 | AfterUnion: false 35 | AfterExternBlock: false 36 | BeforeCatch: true 37 | BeforeElse: true 38 | BeforeLambdaBody: false 39 | BeforeWhile: false 40 | IndentBraces: false 41 | SplitEmptyFunction: true 42 | BreakBeforeTernaryOperators: true 43 | BreakStringLiterals: true 44 | 45 | ColumnLimit: 80 46 | ContinuationIndentWidth: 4 47 | 48 | IncludeBlocks: Preserve 49 | IndentCaseBlocks: false 50 | IndentCaseLabels: false 51 | IndentExternBlock: NoIndent 52 | IndentGotoLabels: false 53 | IndentPPDirectives: None 54 | IndentWidth: 4 55 | IndentWrappedFunctionNames: false 56 | 57 | KeepEmptyLinesAtTheStartOfBlocks: false 58 | 59 | Language: Cpp 60 | 61 | MaxEmptyLinesToKeep: 1 62 | 63 | PointerAlignment: Right 64 | 65 | ReflowComments: true 66 | 67 | SortIncludes: false 68 | SpaceAfterCStyleCast: false 69 | SpaceAfterLogicalNot: false 70 | SpaceAroundPointerQualifiers: Default 71 | SpaceBeforeAssignmentOperators: true 72 | SpaceBeforeCaseColon: false 73 | SpaceBeforeCpp11BracedList: false 74 | SpaceBeforeParens: ControlStatements 75 | SpaceBeforeSquareBrackets: false 76 | SpaceInEmptyBlock: false 77 | SpaceInEmptyParentheses: false 78 | SpacesBeforeTrailingComments: 1 79 | SpacesInCStyleCastParentheses: false 80 | SpacesInConditionalStatement: false 81 | SpacesInParentheses: false 82 | SpacesInSquareBrackets: false 83 | 84 | TabWidth: 8 85 | 86 | UseCRLF: false 87 | UseTab: Never 88 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.[oa] 2 | .DS_Store 3 | test 4 | scripts 5 | attic 6 | autotools 7 | Makefile 8 | Makefile.in 9 | aclocal.m4 10 | autom4te.cache 11 | autotools 12 | config.log 13 | config.status 14 | configure 15 | include/hatrack/config.h 16 | include/hatrack/config.h.in 17 | include/hatrack/stamp-h1 18 | include/stamp-h1 19 | .deps 20 | .dirstamp 21 | *~ 22 | */*~ 23 | */*/*~ 24 | examples/basic 25 | examples/hashable 26 | examples/set1 27 | examples/oldqx 28 | examples/qtest 29 | examples/ring 30 | examples/logringex 31 | examples/array 32 | examples/qperf 33 | docs/~* 34 | #* -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | xxHash (xxhash.h and xxhash.c) is Copyright ©2012-2020 by Yann Collet, 2 | and is licensed under a BSD 2-clause license. 3 | 4 | All other files are Copyright © 2021-2022 John Viega, and 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use these files except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | check_PROGRAMS = tests/test 2 | noinst_PROGRAMS = examples/basic examples/set1 examples/hashable examples/oldqx examples/qtest examples/qperf examples/ring examples/logringex examples/array 3 | 4 | # 64-bit systems will complain up the wazoo about the 128-bit CAS operations. 5 | # Yes, they won't be lock free, but they will be sufficiently fast, thanks. 6 | libhatrack_a_CFLAGS = -Wall -Wextra -Wno-atomic-alignment -Wno-unused-parameter -I./include/ 7 | libhatrack_a_SOURCES = src/support/mmm.c src/support/counters.c src/support/hatrack_common.c src/support/helpmanager.c src/hash/refhat.c src/hash/duncecap.c src/hash/swimcap.c src/hash/newshat.c src/hash/ballcap.c src/hash/hihat.c src/hash/hihat-a.c src/hash/oldhat.c src/hash/lohat.c src/hash/lohat-a.c src/hash/witchhat.c src/hash/woolhat.c src/hash/tophat.c src/hash/crown.c src/hash/tiara.c src/hash/dict.c src/hash/set.c src/hash/xxhash.c src/queue/queue.c src/queue/q64.c src/queue/hq.c src/queue/capq.c src/queue/llstack.c src/queue/stack.c src/queue/hatring.c src/queue/logring.c src/queue/debug.c src/array/flexarray.c src/array/vector.c 8 | 9 | lib_LIBRARIES = libhatrack.a 10 | 11 | tests_test_SOURCES = ${libhatrack_a_SOURCES} tests/test.c tests/testhat.c tests/rand.c tests/config.c tests/functional.c tests/default.c tests/performance.c 12 | tests_test_CFLAGS = -DHATRACK_COMPILE_ALL_ALGORITHMS -Wall -Wextra -Wno-atomic-alignment -Wno-unused-parameter -I./include/ 13 | 14 | examples_basic_SOURCES = examples/basic.c 15 | examples_basic_CFLAGS = -Wall -Wextra -Wno-unused-parameter -I./include 16 | examples_basic_LDADD = ./libhatrack.a 17 | 18 | examples_set1_SOURCES = examples/set1.c 19 | examples_set1_CFLAGS = -Wall -Wextra -I./include 20 | examples_set1_LDADD = ./libhatrack.a 21 | 22 | examples_hashable_SOURCES = examples/hashable.c 23 | examples_hashable_CFLAGS = -Wall -Wextra -Wno-unused-parameter -I./include 24 | examples_hashable_LDADD = ./libhatrack.a 25 | 26 | examples_oldqx_SOURCES = examples/old_queue_ex.c 27 | examples_oldqx_CFLAGS = -Wall -Wextra -I./include 28 | examples_oldqx_LDADD = ./libhatrack.a 29 | 30 | examples_qtest_SOURCES = examples/qtest.c 31 | examples_qtest_CFLAGS = -Wall -Wextra -I./include 32 | examples_qtest_LDADD = ./libhatrack.a 33 | 34 | examples_qperf_SOURCES = examples/qperf.c 35 | examples_qperf_CFLAGS = -Wall -Wextra -I./include 36 | examples_qperf_LDADD = ./libhatrack.a 37 | 38 | examples_ring_SOURCES = examples/ring.c 39 | examples_ring_CFLAGS = -Wall -Wextra -I./include 40 | examples_ring_LDADD = ./libhatrack.a 41 | 42 | examples_logringex_SOURCES = examples/logringex.c 43 | examples_logringex_CFLAGS = -Wall -Wextra -I./include 44 | examples_logringex_LDADD = ./libhatrack.a 45 | 46 | examples_array_SOURCES = examples/array.c 47 | examples_array_CFLAGS = -Wall -Wextra -I./include 48 | examples_array_LDADD = ./libhatrack.a 49 | 50 | include_HEADERS = include/hatrack.h 51 | pkginclude_HEADERS = include/hatrack/xxhash.h include/hatrack/ballcap.h include/hatrack/config.h include/hatrack/counters.h include/hatrack/debug.h include/hatrack/gate.h include/hatrack/dict.h include/hatrack/set.h include/hatrack/duncecap.h include/hatrack/hash.h include/hatrack/hatomic.h include/hatrack/hatrack_common.h include/hatrack/hatrack_config.h include/hatrack/hatvtable.h include/hatrack/hihat.h include/hatrack/lohat-a.h include/hatrack/lohat.h include/hatrack/lohat_common.h include/hatrack/mmm.h include/hatrack/newshat.h include/hatrack/oldhat.h include/hatrack/refhat.h include/hatrack/swimcap.h include/hatrack/tophat.h include/hatrack/witchhat.h include/hatrack/woolhat.h include/hatrack/crown.h include/hatrack/tiara.h include/hatrack/queue.h include/hatrack/q64.h include/hatrack/hq.h include/hatrack/capq.h include/hatrack/flexarray.h include/hatrack/llstack.h include/hatrack/stack.h include/hatrack/hatring.h include/hatrack/logring.h include/hatrack/helpmanager.h include/hatrack/vector.h 52 | 53 | test: check 54 | remake: clean all 55 | format: 56 | @PATH_TO_ENV@ clang-format -i examples/*.c test/*.{c, h} src/*.c include/[a-wyz]*.h 57 | -------------------------------------------------------------------------------- /automake.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mkdir -p autotools 4 | aclocal 5 | autoheader 6 | autoconf 7 | automake 8 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.69]) 5 | AC_INIT([hatrack],[0.9],[john@zork.org]) 6 | AC_CONFIG_AUX_DIR([autotools]) 7 | AM_INIT_AUTOMAKE([subdir-objects foreign]) 8 | AM_SILENT_RULES([yes]) 9 | AC_CONFIG_COMMANDS([default],[],[]) 10 | AC_COPYRIGHT([Copyright 2021-2022, John Viega]) 11 | 12 | 13 | AC_LANG([C]) 14 | AC_PROG_CC(clang cc gcc) 15 | 16 | AC_PROG_RANLIB 17 | 18 | AC_CONFIG_SRCDIR([src/hash]) 19 | AC_CONFIG_HEADERS([include/hatrack/config.h]) 20 | AC_CHECK_HEADERS([include/hatrack.h]) 21 | 22 | AC_CHECK_HEADER_STDBOOL 23 | AC_C_INLINE 24 | AC_TYPE_INT64_T 25 | AC_TYPE_SIZE_T 26 | AC_TYPE_UINT32_T 27 | AC_TYPE_UINT64_T 28 | AC_TYPE_UINT8_T 29 | 30 | AC_CACHE_CHECK([for __int128_t], 31 | [ac_cv_i128], 32 | [AC_COMPILE_IFELSE([ 33 | AC_LANG_PROGRAM([__int128_t x;])], 34 | [ac_cv_i128=yes], 35 | [ac_cv_i128=no])]) 36 | 37 | if test $ac_cv_i128 = yes; then 38 | AC_DEFINE([HAVE___INT128_T], [1], [Defined when __int128_t exists]) 39 | fi 40 | 41 | # Autconf doesn't seem to have an option to check for C11 unfortunately :/ 42 | 43 | # Checks for header files. 44 | AC_CHECK_HEADERS([fcntl.h stdint.h unistd.h stdatomic.h pthread.h stdalign.h]) 45 | 46 | 47 | AC_CHECK_LIB([pthread], [pthread_create], [LDFLAGS="${LDFLAGS} -pthread"]) 48 | AC_CHECK_LIB([atomic], [__atomic_load_16], [LDFLAGS="${LDFLAGS} -latomic"]) 49 | 50 | # Checks for C11's typed enums 51 | # AC_MSG_CHECKING([C11 typed enums]) 52 | AC_CACHE_CHECK([for C11 typed enums], 53 | [ac_cv_c11enum], 54 | [AC_COMPILE_IFELSE([ 55 | AC_LANG_PROGRAM([#include 56 | enum : uint64_t { FIRST_ITEM };])], 57 | [ac_cv_c11enum=yes], [ac_cv_c11enum=no])]) 58 | if test $ac_cv_c11enum = yes; then 59 | AC_DEFINE([HAVE_C11_ENUMS], [1], [Defined when C11 enums are present]) 60 | fi 61 | 62 | AC_CACHE_CHECK([to see if atomic_fetch_add(ptr, 1) adds sizeof(ptr)], 63 | [ac_cv_afa_ptr], 64 | [AC_RUN_IFELSE([ 65 | AC_LANG_PROGRAM([[#include 66 | typedef struct s { int a; int b; int c; } foo_t; 67 | _Atomic(foo_t *) x = NULL;]], 68 | [[atomic_fetch_add(&x, 1); 69 | if ((intptr_t)(atomic_load(&x)) == sizeof(foo_t)) { return 0; } 70 | return 1;]] )], 71 | [ac_cv_afa_ptr=yes], 72 | [ac_cv_afa_ptr=no])]) 73 | 74 | if test $ac_cv_afa_ptr = yes; then 75 | AC_DEFINE([SANE_FETCH_ADD_PTR_SEMANTICS], [1], [Defined when atomic_fetch_add(ptr, 1) adds sizeof(ptr)]) 76 | fi 77 | 78 | 79 | # Checks for library functions. 80 | AC_CHECK_FUNCS([clock_gettime memset strstr]) 81 | 82 | AC_PATH_PROG([PATH_TO_ENV], [env], []) 83 | 84 | AC_CONFIG_FILES([Makefile]) 85 | AC_OUTPUT 86 | 87 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Hatrack Documentation 2 | 3 | This page contains documentation both for developers wanting to use the hatrack library in their programs, and for people wanting to understand how the algorithms work (since most of them have some degree of novelty vs the state of the art). 4 | 5 | This API documentation is limited to algorithms that are both compiled into hatrack by default, and intended for general-purpose use. There are algorithms available primarily for comparative testing and/or instructive purposes. While you may see these algorithms explicitly turned on in the examples that do algorithm comparisons, we do not document them in the public API documentation. 6 | 7 | ## API documentation 8 | 9 | 10 | Coming soon. 11 | 12 | ## Algorithm Documentation 13 | - [The core store migration operation](migration.md) 14 | - [Wait-free O(1) dictionaries](dicts.md) 15 | - [Fast, wait-free sets](sets.md) 16 | - [Hatstack](hatstack.md) 17 | - [Faster wait-free queues](queue.md) 18 | - [Ring buffers, including one suitable for large entries](ring.md) 19 | - [Compare-And-Pop queues](capqs.md) -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | This directory contains the following examples for developers: 4 | 5 | 1) *basic* - A simple example of creating two dictionaries, from argv 6 | and envp. 7 | 8 | 2) *set1* - Shows off how to use the hatrack_set class for set operations. 9 | 10 | 3) *hashable* - Shows off a more complex use case, with a higher-level 11 | object type, caching of hash values, etc. 12 | 13 | That's... currently it. 14 | 15 | -------------------------------------------------------------------------------- /examples/array.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: array.c 17 | * Description: Example for flexarrays. 18 | * 19 | * Author: John Viega, john@zork.org 20 | */ 21 | 22 | #include 23 | #include 24 | 25 | /* This starts out with an empty array, and spawns 8 threads. 26 | * 27 | * Each thread counts i from to 1,000,000 inserting it's ID | i into 28 | * the i'th element. 29 | * 30 | * We do this to keep each thread writing something different, but 31 | * when we read the items back at the end (to confirm that they got 32 | * written out), we mask out the thread IDs. 33 | * 34 | * Whenever an array insert operation fails due to an out of bounds 35 | * error, we increase the size of the array by a mere 100 items. 36 | */ 37 | 38 | const int NUM_ITERS = 10000000; 39 | const int NUM_THREADS = 8; 40 | const int GROW_SIZE = 100; 41 | const uint64_t MASK = 0x00000000ffffffff; 42 | 43 | flexarray_t *array; 44 | 45 | static inline void * 46 | get_fill_value(uint64_t i) 47 | { 48 | return (void *)((mmm_mytid << 32) | i); 49 | } 50 | 51 | void * 52 | fill_array(void * unused) 53 | { 54 | (void)unused; 55 | uint64_t i; 56 | 57 | for (i = 0; i < NUM_ITERS; i++) { 58 | while (!flexarray_set(array, i, get_fill_value(i))) { 59 | flexarray_grow(array, array->store->store_size + GROW_SIZE); 60 | } 61 | } 62 | 63 | return NULL; 64 | } 65 | 66 | int64_t 67 | sum_range(int64_t low, int64_t high) 68 | { 69 | int64_t num_items = high - low + 1; 70 | int64_t pair_value = low + high; 71 | 72 | return (pair_value * num_items) >> 1; 73 | } 74 | 75 | int 76 | main(void) 77 | { 78 | pthread_t threads[NUM_THREADS]; 79 | int i; 80 | int status; 81 | int64_t sum1 = sum_range(0, NUM_ITERS - 1); 82 | int64_t sum2 = 0; 83 | uint64_t item; 84 | 85 | array = flexarray_new(0); 86 | 87 | for (i = 0; i < NUM_THREADS; i++) { 88 | pthread_create(&threads[i], NULL, fill_array, NULL); 89 | } 90 | 91 | for (i = 0; i < NUM_THREADS; i++) { 92 | pthread_join(threads[i], NULL); 93 | } 94 | 95 | for (i = 0; i < NUM_ITERS; i++) { 96 | item = (uint64_t)flexarray_get(array, i, &status); 97 | sum2 += (item & MASK); 98 | } 99 | 100 | printf("Expected sum: %ld\n", sum1); 101 | printf("Computed sum: %ld\n", sum2); 102 | 103 | return 0; 104 | } 105 | -------------------------------------------------------------------------------- /examples/basic.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: basic.c 17 | * 18 | * Description: This example creates and uses two dictionaries. 19 | * 20 | * The first is a dictionary where the values are the 21 | * command-line arguments, and the keys are the index 22 | * associated with that argument. 23 | * 24 | * The second is a dictionary containing all environment 25 | * variables passed. 26 | * 27 | * The dictionary containing environment variable 28 | * information dynamically allocates the keys and 29 | * values, and uses a "free handler" to ask our code 30 | * to deallocate whatever needs to be deallocated 31 | * when the items are being ejected (which happens 32 | * when we delete the table, before exit). 33 | * 34 | * Author: John Viega, john@zork.org 35 | */ 36 | 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | static void 43 | print_argv(hatrack_dict_t *argv, bool ordered) 44 | { 45 | hatrack_dict_value_t *values; 46 | uint64_t i, num; 47 | 48 | if (ordered) { 49 | fprintf(stderr, "argv (cmd line order): \n "); 50 | 51 | values = hatrack_dict_values_sort(argv, &num); 52 | } 53 | else { 54 | fprintf(stderr, "argv (hash order): \n "); 55 | values = hatrack_dict_values_nosort(argv, &num); 56 | } 57 | 58 | for (i = 0; i < num; i++) { 59 | fprintf(stderr, "%s ", (char *)values[i]); 60 | } 61 | fprintf(stderr, "\n"); 62 | 63 | return; 64 | } 65 | 66 | static void 67 | print_envp(hatrack_dict_t *argv, bool ordered) 68 | { 69 | hatrack_dict_item_t *items; 70 | uint64_t i, num; 71 | 72 | if (ordered) { 73 | fprintf(stderr, "env (actual order): \n"); 74 | items = hatrack_dict_items_sort(argv, &num); 75 | } 76 | else { 77 | fprintf(stderr, "env (hash order): \n"); 78 | items = hatrack_dict_items_nosort(argv, &num); 79 | } 80 | 81 | for (i = 0; i < num; i++) { 82 | fprintf(stderr, 83 | " %s: %s\n", 84 | (char *)items[i].key, 85 | (char *)items[i].value); 86 | } 87 | fprintf(stderr, "\n"); 88 | 89 | return; 90 | } 91 | 92 | static void 93 | envp_free_handler(hatrack_dict_t *unused, hatrack_dict_item_t *item) 94 | { 95 | fprintf(stderr, 96 | "Freeing: %s: %s\n", 97 | (char *)item->key, 98 | (char *)item->value); 99 | 100 | free(item->key); 101 | free(item->value); 102 | 103 | return; 104 | } 105 | 106 | int 107 | main(int argc, char *argv[], char *envp[]) 108 | { 109 | hatrack_dict_t *argv_dict; 110 | hatrack_dict_t *envp_dict; 111 | uint64_t i; 112 | char *env_eq; // Pointer to the equals mark. 113 | char *env_key; 114 | char *env_val; 115 | char *p; 116 | 117 | argv_dict = hatrack_dict_new(HATRACK_DICT_KEY_TYPE_INT); 118 | envp_dict = hatrack_dict_new(HATRACK_DICT_KEY_TYPE_CSTR); 119 | 120 | hatrack_dict_set_free_handler(envp_dict, 121 | (hatrack_mem_hook_t)envp_free_handler); 122 | 123 | for (i = 0; i < (uint64_t)argc; i++) { 124 | hatrack_dict_put(argv_dict, (void *)i, argv[i]); 125 | } 126 | 127 | i = 0; 128 | 129 | /* Environment variables are of the form KEY=VALUE. When we're 130 | * hashing, the hash function is going to run on the key, but will 131 | * be looking for a null terminator. 132 | * 133 | * We could modify the string in place, replacing the = with a null, 134 | * or we could just stdrup the key (and the value while we're at it) 135 | * to show how to use the hatrack free handler. 136 | */ 137 | while (envp[i]) { 138 | p = envp[i]; 139 | env_eq = strchr(p, '='); 140 | env_key = strndup(p, env_eq - p); 141 | env_val = strdup(env_eq + 1); 142 | 143 | hatrack_dict_put(envp_dict, env_key, env_val); 144 | 145 | i++; 146 | } 147 | 148 | print_envp(envp_dict, false); 149 | print_envp(envp_dict, true); 150 | 151 | fprintf(stderr, "\n"); 152 | 153 | print_argv(argv_dict, false); 154 | print_argv(argv_dict, true); 155 | 156 | hatrack_dict_delete(argv_dict); 157 | hatrack_dict_delete(envp_dict); 158 | 159 | return 0; 160 | } 161 | -------------------------------------------------------------------------------- /examples/logringex.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | logring_t *ring; 5 | 6 | typedef struct { 7 | uint64_t tid; 8 | uint64_t mid; 9 | char msg[112]; 10 | } log_msg_t; 11 | 12 | void * 13 | log_thread(void *item) 14 | { 15 | log_msg_t log; 16 | uint64_t id; 17 | 18 | strcpy(log.msg, "This is a log message!"); 19 | log.tid = (uint64_t)item; 20 | 21 | for (id = 0; id < 512; id++) { 22 | log.mid = id; 23 | logring_enqueue(ring, &log, sizeof(log)); 24 | } 25 | 26 | return NULL; 27 | } 28 | 29 | void 30 | output_logs(void) 31 | { 32 | log_msg_t log; 33 | log_msg_t *msg; 34 | uint64_t len; 35 | logring_view_t *view; 36 | 37 | view = logring_view(ring, false); 38 | 39 | while (logring_dequeue(ring, &log, &len)) { 40 | printf("tid=%lu; mid=%lu; msg=%s\n", log.tid, log.mid, log.msg); 41 | } 42 | 43 | printf("----------------------------\n"); 44 | 45 | while ((msg = (log_msg_t *)logring_view_next(view, &len))) { 46 | printf("tid=%lu; mid=%lu; msg=%s\n", msg->tid, msg->mid, msg->msg); 47 | free(msg); 48 | } 49 | 50 | logring_view_delete(view); 51 | 52 | return; 53 | } 54 | 55 | int 56 | main(void) 57 | { 58 | pthread_t threads[4]; 59 | uint64_t i; 60 | 61 | ring = logring_new(1024, sizeof(log_msg_t)); 62 | 63 | for (i = 0; i < 4; i++) { 64 | pthread_create(&threads[i], NULL, log_thread, (void *)(i + 1)); 65 | } 66 | 67 | for (i = 0; i < 4; i++) { 68 | pthread_join(threads[i], NULL); 69 | } 70 | 71 | output_logs(); 72 | 73 | return 0; 74 | } 75 | -------------------------------------------------------------------------------- /examples/old_queue_ex.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | /* This is not the best test of anything. The workload doesn't feel 6 | * particularly realistic, and if we're looking for best-case timing 7 | * of just the operations, there's a bit of extra cruft in here. 8 | * 9 | * But this is a reasonable proof-of-concept for using it. There's 10 | * a newer queue example, which is a better timing test for any of 11 | * our queues including LIFOs (stacks). 12 | */ 13 | 14 | #define TOTAL_ENQUEUES 10000000 15 | 16 | static double 17 | time_diff(struct timespec *end, struct timespec *start) 18 | { 19 | return ((double)(end->tv_sec - start->tv_sec)) 20 | + ((end->tv_nsec - start->tv_nsec) / 1000000000.0); 21 | } 22 | 23 | static struct timespec stop_times[HATRACK_THREADS_MAX]; 24 | static basic_gate_t starting_gate = 0; 25 | static queue_t *mt_queue; 26 | 27 | static void 28 | clear_timestamps(void) 29 | { 30 | for (int i = 0; i < HATRACK_THREADS_MAX; i++) { 31 | stop_times[i].tv_sec = 0; 32 | stop_times[i].tv_nsec = 0; 33 | } 34 | 35 | return; 36 | } 37 | 38 | void * 39 | multi_threaded_enqueues(void *info) 40 | { 41 | uint64_t max_counter = (uint64_t)info; 42 | uint64_t my_counter = max_counter & 0xffffffff00000000; 43 | 44 | mmm_register_thread(); 45 | 46 | basic_gate_thread_ready(&starting_gate); 47 | 48 | while (my_counter < max_counter) { 49 | my_counter++; 50 | queue_enqueue(mt_queue, (void *)my_counter); 51 | } 52 | 53 | clock_gettime(CLOCK_MONOTONIC, &stop_times[mmm_mytid]); 54 | mmm_clean_up_before_exit(); 55 | 56 | return NULL; 57 | } 58 | 59 | static __thread uint64_t last_dequeue[HATRACK_THREADS_MAX]; 60 | 61 | void * 62 | multi_threaded_dequeues(void *info) 63 | { 64 | uint64_t num_iters = (uint64_t)info; 65 | uint64_t i; 66 | uint64_t res; 67 | uint32_t tid; 68 | 69 | for (i = 0; i < HATRACK_THREADS_MAX; i++) { 70 | last_dequeue[i] = 0; 71 | } 72 | 73 | mmm_register_thread(); 74 | 75 | basic_gate_thread_ready(&starting_gate); 76 | 77 | for (i = 0; i < num_iters; i++) { 78 | res = (uint64_t)queue_dequeue(mt_queue, NULL); 79 | 80 | if (!res) { 81 | continue; 82 | } 83 | 84 | tid = res >> 32; 85 | 86 | if (last_dequeue[tid] >= res) { 87 | abort(); 88 | } 89 | last_dequeue[tid] = res; 90 | } 91 | 92 | clock_gettime(CLOCK_MONOTONIC, &stop_times[mmm_mytid]); 93 | mmm_clean_up_before_exit(); 94 | 95 | return NULL; 96 | } 97 | 98 | void 99 | multi_threaded_v1(int num_threads) 100 | { 101 | int i; 102 | uint64_t num_iters; 103 | uint64_t next_threadid = 1; 104 | pthread_t enqueue_threads[num_threads]; 105 | pthread_t dequeue_threads[num_threads]; 106 | struct timespec start_time; 107 | double cur, min, max; 108 | 109 | mt_queue = queue_new_size(25); 110 | clear_timestamps(); 111 | basic_gate_init(&starting_gate); 112 | 113 | num_iters = TOTAL_ENQUEUES / num_threads; 114 | 115 | for (i = 0; i < num_threads; i++) { 116 | pthread_create(&enqueue_threads[i], 117 | NULL, 118 | multi_threaded_enqueues, 119 | (void *)((next_threadid << 32) | num_iters)); 120 | pthread_create(&dequeue_threads[i], 121 | NULL, 122 | multi_threaded_dequeues, 123 | (void *)num_iters); 124 | next_threadid++; 125 | } 126 | 127 | basic_gate_open(&starting_gate, num_threads * 2, &start_time); 128 | 129 | for (i = 0; i < num_threads; i++) { 130 | pthread_join(enqueue_threads[i], NULL); 131 | pthread_join(dequeue_threads[i], NULL); 132 | } 133 | 134 | min = 0; 135 | max = 0; 136 | 137 | for (i = 0; i < HATRACK_THREADS_MAX; i++) { 138 | if (stop_times[i].tv_sec || stop_times[i].tv_nsec) { 139 | cur = time_diff(&stop_times[i], &start_time); 140 | 141 | if (!min || cur < min) { 142 | min = cur; 143 | } 144 | if (!max || cur > max) { 145 | max = cur; 146 | } 147 | } 148 | } 149 | 150 | fprintf(stderr, 151 | "mt1(%d threads): %.4f sec; Ops/sec: %lu\n", 152 | num_threads, 153 | max, 154 | (unsigned long)((2 * (double)TOTAL_ENQUEUES) / max)); 155 | 156 | return; 157 | } 158 | 159 | void 160 | single_threaded_v1(void) 161 | { 162 | queue_t *queue; 163 | uint64_t i; 164 | double diff; 165 | struct timespec start_time; 166 | struct timespec stop_time; 167 | 168 | queue = queue_new(); 169 | 170 | clock_gettime(CLOCK_MONOTONIC, &start_time); 171 | 172 | for (i = 0; i < TOTAL_ENQUEUES; i++) { 173 | queue_enqueue(queue, (void *)(i + 1)); 174 | } 175 | 176 | for (i = 0; i < TOTAL_ENQUEUES; i++) { 177 | if (((uint64_t)queue_dequeue(queue, NULL)) != (i + 1)) { 178 | abort(); 179 | } 180 | } 181 | 182 | clock_gettime(CLOCK_MONOTONIC, &stop_time); 183 | 184 | diff = time_diff(&stop_time, &start_time); 185 | 186 | printf("Test 1: %.4f sec; Ops/sec: %lu\n", 187 | diff, 188 | (unsigned long)(((double)(TOTAL_ENQUEUES * 2)) / diff)); 189 | 190 | queue_delete(queue); 191 | 192 | return; 193 | } 194 | 195 | void 196 | single_threaded_v2(void) 197 | { 198 | queue_t *queue; 199 | uint64_t i; 200 | double diff; 201 | struct timespec start_time; 202 | struct timespec stop_time; 203 | 204 | queue = queue_new(); 205 | 206 | clock_gettime(CLOCK_MONOTONIC, &start_time); 207 | 208 | for (i = 0; i < TOTAL_ENQUEUES; i++) { 209 | queue_enqueue(queue, (void *)(i + 1)); 210 | if (((uint64_t)queue_dequeue(queue, NULL)) != (i + 1)) { 211 | abort(); 212 | } 213 | } 214 | 215 | clock_gettime(CLOCK_MONOTONIC, &stop_time); 216 | 217 | diff = time_diff(&stop_time, &start_time); 218 | 219 | printf("Test 2: %.4f sec; Ops/sec: %lu\n", 220 | diff, 221 | (unsigned long)(((double)(TOTAL_ENQUEUES * 2)) / diff)); 222 | 223 | queue_delete(queue); 224 | 225 | return; 226 | } 227 | 228 | int 229 | main(void) 230 | { 231 | single_threaded_v1(); 232 | single_threaded_v2(); 233 | multi_threaded_v1(2); 234 | multi_threaded_v1(3); 235 | multi_threaded_v1(4); 236 | multi_threaded_v1(8); 237 | multi_threaded_v1(20); 238 | multi_threaded_v1(100); 239 | 240 | return 0; 241 | } 242 | -------------------------------------------------------------------------------- /examples/ring.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #undef CONSISTENCY_CHECK 5 | 6 | typedef uint64_t thread_params_t[2]; 7 | 8 | typedef struct { 9 | uint64_t start; 10 | uint64_t end; 11 | } thread_info_t; 12 | 13 | static gate_t *gate; 14 | static hatring_t *ring; 15 | static const uint64_t num_ops = 1 << 20; 16 | _Atomic uint64_t finished_enqueuers; 17 | _Atomic uint64_t failed_dequeues; 18 | _Atomic uint64_t successful_dequeues; 19 | _Atomic int64_t enqueue_result; 20 | _Atomic int64_t dequeue_result; 21 | _Atomic int64_t eject_result; 22 | static pthread_t enqueue_threads[HATRACK_THREADS_MAX]; 23 | static pthread_t dequeue_threads[HATRACK_THREADS_MAX]; 24 | 25 | // clang-format off 26 | 27 | static const int ring_test_sizes[] = { 28 | 16, 128, 1024, 4096, 32768, 0 29 | }; 30 | 31 | static const thread_params_t thread_params[] = { 32 | {1, 1}, {2, 2}, {4, 4}, {8, 8}, 33 | {2, 1}, {4, 1}, {8, 1}, 34 | {1, 2}, {1, 4}, {1, 8}, 35 | {0, 0} 36 | }; 37 | 38 | static void 39 | state_reset(void) 40 | { 41 | gate_init (gate, gate->max_threads); 42 | atomic_store(&finished_enqueuers, 0); 43 | atomic_store(&failed_dequeues, 0); 44 | atomic_store(&successful_dequeues, 0); 45 | atomic_store(&enqueue_result, 0); 46 | atomic_store(&dequeue_result, 0); 47 | atomic_store(&eject_result, 0); 48 | 49 | return; 50 | } 51 | 52 | void 53 | handle_eject(void *value) 54 | { 55 | atomic_fetch_add(&eject_result, (uint64_t)value); 56 | } 57 | 58 | void * 59 | enqueue_thread(void *info) 60 | { 61 | thread_info_t *enqueue_info; 62 | uint64_t end; 63 | uint64_t i; 64 | uint64_t sum = 0; 65 | 66 | mmm_register_thread(); 67 | 68 | enqueue_info = (thread_info_t *)info; 69 | end = enqueue_info->end; 70 | 71 | gate_thread_ready(gate); 72 | 73 | for (i = enqueue_info->start; i < end; i++) { 74 | hatring_enqueue(ring, (void *)(i)); 75 | sum += i; 76 | } 77 | 78 | atomic_fetch_add(&finished_enqueuers, 1); 79 | atomic_fetch_add(&enqueue_result, sum); 80 | gate_thread_done(gate); 81 | 82 | free(enqueue_info); 83 | 84 | return NULL; 85 | } 86 | 87 | void * 88 | dequeue_thread(void *info) 89 | { 90 | bool ending = false; 91 | bool status; 92 | uint64_t enqueuers; 93 | uint64_t success; 94 | uint64_t fail; 95 | uint64_t sum; 96 | uint64_t n; 97 | 98 | 99 | enqueuers = (uint64_t)info; 100 | success = 0; 101 | fail = 0; 102 | sum = 0; 103 | 104 | mmm_register_thread(); 105 | 106 | gate_thread_ready(gate); 107 | 108 | while (true) { 109 | n = (uint64_t)hatring_dequeue(ring, &status); 110 | if (!status) { 111 | if (ending) { 112 | break; 113 | } 114 | fail++; 115 | if (atomic_read(&finished_enqueuers) >= enqueuers) { 116 | ending = true; 117 | } 118 | continue; 119 | } 120 | sum += n; 121 | success++; 122 | } 123 | 124 | gate_thread_done(gate); 125 | atomic_fetch_add(&failed_dequeues, fail); 126 | atomic_fetch_add(&successful_dequeues, success); 127 | atomic_fetch_add(&dequeue_result, sum); 128 | mmm_clean_up_before_exit(); 129 | 130 | return NULL; 131 | } 132 | 133 | void 134 | run_one_ring_test(uint64_t enqueuers, uint64_t dequeuers, uint64_t ring_size) 135 | { 136 | uint64_t i; 137 | uint64_t ops_per_thread; 138 | // uint64_t total_ops; 139 | thread_info_t *info; 140 | double max; 141 | 142 | fprintf(stdout, 143 | "#e= %2lu, #d= %2lu, sz= %05lu -> ", 144 | enqueuers, 145 | dequeuers, 146 | ring_size); 147 | fflush(stdout); 148 | 149 | state_reset(); 150 | 151 | ring = hatring_new(ring_size); 152 | ops_per_thread = num_ops / enqueuers; 153 | // total_ops = num_ops * enqueuers; 154 | 155 | #ifdef CONSISTENCY_CHECK 156 | hatring_set_drop_handler(ring, handle_eject); 157 | #endif 158 | 159 | for (i = 0; i < enqueuers; i++) { 160 | info = (thread_info_t *)malloc(sizeof(thread_info_t)); 161 | info->start = (i * ops_per_thread) + 1; 162 | info->end = ((i + 1) * ops_per_thread) + 1; 163 | 164 | pthread_create(&enqueue_threads[i], 165 | NULL, 166 | enqueue_thread, 167 | (void *)info); 168 | } 169 | 170 | for (i = 0; i < dequeuers; i++) { 171 | pthread_create(&dequeue_threads[i], 172 | NULL, 173 | dequeue_thread, 174 | (void *)enqueuers); 175 | } 176 | 177 | gate_open(gate, enqueuers + dequeuers); 178 | 179 | for (i = 0; i < enqueuers; i++) { 180 | pthread_join(enqueue_threads[i], NULL); 181 | } 182 | 183 | for (i = 0; i < dequeuers; i++) { 184 | pthread_join(dequeue_threads[i], NULL); 185 | } 186 | 187 | max = gate_close(gate); 188 | hatring_delete(ring); 189 | 190 | fprintf(stdout, "Qs=%lu; ", num_ops); 191 | fprintf(stdout, "DQs=%lu; ", successful_dequeues); 192 | fprintf(stdout, "(⊥=%lu in ", failed_dequeues); 193 | fprintf(stdout, "%.3f sec ", max); 194 | fprintf(stdout, "(%.3f MOp/s, ", 195 | (((double)(num_ops + successful_dequeues))/1000000.) / max); 196 | fprintf(stdout, "%.3f MOp/s w/ ⊥s)\n", 197 | (((double)(num_ops + successful_dequeues + failed_dequeues 198 | ))/1000000.) / max); 199 | 200 | 201 | #ifdef CONSISTENCY_CHECK 202 | fprintf(stdout, 203 | "pushed value: %lu; dq + eject: %lu; " 204 | "diff: %ld\n", 205 | enqueue_result, 206 | dequeue_result + eject_result, 207 | enqueue_result - (dequeue_result + eject_result) 208 | ); 209 | #endif 210 | 211 | return; 212 | } 213 | 214 | static const char LINE[] 215 | = "-----------------------------------------------------------\n"; 216 | int 217 | main(void) 218 | { 219 | uint64_t i, j; 220 | 221 | gate = gate_new(); 222 | i = 0; 223 | j = 0; 224 | 225 | while (thread_params[i][0]) { 226 | j = 0; 227 | 228 | while (ring_test_sizes[j]) { 229 | run_one_ring_test(thread_params[i][0], 230 | thread_params[i][1], 231 | ring_test_sizes[j++]); 232 | } 233 | printf(LINE); 234 | i++; 235 | } 236 | 237 | return 0; 238 | } 239 | -------------------------------------------------------------------------------- /include/hatrack.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: hatrack.h 17 | * Description: Single header file to pull in all functionality. 18 | * 19 | * Author: John Viega, john@zork.org 20 | */ 21 | 22 | #ifndef __HATRACK_H__ 23 | #define __HATRACK_H__ 24 | 25 | #include 26 | 27 | // Currently pulls in Crown. 28 | #include 29 | 30 | // Currently pulls in Woolhat. 31 | #include 32 | #include 33 | 34 | #ifdef HATRACK_COMPILE_ALL_ALGORITHMS 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #endif 48 | 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | #include 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /include/hatrack/ballcap.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: ballcap.h 17 | * Description: Besides a Lot of Locking, Clearly Awesomely Parallel 18 | * 19 | * Uses pthread locks on a per-bucket basis, and 20 | * allows multiple simultaneous writers, except when 21 | * performing table migration. 22 | * 23 | * Also uses our strategy from Lohat to ensure we can 24 | * provide a fully consistent ordered view of the hash 25 | * table. 26 | * 27 | * This table is based on newshat, but changes the 28 | * record structure to use our approach to managing 29 | * record history that we developed for the lohat 30 | * algorithms (including woolhat, which is an 31 | * incremental improvement on lohat, adding wait 32 | * freedom; something a lock-based algorithm can 33 | * never have). 34 | * 35 | * This hash table is mainly intended to allow us to 36 | * benchmark our lock-free and wait free hash tables 37 | * that use consistent ordering, with a table that 38 | * uses locks, but has the same consistent ordering 39 | * capabilities. And indeed, in my testing so far, 40 | * performance is not appreciably different from the 41 | * lohats (though it's probably better on systems 42 | * without a 128-bit CAS operation). 43 | * 44 | * That is to say, I don't really expect people to 45 | * consider using this table in real-world 46 | * applications (or patterning a table off of it). 47 | * 48 | * Given all the above, the implementation of ballcap 49 | * is mostly uncommented; see implementations of 50 | * either newshat or lohat0 for documentation on the 51 | * algorithm. 52 | * 53 | * Author: John Viega, john@zork.org 54 | */ 55 | 56 | #ifndef __BALLCAP_H__ 57 | #define __BALLCAP_H__ 58 | 59 | #include 60 | 61 | #include 62 | 63 | // clang-format off 64 | typedef struct ballcap_record_st ballcap_record_t; 65 | 66 | /* 67 | * Ballcap is our only locking table with a flag on records to 68 | * indicate whether or not they're deleted. Everything else relies on 69 | * an explicit epoch counter that lives in the data structure being 70 | * set to 0. 71 | * 72 | * However, like the lohat family (including woolhat), records are 73 | * allocated via mmm, and so the epoch field is somewhat transparent 74 | * to us. We could use a macro to extract it quickly, since it's at a 75 | * fixed offset, but space is cheap. All of those implementations 76 | * support a delete field. 77 | * 78 | * For the lock-free variants that do not use mmm (and do not provide 79 | * consistency), we go back to using a 0 epoch to indicate deletion. 80 | * 81 | * 82 | */ 83 | 84 | struct ballcap_record_st { 85 | bool deleted; 86 | void *item; 87 | ballcap_record_t *next; 88 | }; 89 | 90 | typedef struct { 91 | hatrack_hash_t hv; 92 | ballcap_record_t *record; 93 | bool migrated; 94 | pthread_mutex_t mutex; 95 | } ballcap_bucket_t; 96 | 97 | typedef struct { 98 | uint64_t last_slot; 99 | uint64_t threshold; 100 | _Atomic uint64_t used_count; 101 | ballcap_bucket_t buckets[]; 102 | } ballcap_store_t; 103 | 104 | typedef struct { 105 | _Atomic uint64_t item_count; 106 | ballcap_store_t *store_current; 107 | pthread_mutex_t migrate_mutex; 108 | } ballcap_t; 109 | 110 | ballcap_t *ballcap_new (void); 111 | ballcap_t *ballcap_new_size (char); 112 | void ballcap_init (ballcap_t *); 113 | void ballcap_init_size(ballcap_t *, char); 114 | void ballcap_cleanup (ballcap_t *); 115 | void ballcap_delete (ballcap_t *); 116 | void *ballcap_get (ballcap_t *, hatrack_hash_t, bool *); 117 | void *ballcap_put (ballcap_t *, hatrack_hash_t, void *, bool *); 118 | void *ballcap_replace (ballcap_t *, hatrack_hash_t, void *, bool *); 119 | bool ballcap_add (ballcap_t *, hatrack_hash_t, void *); 120 | void *ballcap_remove (ballcap_t *, hatrack_hash_t, bool *); 121 | uint64_t ballcap_len (ballcap_t *); 122 | hatrack_view_t *ballcap_view (ballcap_t *, uint64_t *, bool); 123 | 124 | #endif 125 | -------------------------------------------------------------------------------- /include/hatrack/capq.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: capq.c 17 | * Description: A queue whose primary dequeue operation only 18 | * dequeues if the top value is as expected. 19 | * 20 | * The naive pop() operation on top of cap() retries 21 | * until it succeeds, making that operation 22 | * lock-free. 23 | * 24 | * However, the whole purpose of this queue is to 25 | * support a wait-free help system, where threads 26 | * stick jobs into the queue, and then process items 27 | * until their item has been processed. 28 | * 29 | * The compare-and-pop operation makes sure that 30 | * threads can "help" the top() item, yet, if 31 | * multiple threads try to pop it, only one will 32 | * succeed. Threads in that situation do NOT retry 33 | * the cap, so as long as the enqueue and cap 34 | * operations are wait-free, we're in good shape. 35 | * 36 | * In this queue, the head acts much like hq, in that 37 | * it FAA's, in a ring buffer, and if it catches up 38 | * with the tail, then it resizes the queue. 39 | * 40 | * The tail only updates via CAS. We use the 'epoch' 41 | * as the thing that we compare against, and the tail 42 | * epoch is bumped up by 1<<32 for each migration, 43 | * just to ensure there's never any reuse. 44 | * 45 | * Author: John Viega, john@zork.org 46 | */ 47 | 48 | #ifndef __CAPQ_H__ 49 | #define __CAPQ_H__ 50 | 51 | #include 52 | #include 53 | #include 54 | #include 55 | 56 | 57 | // clang-format off 58 | typedef struct { 59 | void *item; 60 | uint64_t state; 61 | } capq_item_t; 62 | 63 | typedef capq_item_t capq_top_t; 64 | 65 | typedef _Atomic capq_item_t capq_cell_t; 66 | 67 | typedef struct capq_store_t capq_store_t; 68 | 69 | struct capq_store_t { 70 | alignas(8) 71 | _Atomic (capq_store_t *)next_store; 72 | uint64_t size; 73 | _Atomic uint64_t enqueue_index; 74 | _Atomic uint64_t dequeue_index; 75 | capq_cell_t cells[]; 76 | }; 77 | 78 | typedef struct { 79 | alignas(8) 80 | _Atomic (capq_store_t *)store; 81 | _Atomic int64_t len; 82 | } capq_t; 83 | 84 | enum { 85 | CAPQ_EMPTY = 0x0000000000000000, 86 | CAPQ_ENQUEUED = 0x1000000000000000, 87 | CAPQ_DEQUEUED = 0x2000000000000000, 88 | CAPQ_MOVED = 0x4000000000000000, 89 | CAPQ_MOVING = 0x8000000000000000, 90 | CAPQ_FLAG_MASK = 0xf000000000000000, 91 | CAPQ_STORE_INITIALIZING = 0xffffffffffffffff 92 | }; 93 | 94 | static inline int64_t 95 | capq_len(hq_t *self) 96 | { 97 | return atomic_read(&self->len); 98 | } 99 | 100 | capq_t *capq_new (void); 101 | capq_t *capq_new_size (uint64_t); 102 | void capq_init (capq_t *); 103 | void capq_init_size (capq_t *, uint64_t); 104 | void capq_cleanup (capq_t *); 105 | void capq_delete (capq_t *); 106 | uint64_t capq_enqueue (capq_t *, void *); 107 | capq_top_t capq_top (capq_t *, bool *); 108 | bool capq_cap (capq_t *, uint64_t); 109 | void *capq_dequeue (capq_t *, bool *); 110 | 111 | static inline uint64_t 112 | capq_set_enqueued(uint64_t ix) 113 | { 114 | return CAPQ_ENQUEUED | ix; 115 | } 116 | 117 | static inline bool 118 | capq_is_moving(uint64_t state) 119 | { 120 | return state & CAPQ_MOVING; 121 | } 122 | 123 | static inline bool 124 | capq_is_moved(uint64_t state) 125 | { 126 | return state & CAPQ_MOVED; 127 | } 128 | 129 | static inline bool 130 | capq_is_enqueued(uint64_t state) 131 | { 132 | return state & CAPQ_ENQUEUED; 133 | } 134 | 135 | static inline bool 136 | capq_is_dequeued(uint64_t state) 137 | { 138 | return state & CAPQ_DEQUEUED; 139 | } 140 | 141 | static inline uint64_t 142 | capq_extract_epoch(uint64_t state) 143 | { 144 | return state & ~(CAPQ_FLAG_MASK); 145 | } 146 | 147 | static inline uint64_t 148 | capq_ix(uint64_t seq, uint64_t sz) 149 | { 150 | return seq & (sz-1); 151 | } 152 | 153 | static inline uint64_t 154 | capq_set_state_dequeued(uint64_t state) 155 | { 156 | return (state & ~CAPQ_ENQUEUED) | CAPQ_DEQUEUED; 157 | } 158 | 159 | static inline uint64_t 160 | capq_clear_moving(uint64_t state) 161 | { 162 | return state & (~(CAPQ_MOVING|CAPQ_MOVED)); 163 | } 164 | 165 | // Precondition-- we are looking at the right epoch. 166 | static inline bool 167 | capq_should_return(uint64_t state) 168 | { 169 | if (capq_is_enqueued(state) || capq_is_dequeued(state)) { 170 | return true; 171 | } 172 | 173 | return false; 174 | } 175 | 176 | #endif 177 | -------------------------------------------------------------------------------- /include/hatrack/counters.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: counters.h 17 | * Description: In-memory counters for performance monitoring, 18 | * when HATRACK_COUNTERS is on. 19 | * 20 | * When these are off, no counter-related code will 21 | * be generated. For instance, hatomic.h has 22 | * compare-and-swap wrapper macros that can increment 23 | * a Yes / No counter, based on the result, but if 24 | * HATRACK_COUNTERS is not defined, it just compiles 25 | * down to atomic_compare_exchange_strong(). 26 | * 27 | * When they're on, we still do what we can to 28 | * minimize the performance impact, while keeping 29 | * them atomic, using sequentially consistent updates 30 | * (via atomic_fetch_add()), and keeping them in 31 | * statically allocated memory (though, it would be 32 | * better to keep them in dynamic memory, if it's in 33 | * the same cache line as what we're operating on at 34 | * the time). 35 | * 36 | * Right now, I'm primarily using these to monitor 37 | * CAS success and fail rates, in lock free 38 | * algorithms. And they do add about 80% overhead, 39 | * since I'm putting these counters in critical 40 | * sections and creating a lot of unnecessary cache 41 | * misses. 42 | * 43 | * Still, even when monitoring every single CAS and 44 | * more, while performance degrades, almost every 45 | * program under the sun could tolerate keeping these 46 | * on all the time. 47 | * 48 | * 49 | * Author: John Viega, john@zork.org 50 | * 51 | */ 52 | 53 | #ifndef __COUNTERS_H__ 54 | #define __COUNTERS_H__ 55 | 56 | #include 57 | 58 | #include 59 | #include 60 | #include 61 | 62 | #ifdef HATRACK_COUNTERS 63 | extern _Atomic uint64_t hatrack_counters[]; 64 | extern char *hatrack_counter_names[]; 65 | 66 | extern _Atomic uint64_t hatrack_yn_counters[][2]; 67 | extern char *hatrack_yn_counter_names[]; 68 | 69 | enum64(hatrack_counter_names_enum, 70 | HATRACK_CTR_MALLOCS, 71 | HATRACK_CTR_FREES, 72 | HATRACK_CTR_RETIRE_UNUSED, 73 | HATRACK_CTR_STORE_SHRINK, 74 | HATRACK_CTR_WH_HELP_REQUESTS, 75 | HATRACK_CTR_HIa_SLEEP1_WORKED, 76 | HATRACK_CTR_HIa_SLEEP1_FAILED, 77 | HATRACK_CTR_HIa_SLEEP2_WORKED, 78 | HATRACK_CTR_HIa_SLEEP2_FAILED, 79 | HATRACK_COUNTERS_NUM 80 | ); 81 | 82 | enum64(hatrack_yn_counter_names_enum, 83 | HATRACK_CTR_LINEAR_EPOCH_EQ = 0, 84 | HATRACK_CTR_COMMIT = 1, 85 | HATRACK_CTR_COMMIT_HELPS = 2, 86 | LOHAT_CTR_BUCKET_ACQUIRE = 3, 87 | LOHAT_CTR_REC_INSTALL = 4, 88 | LOHAT_CTR_DEL = 5, 89 | LOHAT_CTR_NEW_STORE = 6, 90 | LOHAT_CTR_F_MOVING = 7, 91 | LOHAT_CTR_F_MOVED1 = 8, 92 | LOHAT_CTR_F_MOVED2 = 9, 93 | LOHAT_CTR_MIGRATE_HV = 10, 94 | LOHAT_CTR_MIG_REC = 11, 95 | LOHAT_CTR_F_MOVED3 = 12, 96 | LOHAT_CTR_LEN_INSTALL = 13, 97 | LOHAT_CTR_STORE_INSTALL = 14, 98 | LOHATa_CTR_BUCKET_ACQUIRE = 15, 99 | LOHATa_CTR_PTR_INSTALL = 16, 100 | LOHATa_CTR_HIST_HASH = 17, 101 | LOHATa_CTR_REC_INSTALL = 18, 102 | LOHATa_CTR_DEL = 19, 103 | LOHATa_CTR_NEW_STORE = 20, 104 | LOHATa_CTR_F_MOVING = 21, 105 | LOHATa_CTR_F_MOVED1 = 22, 106 | LOHATa_CTR_F_MOVED2 = 23, 107 | LOHATa_CTR_MIGRATE_HV = 24, 108 | LOHATa_CTR_MIG_REC = 25, 109 | LOHATa_CTR_MV_IH = 26, 110 | LOHATa_CTR_NEW_PTR = 27, 111 | LOHATa_CTR_F_MOVED3 = 28, 112 | LOHATa_CTR_F_HIST = 29, 113 | LOHATa_CTR_STORE_INSTALL = 30, 114 | LOHATb_CTR_BUCKET_ACQUIRE = 31, 115 | LOHATb_CTR_PTR_INSTALL = 32, 116 | LOHATb_CTR_HIST_HASH = 33, 117 | LOHATb_CTR_FWD = 34, 118 | LOHATb_CTR_REC_INSTALL = 35, 119 | LOHATb_CTR_DEL = 36, 120 | LOHATb_CTR_NEW_STORE = 37, 121 | LOHATb_CTR_F_MOVING = 38, 122 | LOHATb_CTR_F_MOVED1 = 39, 123 | LOHATb_CTR_F_MOVED2 = 40, 124 | LOHATb_CTR_MIGRATE_HV = 41, 125 | LOHATb_CTR_MIG_REC = 42, 126 | LOHATb_CTR_MV_IH = 43, 127 | LOHATb_CTR_NEW_PTR = 44, 128 | LOHATb_CTR_F_MOVED3 = 45, 129 | LOHATb_CTR_F_HIST = 46, 130 | LOHATb_CTR_STORE_INSTALL = 47, 131 | HIHAT_CTR_BUCKET_ACQUIRE = 48, 132 | HIHAT_CTR_REC_INSTALL = 49, 133 | HIHAT_CTR_DEL = 50, 134 | HIHAT_CTR_NEW_STORE = 51, 135 | HIHAT_CTR_F_MOVING = 52, 136 | HIHAT_CTR_F_MOVED1 = 53, 137 | HIHAT_CTR_MIGRATE_HV = 54, 138 | HIHAT_CTR_MIG_REC = 55, 139 | HIHAT_CTR_F_MOVED2 = 56, 140 | HIHAT_CTR_LEN_INSTALL = 57, 141 | HIHAT_CTR_STORE_INSTALL = 58, 142 | HIHAT_CTR_SLEEP_NO_JOB = 59, 143 | WITCHHAT_CTR_BUCKET_ACQUIRE = 60, 144 | WITCHHAT_CTR_REC_INSTALL = 61, 145 | WITCHHAT_CTR_DEL = 62, 146 | WITCHHAT_CTR_F_MOVING = 63, 147 | WITCHHAT_CTR_NEW_STORE = 64, 148 | WITCHHAT_CTR_F_MOVED1 = 65, 149 | WITCHHAT_CTR_MIGRATE_HV = 66, 150 | WITCHHAT_CTR_MIG_REC = 67, 151 | WITCHHAT_CTR_F_MOVED2 = 68, 152 | WITCHHAT_CTR_LEN_INSTALL = 69, 153 | WITCHHAT_CTR_STORE_INSTALL = 70, 154 | WOOLHAT_CTR_BUCKET_ACQUIRE = 71, 155 | WOOLHAT_CTR_REC_INSTALL = 72, 156 | WOOLHAT_CTR_DEL = 73, 157 | WOOLHAT_CTR_NEW_STORE = 74, 158 | WOOLHAT_CTR_F_MOVING = 75, 159 | WOOLHAT_CTR_F_MOVED1 = 76, 160 | WOOLHAT_CTR_F_MOVED2 = 77, 161 | WOOLHAT_CTR_MIGRATE_HV = 78, 162 | WOOLHAT_CTR_MIG_REC = 79, 163 | WOOLHAT_CTR_F_MOVED3 = 80, 164 | WOOLHAT_CTR_LEN_INSTALL = 81, 165 | WOOLHAT_CTR_STORE_INSTALL = 82, 166 | HATRACK_YN_COUNTERS_NUM 167 | ); 168 | 169 | static inline _Bool 170 | hatrack_yn_ctr_t(uint64_t id) 171 | { 172 | atomic_fetch_add(&hatrack_yn_counters[id][0], 1); 173 | 174 | return 1; 175 | } 176 | 177 | static inline _Bool 178 | hatrack_yn_ctr_f(uint64_t id) 179 | { 180 | atomic_fetch_add(&hatrack_yn_counters[id][1], 1); 181 | 182 | return 0; 183 | } 184 | 185 | void counters_output_delta(void); 186 | void counters_output_alltime(void); 187 | 188 | #define HATRACK_CTR_ON(id) atomic_fetch_add(&hatrack_counters[id], 1) 189 | #define HATRACK_CTR_OFF(id) 190 | #define HATRACK_YN_ON(x, id) ((x) ? hatrack_yn_ctr_t(id) : hatrack_yn_ctr_f(id)) 191 | #define HATRACK_YN_OFF(x, id) (x) 192 | #define HATRACK_YN_ON_NORET(x, id) \ 193 | ((x) ? hatrack_yn_ctr_t(id) : hatrack_yn_ctr_f(id)) 194 | #define HATRACK_YN_OFF_NORET(x, id) 195 | 196 | #else 197 | 198 | #define HATRACK_CTR_ON(id) 199 | #define HATRACK_CTR_OFF(id) 200 | #define HATRACK_YN_ON(x, id) (x) 201 | #define HATRACK_YN_ON_NORET(x, id) 202 | #define HATRACK_YN_OFF(x, id) (x) 203 | #define HATRACK_YN_OFF_NORET(x, id) 204 | #define counters_output_delta() 205 | #define counters_output_alltime() 206 | 207 | #endif /* defined(HATRACK_COUNTERS) */ 208 | 209 | #define HATRACK_CTR(id) HATRACK_CTR_ON(id) 210 | #define HATRACK_YN_CTR(x, id) HATRACK_YN_ON(x, id) 211 | #define HATRACK_YN_CTR_NORET(x, id) HATRACK_YN_ON_NORET(x, id) 212 | 213 | #endif /* __COUNTERS_H__ */ 214 | -------------------------------------------------------------------------------- /include/hatrack/crown.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: crown.h 17 | * Description: Crown Really Overcomplicates Witchhat Now 18 | * 19 | * Crown is a slight modification of witchhat-- it 20 | * changes the probing strategy for buckets. 21 | * 22 | * Refer to crown.c for implementation notes. 23 | * 24 | * Author: John Viega, john@zork.org 25 | */ 26 | 27 | #ifndef __CROWN_H__ 28 | #define __CROWN_H__ 29 | 30 | #include 31 | 32 | #ifdef HATRACK_32_BIT_HOP_TABLE 33 | 34 | 35 | #define CROWN_HOME_BIT 0x80000000 36 | 37 | typedef uint32_t hop_t; 38 | 39 | #define CLZ(n) __builtin_clzl(n) 40 | 41 | #else 42 | 43 | #define CROWN_HOME_BIT 0x8000000000000000 44 | 45 | typedef uint64_t hop_t; 46 | 47 | #define CLZ(n) __builtin_clzll(n) 48 | 49 | #endif 50 | 51 | typedef struct { 52 | void *item; 53 | uint64_t info; 54 | } crown_record_t; 55 | 56 | enum64(crown_flag_t, 57 | CROWN_F_MOVING = 0x8000000000000000, 58 | CROWN_F_MOVED = 0x4000000000000000, 59 | CROWN_F_INITED = 0x2000000000000000, 60 | CROWN_EPOCH_MASK = 0x1fffffffffffffff); 61 | 62 | typedef struct { 63 | _Atomic hatrack_hash_t hv; 64 | _Atomic crown_record_t record; 65 | 66 | #ifdef HATRACK_32_BIT_HOP_TABLE 67 | _Atomic uint32_t neighbor_map; 68 | #else 69 | _Atomic uint64_t neighbor_map; 70 | #endif 71 | } crown_bucket_t; 72 | 73 | typedef struct crown_store_st crown_store_t; 74 | 75 | // clang-format off 76 | struct crown_store_st { 77 | alignas(8) 78 | uint64_t last_slot; 79 | uint64_t threshold; 80 | _Atomic uint64_t used_count; 81 | _Atomic(crown_store_t *) store_next; 82 | _Atomic bool claimed; 83 | alignas(16) 84 | crown_bucket_t buckets[]; 85 | }; 86 | 87 | typedef struct { 88 | alignas(8) 89 | _Atomic(crown_store_t *) store_current; 90 | _Atomic uint64_t item_count; 91 | _Atomic uint64_t help_needed; 92 | uint64_t next_epoch; 93 | 94 | } crown_t; 95 | 96 | 97 | crown_t *crown_new (void); 98 | crown_t *crown_new_size (char); 99 | void crown_init (crown_t *); 100 | void crown_init_size (crown_t *, char); 101 | void crown_cleanup (crown_t *); 102 | void crown_delete (crown_t *); 103 | void *crown_get (crown_t *, hatrack_hash_t, bool *); 104 | void *crown_put (crown_t *, hatrack_hash_t, void *, bool *); 105 | void *crown_replace (crown_t *, hatrack_hash_t, void *, bool *); 106 | bool crown_add (crown_t *, hatrack_hash_t, void *); 107 | void *crown_remove (crown_t *, hatrack_hash_t, bool *); 108 | uint64_t crown_len (crown_t *); 109 | hatrack_view_t *crown_view (crown_t *, uint64_t *, bool); 110 | hatrack_view_t *crown_view_fast (crown_t *, uint64_t *, bool); 111 | hatrack_view_t *crown_view_slow (crown_t *, uint64_t *, bool); 112 | 113 | /* These need to be non-static because tophat and hatrack_dict both 114 | * need them, so that they can call in without a second call to 115 | * MMM. But, they should be considered "friend" functions, and not 116 | * part of the public API. 117 | */ 118 | crown_store_t *crown_store_new (uint64_t); 119 | void *crown_store_get (crown_store_t *, hatrack_hash_t, bool *); 120 | void *crown_store_put (crown_store_t *, crown_t *, 121 | hatrack_hash_t, void *, bool *, uint64_t); 122 | void *crown_store_replace(crown_store_t *, crown_t *, 123 | hatrack_hash_t, void *, bool *, uint64_t); 124 | bool crown_store_add (crown_store_t *, crown_t *, 125 | hatrack_hash_t, void *, uint64_t); 126 | void *crown_store_remove (crown_store_t *, crown_t *, 127 | hatrack_hash_t, bool *, uint64_t); 128 | 129 | #endif 130 | -------------------------------------------------------------------------------- /include/hatrack/dict.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: dict.h 17 | * Description: Higher level dictionary interface based on crown. 18 | * 19 | * Author: John Viega, john@zork.org 20 | */ 21 | 22 | #ifndef __HATRACK_DICT_H__ 23 | #define __HATRACK_DICT_H__ 24 | 25 | #include 26 | 27 | enum 28 | { 29 | HATRACK_DICT_KEY_TYPE_INT, 30 | HATRACK_DICT_KEY_TYPE_REAL, 31 | HATRACK_DICT_KEY_TYPE_CSTR, 32 | HATRACK_DICT_KEY_TYPE_PTR, 33 | HATRACK_DICT_KEY_TYPE_OBJ_INT, 34 | HATRACK_DICT_KEY_TYPE_OBJ_REAL, 35 | HATRACK_DICT_KEY_TYPE_OBJ_CSTR, 36 | HATRACK_DICT_KEY_TYPE_OBJ_PTR, 37 | HATRACK_DICT_KEY_TYPE_OBJ_CUSTOM 38 | }; 39 | 40 | enum 41 | { 42 | HATRACK_DICT_NO_CACHE = 0xffffffff 43 | }; 44 | 45 | typedef struct { 46 | int32_t hash_offset; 47 | int32_t cache_offset; 48 | } hatrack_offset_info_t; 49 | 50 | typedef struct { 51 | void *key; 52 | void *value; 53 | } hatrack_dict_item_t; 54 | 55 | 56 | typedef struct hatrack_dict_st hatrack_dict_t; 57 | 58 | typedef void *hatrack_dict_key_t; 59 | typedef void *hatrack_dict_value_t; 60 | 61 | typedef union { 62 | hatrack_offset_info_t offsets; 63 | hatrack_hash_func_t custom_hash; 64 | } hatrack_hash_info_t; 65 | 66 | struct hatrack_dict_st { 67 | crown_t crown_instance; 68 | hatrack_hash_info_t hash_info; 69 | hatrack_mem_hook_t free_handler; 70 | hatrack_mem_hook_t key_return_hook; 71 | hatrack_mem_hook_t val_return_hook; 72 | uint32_t key_type; 73 | bool slow_views; 74 | bool sorted_views; 75 | }; 76 | 77 | // clang-format off 78 | hatrack_dict_t *hatrack_dict_new (uint32_t); 79 | void hatrack_dict_init (hatrack_dict_t *, uint32_t); 80 | void hatrack_dict_cleanup(hatrack_dict_t *); 81 | void hatrack_dict_delete (hatrack_dict_t *); 82 | 83 | void hatrack_dict_set_hash_offset (hatrack_dict_t *, int32_t); 84 | void hatrack_dict_set_cache_offset (hatrack_dict_t *, int32_t); 85 | void hatrack_dict_set_custom_hash (hatrack_dict_t *, hatrack_hash_func_t); 86 | void hatrack_dict_set_free_handler (hatrack_dict_t *, hatrack_mem_hook_t); 87 | void hatrack_dict_set_key_return_hook (hatrack_dict_t *, hatrack_mem_hook_t); 88 | void hatrack_dict_set_val_return_hook (hatrack_dict_t *, hatrack_mem_hook_t); 89 | void hatrack_dict_set_consistent_views(hatrack_dict_t *, bool); 90 | void hatrack_dict_set_sorted_views (hatrack_dict_t *, bool); 91 | bool hatrack_dict_get_consistent_views(hatrack_dict_t *); 92 | bool hatrack_dict_get_sorted_views (hatrack_dict_t *); 93 | 94 | void *hatrack_dict_get (hatrack_dict_t *, void *, bool *); 95 | void hatrack_dict_put (hatrack_dict_t *, void *, void *); 96 | bool hatrack_dict_replace(hatrack_dict_t *, void *, void *); 97 | bool hatrack_dict_add (hatrack_dict_t *, void *, void *); 98 | bool hatrack_dict_remove (hatrack_dict_t *, void *); 99 | 100 | hatrack_dict_key_t *hatrack_dict_keys (hatrack_dict_t *, uint64_t *); 101 | hatrack_dict_value_t *hatrack_dict_values (hatrack_dict_t *, uint64_t *); 102 | hatrack_dict_item_t *hatrack_dict_items (hatrack_dict_t *, uint64_t *); 103 | hatrack_dict_key_t *hatrack_dict_keys_sort (hatrack_dict_t *, uint64_t *); 104 | hatrack_dict_value_t *hatrack_dict_values_sort (hatrack_dict_t *, uint64_t *); 105 | hatrack_dict_item_t *hatrack_dict_items_sort (hatrack_dict_t *, uint64_t *); 106 | hatrack_dict_key_t *hatrack_dict_keys_nosort (hatrack_dict_t *, uint64_t *); 107 | hatrack_dict_value_t *hatrack_dict_values_nosort(hatrack_dict_t *, uint64_t *); 108 | hatrack_dict_item_t *hatrack_dict_items_nosort (hatrack_dict_t *, uint64_t *); 109 | 110 | #endif 111 | -------------------------------------------------------------------------------- /include/hatrack/flexarray.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: flexarray.h 17 | * Description: A fast, wait-free flex array. 18 | * 19 | * This ONLY allows indexing and resizing the array. 20 | * If you need append/pop operations in addition, see 21 | * the vector_t type. 22 | * 23 | * Author: John Viega, john@zork.org 24 | */ 25 | 26 | #ifndef __FLEXARRAY_H__ 27 | #define __FLEXARRAY_H__ 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | 35 | #define FLEXARRAY_MIN_STORE_SZ_LOG 4 36 | 37 | // clang-format off 38 | typedef void (*flex_callback_t)(void *); 39 | 40 | typedef struct { 41 | void *item; 42 | uint64_t state; 43 | } flex_item_t; 44 | 45 | typedef _Atomic flex_item_t flex_cell_t; 46 | 47 | typedef struct flex_store_t flex_store_t; 48 | 49 | typedef struct { 50 | uint64_t next_ix; 51 | flex_store_t *contents; 52 | flex_callback_t eject_callback; 53 | } flex_view_t; 54 | 55 | struct flex_store_t { 56 | alignas(8) 57 | uint64_t store_size; 58 | _Atomic uint64_t array_size; 59 | _Atomic (flex_store_t *)next; 60 | _Atomic bool claimed; 61 | flex_cell_t cells[]; 62 | }; 63 | 64 | typedef struct { 65 | flex_callback_t ret_callback; 66 | flex_callback_t eject_callback; 67 | _Atomic (flex_store_t *)store; 68 | } flexarray_t; 69 | 70 | flexarray_t *flexarray_new (uint64_t); 71 | void flexarray_init (flexarray_t *, uint64_t); 72 | void flexarray_set_ret_callback (flexarray_t *, flex_callback_t); 73 | void flexarray_set_eject_callback (flexarray_t *, flex_callback_t); 74 | void flexarray_cleanup (flexarray_t *); 75 | void flexarray_delete (flexarray_t *); 76 | void *flexarray_get (flexarray_t *, uint64_t, int *); 77 | bool flexarray_set (flexarray_t *, uint64_t, void *); 78 | void flexarray_grow (flexarray_t *, uint64_t); 79 | void flexarray_shrink (flexarray_t *, uint64_t); 80 | uint64_t flexarray_len (flexarray_t *); 81 | flex_view_t *flexarray_view (flexarray_t *); 82 | void *flexarray_view_next (flex_view_t *, bool *); 83 | void flexarray_view_delete (flex_view_t *); 84 | void *flexarray_view_get (flex_view_t *, uint64_t, int *); 85 | uint64_t flexarray_view_len (flex_view_t *); 86 | flexarray_t *flexarray_add (flexarray_t *, flexarray_t *); 87 | 88 | enum64(flex_enum_t, 89 | FLEX_ARRAY_SHRINK = 0x8000000000000000, 90 | FLEX_ARRAY_MOVING = 0x4000000000000000, 91 | FLEX_ARRAY_MOVED = 0x2000000000000000, 92 | FLEX_ARRAY_USED = 0x1000000000000000 93 | ); 94 | 95 | enum { 96 | FLEX_OK, 97 | FLEX_OOB, 98 | FLEX_UNINITIALIZED 99 | }; 100 | 101 | #endif 102 | -------------------------------------------------------------------------------- /include/hatrack/gate.h: -------------------------------------------------------------------------------- 1 | /* Copyright © 2021-2022 John Viega 2 | * 3 | * See LICENSE.txt for licensing info. 4 | * 5 | * Name: gate.c 6 | * 7 | * Description: A spin-lock intented primarily to aid in timing 8 | * multi-threaded operations. 9 | * 10 | * The basic idea is that we want to open the 11 | * starting gate only when the worker threads we're 12 | * measuring are in position to start (e.g., they've 13 | * done all their pre-test initialization). 14 | * 15 | * Workers signal with gate_thread_ready(), which 16 | * then spins until the starting gun fires, 17 | * metaphorically speaking. 18 | * 19 | * Meanwhile, the thread management thread calls 20 | * gate_open(), which spins until the requested 21 | * number threads are ready (which should be all 22 | * threads we want to benchmark). 23 | * 24 | * When the worker threads are ready, the manager's 25 | * thread records a timestamp of the starting time, 26 | * and then fires the starting gun by writing the 27 | * value GATE_OPEN to the gate, which causes the 28 | * worker threads to go. 29 | * 30 | * The management thread can then join() on all the 31 | * thread ids in order to sleep until it's time to 32 | * look at the results. 33 | * 34 | * Author: John Viega, john@zork.org 35 | */ 36 | 37 | #ifndef __GATE_H__ 38 | #define __GATE_H__ 39 | 40 | #include 41 | #include 42 | 43 | typedef struct { 44 | _Atomic int64_t count; 45 | uint64_t max_threads; 46 | double elapsed_time; 47 | double fastest_time; 48 | double avg_time; 49 | struct timespec start_time; 50 | struct timespec end_times[]; 51 | } gate_t; 52 | 53 | #define GATE_OPEN 0xffffffffffffffff 54 | 55 | static inline double 56 | gate_time_diff(struct timespec *end, struct timespec *start) 57 | { 58 | return ((double)(end->tv_sec - start->tv_sec)) 59 | + ((end->tv_nsec - start->tv_nsec) / 1000000000.0); 60 | } 61 | 62 | static inline void 63 | gate_init(gate_t *gate, uint64_t max_threads) 64 | { 65 | bzero(gate->end_times, sizeof(struct timespec) * max_threads); 66 | 67 | gate->max_threads = max_threads; 68 | gate->count = 0; 69 | gate->elapsed_time = 0; 70 | 71 | return; 72 | } 73 | 74 | static inline gate_t * 75 | gate_new_size(uint64_t mt) 76 | { 77 | gate_t *ret; 78 | 79 | ret = (gate_t *)malloc(sizeof(gate_t) + sizeof(struct timespec) * mt); 80 | 81 | gate_init(ret, mt); 82 | 83 | return ret; 84 | } 85 | 86 | static inline gate_t * 87 | gate_new(void) 88 | { 89 | return gate_new_size(HATRACK_THREADS_MAX); 90 | } 91 | 92 | static inline void 93 | gate_delete(gate_t *gate) 94 | { 95 | free(gate); 96 | 97 | return; 98 | } 99 | 100 | static inline void 101 | gate_thread_ready(gate_t *gate) 102 | { 103 | atomic_fetch_add(&gate->count, 1); 104 | 105 | while (atomic_read(&gate->count) != (int64_t)GATE_OPEN) 106 | ; 107 | 108 | return; 109 | } 110 | 111 | static inline void 112 | gate_thread_done(gate_t *gate) 113 | { 114 | clock_gettime(CLOCK_MONOTONIC, &gate->end_times[mmm_mytid]); 115 | 116 | return; 117 | } 118 | 119 | static inline void 120 | gate_open(gate_t *gate, int64_t num_threads) 121 | { 122 | while (atomic_read(&gate->count) != num_threads) 123 | ; 124 | 125 | atomic_signal_fence(memory_order_seq_cst); 126 | clock_gettime(CLOCK_MONOTONIC, &gate->start_time); 127 | atomic_signal_fence(memory_order_seq_cst); 128 | 129 | atomic_store(&gate->count, GATE_OPEN); 130 | 131 | return; 132 | } 133 | 134 | static inline double 135 | gate_close(gate_t *gate) 136 | { 137 | uint64_t i, n; 138 | double cur, min, max, total; 139 | 140 | min = 0; 141 | max = 0; 142 | total = 0; 143 | n = 0; 144 | 145 | for (i = 0; i < gate->max_threads; i++) { 146 | if (gate->end_times[i].tv_sec || gate->end_times[i].tv_nsec) { 147 | cur = gate_time_diff(&gate->end_times[i], &gate->start_time); 148 | total += cur; 149 | 150 | n++; 151 | 152 | if (!min || cur < min) { 153 | min = cur; 154 | } 155 | if (!max || cur > max) { 156 | max = cur; 157 | } 158 | } 159 | } 160 | 161 | gate->elapsed_time = max; 162 | gate->fastest_time = min; 163 | gate->avg_time = total / n; 164 | 165 | return max; 166 | } 167 | 168 | static inline double 169 | gate_get_avg(gate_t *gate) 170 | { 171 | return gate->avg_time; 172 | } 173 | 174 | static inline double 175 | gate_get_min(gate_t *gate) 176 | { 177 | return gate->fastest_time; 178 | } 179 | 180 | // Basic gates can be used w/o timing, or can do the start time, and 181 | // then you can handle the rest manually. 182 | typedef _Atomic int64_t basic_gate_t; 183 | 184 | static inline void 185 | basic_gate_init(basic_gate_t *gate) 186 | { 187 | atomic_store(gate, 0); 188 | 189 | return; 190 | } 191 | 192 | static inline void 193 | basic_gate_open(basic_gate_t *gate, 194 | int64_t num_threads, 195 | struct timespec *ts) 196 | { 197 | while (atomic_read(gate) != num_threads) 198 | ; 199 | 200 | atomic_signal_fence(memory_order_seq_cst); 201 | if (ts) { 202 | clock_gettime(CLOCK_MONOTONIC, ts); 203 | } 204 | atomic_signal_fence(memory_order_seq_cst); 205 | atomic_store(gate, -1); 206 | 207 | return; 208 | } 209 | 210 | static inline void 211 | basic_gate_thread_ready(basic_gate_t *gate) { 212 | atomic_fetch_add(gate, 1); 213 | 214 | while (atomic_read(gate) != -1) 215 | ; 216 | 217 | return; 218 | } 219 | 220 | #endif 221 | 222 | 223 | 224 | 225 | 226 | 227 | -------------------------------------------------------------------------------- /include/hatrack/hash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: hash.h 17 | * Description: Hash functions for common data tyes, using the 18 | * third-party XXH3-128 hash function. 19 | * 20 | * Note that these hash functions are not used by the 21 | * core algorithms. Instead, they are used in the 22 | * wrapper in testhat.c, which we use for our test 23 | * harness that dispatches to the algorithm. 24 | * 25 | * Author: John Viega, john@zork.org 26 | */ 27 | 28 | #ifndef __HATRACK_HASH_H__ 29 | #define __HATRACK_HASH_H__ 30 | 31 | #include 32 | #include 33 | 34 | #include 35 | 36 | /* 37 | * I use this union to help the compiler realize that the 128-bit 38 | * result that we get from XXH3_128 is identical to the 128 bits of a 39 | * hatrack_hash_t, without lots of nasty casting. 40 | */ 41 | typedef union { 42 | hatrack_hash_t lhv; 43 | XXH128_hash_t xhv; 44 | } hash_internal_conversion_t; 45 | 46 | static inline hatrack_hash_t 47 | hash_cstr(char *key) 48 | { 49 | hash_internal_conversion_t u; 50 | 51 | u.xhv = XXH3_128bits(key, strlen(key)); 52 | 53 | return u.lhv; 54 | } 55 | 56 | static inline hatrack_hash_t 57 | hash_int(uint64_t key) 58 | { 59 | hash_internal_conversion_t u; 60 | 61 | u.xhv = XXH3_128bits(&key, sizeof(uint64_t)); 62 | 63 | return u.lhv; 64 | } 65 | 66 | static inline hatrack_hash_t 67 | hash_double(double key) 68 | { 69 | hash_internal_conversion_t u; 70 | 71 | u.xhv = XXH3_128bits(&key, sizeof(double)); 72 | 73 | return u.lhv; 74 | } 75 | 76 | static inline hatrack_hash_t 77 | hash_pointer(void *key) 78 | { 79 | hash_internal_conversion_t u; 80 | 81 | u.xhv = XXH3_128bits(&key, sizeof(void *)); 82 | 83 | return u.lhv; 84 | } 85 | 86 | #endif 87 | -------------------------------------------------------------------------------- /include/hatrack/hatomic.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: hatomic.h 17 | * Description: Macros useful for atomicity 18 | * 19 | * Author: John Viega, john@zork.org 20 | */ 21 | 22 | #ifndef __HATOMIC_H__ 23 | #define __HATOMIC_H__ 24 | 25 | #include 26 | 27 | #include 28 | 29 | /* While we don't explicitly discuss it much in the comments of the 30 | * algorithms, most of our reads are agnostic to memory ordering. 31 | * 32 | * We use this macro whenever it doesn't matter if an update is 33 | * happening concurrently, we'll take either version. 34 | * 35 | * We continue to use atomic_load() if we DO want a memory barier on 36 | * the read operation, for instance in our debugging support code. 37 | */ 38 | #define atomic_read(x) atomic_load_explicit(x, memory_order_relaxed) 39 | 40 | /* Most of our writes will need ordering, except when initializing 41 | * variables, which isn't worth worrying about. 42 | * 43 | * Compare-And-Swap is our workhorse for writing, and by default 44 | * provides sequentially consistent memory ordering. 45 | * 46 | * Note that our LCAS macro keeps tally of whether a CAS succeeds or 47 | * fails, as per counters.{h,c}. When HATRACK_COUNTERS is undefined, 48 | * the extra accounting gets compiled out, and we end up with 49 | * an atomic_compare_exchange_strong() only. 50 | */ 51 | 52 | #define CAS(target, expected, desired) \ 53 | atomic_compare_exchange_strong(target, expected, desired) 54 | 55 | #ifdef HATRACK_COUNTERS 56 | #define LCAS_DEBUG(target, expected, desired, id) \ 57 | HATRACK_YN_CTR(CAS(target, expected, desired), id) 58 | #else 59 | #define LCAS_DEBUG(target, expected, desired, id) CAS(target, expected, desired) 60 | #endif 61 | 62 | #define LCAS(target, expected, desired, id) \ 63 | LCAS_DEBUG(target, expected, desired, id) 64 | #define LCAS_SKIP(target, expected, desired, id) CAS(target, expected, desired) 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /include/hatrack/hatring.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: hatring.h 17 | * Description: A wait-free ring buffer. 18 | * 19 | * Note that, after finishing this, I just searched for other 20 | * multi-consumer, multi-producer implementations, and found 21 | * something I had not previously seen (I had searched, but nothing 22 | * reasonable came up on the top page I guess). There's a 2015 paper 23 | * by Steven Feldman et al. that's worth mentioning. 24 | * 25 | * I might implement their algorithm at some point to compare, and it 26 | * takes a similar approach to wait freedom (exponential backoff, 27 | * which is a pretty obvious and easy approach, especially when you 28 | * can't exponentially grow the storage as with other algorithms). 29 | * 30 | * However, from their paper, the thing that's most surprising to me 31 | * is that, for what is allegedly a "ring buffer", enqueues can fail 32 | * if a buffer is full. While I could definitely speed my ring up by 33 | * allowing that, it seems like the antithesis of what ring buffers 34 | * are for... the newest data should be guaranteed an enqueue slot, 35 | * at the expense of dropping older, unread data. 36 | * 37 | * Without that, they haven't produced what I consider a true ring 38 | * buffer-- it's just a fixed-sized FIFO. 39 | * 40 | * Author: John Viega, john@zork.org 41 | */ 42 | 43 | #ifndef __HATRING_H__ 44 | #define __HATRING_H__ 45 | 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | 52 | 53 | typedef struct { 54 | void *item; 55 | uint64_t state; 56 | } hatring_item_t; 57 | 58 | typedef _Atomic hatring_item_t hatring_cell_t; 59 | 60 | typedef void (*hatring_drop_handler)(void *); 61 | 62 | typedef struct { 63 | uint64_t next_ix; 64 | uint64_t num_items; 65 | void *cells[]; 66 | } hatring_view_t; 67 | 68 | typedef struct { 69 | alignas(16) 70 | _Atomic uint64_t epochs; 71 | hatring_drop_handler drop_handler; 72 | uint64_t last_slot; 73 | uint64_t size; 74 | hatring_cell_t cells[]; 75 | } hatring_t; 76 | 77 | enum { 78 | HATRING_ENQUEUED = 0x8000000000000000, 79 | HATRING_DEQUEUED = 0x4000000000000000, 80 | HATRING_MASK = 0xcfffffffffffffff 81 | }; 82 | 83 | static inline bool 84 | hatring_is_lagging(uint32_t read_epoch, uint32_t write_epoch, uint64_t size) 85 | { 86 | if (read_epoch + size < write_epoch) { 87 | return true; 88 | } 89 | 90 | return false; 91 | } 92 | 93 | static inline uint32_t 94 | hatring_enqueue_epoch(uint64_t ptrs) 95 | { 96 | return (uint32_t)(ptrs >> 32); 97 | } 98 | 99 | static inline uint32_t 100 | hatring_dequeue_epoch(uint64_t ptrs) 101 | { 102 | return (uint32_t)(ptrs & 0x00000000ffffffff); 103 | } 104 | 105 | static inline uint32_t 106 | hatring_dequeue_ix(uint64_t epochs, uint32_t last_slot) 107 | { 108 | return (uint32_t)(epochs & last_slot); 109 | } 110 | 111 | static inline uint32_t 112 | hatring_cell_epoch(uint64_t state) 113 | { 114 | return (uint32_t)(state & 0x00000000ffffffff); 115 | } 116 | 117 | static inline bool 118 | hatring_is_enqueued(uint64_t state) { 119 | return state & HATRING_ENQUEUED; 120 | } 121 | 122 | static inline uint64_t 123 | hatring_fixed_epoch(uint32_t write_epoch, uint64_t store_size) 124 | { 125 | return (((uint64_t)write_epoch) << 32) | (write_epoch - store_size); 126 | } 127 | 128 | hatring_t *hatring_new (uint64_t); 129 | void hatring_init (hatring_t *, uint64_t); 130 | void hatring_cleanup (hatring_t *); 131 | void hatring_delete (hatring_t *); 132 | uint32_t hatring_enqueue (hatring_t *, void *); 133 | void *hatring_dequeue (hatring_t *, bool *); 134 | void *hatring_dequeue_w_epoch (hatring_t *, bool *, uint32_t *); 135 | hatring_view_t *hatring_view (hatring_t *); 136 | void *hatring_view_next (hatring_view_t *, bool *); 137 | void hatring_view_delete (hatring_view_t *); 138 | void hatring_set_drop_handler(hatring_t *, hatring_drop_handler); 139 | 140 | #endif 141 | -------------------------------------------------------------------------------- /include/hatrack/hatvtable.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: hatvtable.h 17 | * Description: Support for virtual call tables, used both in the 18 | * tophat algorithm (to switch tables), and in our 19 | * test harness. 20 | * 21 | * Author: John Viega, john@zork.org 22 | */ 23 | 24 | #ifndef __HAT_VTABLE_H__ 25 | #define __HAT_VTABLE_H__ 26 | 27 | #include 28 | 29 | /* For testing, and for our tophat implementation (which switches the 30 | * backend hash table out when it notices multiple writers), we keep 31 | * vtables of the operations to make it easier to switch between 32 | * different algorithms for testing. These types are aliases for the 33 | * methods that we expect to see. 34 | * 35 | * We use void * in the first parameter to all of these methods to 36 | * stand in for an arbitrary pointer to a hash table. 37 | */ 38 | 39 | // clang-format off 40 | typedef void (*hatrack_init_func) (void *); 41 | typedef void (*hatrack_init_sz_func)(void *, char); 42 | typedef void * (*hatrack_get_func) (void *, hatrack_hash_t, bool *); 43 | typedef void * (*hatrack_put_func) (void *, hatrack_hash_t, void *, 44 | bool *); 45 | typedef void * (*hatrack_replace_func)(void *, hatrack_hash_t, void *, 46 | bool *); 47 | typedef bool (*hatrack_add_func) (void *, hatrack_hash_t, void *); 48 | typedef void * (*hatrack_remove_func) (void *, hatrack_hash_t, bool *); 49 | typedef void (*hatrack_delete_func) (void *); 50 | typedef uint64_t (*hatrack_len_func) (void *); 51 | typedef hatrack_view_t *(*hatrack_view_func) (void *, uint64_t *, bool); 52 | 53 | typedef struct { 54 | hatrack_init_func init; 55 | hatrack_init_sz_func init_sz; 56 | hatrack_get_func get; 57 | hatrack_put_func put; 58 | hatrack_replace_func replace; 59 | hatrack_add_func add; 60 | hatrack_remove_func remove; 61 | hatrack_delete_func delete; 62 | hatrack_len_func len; 63 | hatrack_view_func view; 64 | } hatrack_vtable_t; 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /include/hatrack/helpmanager.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: helpmanager.h 17 | * Description: Support for data-structure specific announce arrays, for 18 | * supporting wait freedom. 19 | * 20 | * Our approach to this problem is to fully linearize 21 | * all operations subject to help. The help manager 22 | * has a master help record that dictates the 23 | * operation we're currently working on. 24 | * 25 | * If a thread comes in with work to be done, they 26 | * first work to complete the existing operation. 27 | * 28 | * If they find that some thread has signaled for 29 | * help, they will scan the announce array helping 30 | * all requests, before installing theirs. 31 | * 32 | * If installation fails more than a fixed number of 33 | * times, they enqueue their own help request. 34 | * 35 | * 36 | * Author: John Viega, john@zork.org 37 | */ 38 | 39 | #ifndef __HELPMANAGER_H__ 40 | #define __HELPMANAGER_H__ 41 | 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | 48 | typedef struct { 49 | void *data; 50 | int64_t jobid; 51 | } help_cell_t; 52 | 53 | typedef struct { 54 | uint64_t op; 55 | int64_t jobid; 56 | } help_op_t; 57 | 58 | typedef struct { 59 | uint64_t op; 60 | void *input; 61 | void *aux; 62 | _Atomic help_cell_t success; 63 | _Atomic help_cell_t retval; 64 | } help_record_t; 65 | 66 | typedef _Atomic help_record_t help_record_atomic_t; 67 | 68 | typedef void (*helper_func)(void *, help_record_t *, uint64_t); 69 | 70 | static help_record_t thread_records[HATRACK_THREADS_MAX]; 71 | 72 | typedef struct { 73 | void *parent; 74 | helper_func *vtable; 75 | capq_t capq; 76 | } help_manager_t; 77 | 78 | static inline void * 79 | hatrack_help_get_parent(help_manager_t *manager) 80 | { 81 | return manager->parent; 82 | } 83 | 84 | void hatrack_help_init (help_manager_t *, void *, helper_func *, bool); 85 | void *hatrack_perform_wf_op(help_manager_t *, uint64_t, void *, void *, bool *); 86 | void hatrack_complete_help(help_manager_t *, help_record_t *, int64_t, void *, 87 | bool); 88 | 89 | #endif 90 | -------------------------------------------------------------------------------- /include/hatrack/hq.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: hq.h 17 | * Description: A fast, wait-free queue implementation. 18 | * 19 | * Author: John Viega, john@zork.org 20 | * 21 | * I've decided to build a queue that doesn't use segments; instead, 22 | * it uses a single buffer in a ring, and resizes up if the head 23 | * pointer catches up to the tail pointer. 24 | * 25 | * Once I realized it was possible, it seemed more likely to work well 26 | * than continually allocing and freeing segments. And that does 27 | * indeed seem to be the case. The difference is particularly stark 28 | * when there are few enqueuers, but lots of dequeuers; the "skipping" 29 | * of the wait mechanism slows down the segment queue, because it will 30 | * end up doing a lot more mallocing than this one, which only ever 31 | * doubles the entire queue in size when it needs to grow. 32 | */ 33 | 34 | #ifndef __HQ_H__ 35 | #define __HQ_H__ 36 | 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | 43 | // clang-format off 44 | typedef struct { 45 | void *item; 46 | uint64_t state; 47 | } hq_item_t; 48 | 49 | typedef _Atomic hq_item_t hq_cell_t; 50 | 51 | typedef struct hq_store_t hq_store_t; 52 | 53 | typedef struct { 54 | uint64_t start_epoch; 55 | uint64_t last_epoch; 56 | uint64_t next_ix; 57 | hq_store_t *store; 58 | } hq_view_t; 59 | 60 | struct hq_store_t { 61 | alignas(8) 62 | _Atomic (hq_store_t *)next_store; 63 | uint64_t size; 64 | _Atomic uint64_t enqueue_index; 65 | _Atomic uint64_t dequeue_index; 66 | _Atomic bool claimed; 67 | hq_cell_t cells[]; 68 | }; 69 | 70 | typedef struct { 71 | alignas(8) 72 | _Atomic (hq_store_t *)store; 73 | _Atomic int64_t len; 74 | } hq_t; 75 | 76 | enum { 77 | HQ_EMPTY = 0x0000000000000000, 78 | HQ_TOOSLOW = 0x1000000000000000, 79 | HQ_USED = 0x2000000000000000, 80 | HQ_MOVED = 0x4000000000000000, 81 | HQ_MOVING = 0x8000000000000000, 82 | HQ_FLAG_MASK = 0xf000000000000000, 83 | HQ_STORE_INITIALIZING = 0xffffffffffffffff 84 | }; 85 | 86 | static inline int64_t 87 | hq_len(hq_t *self) 88 | { 89 | return atomic_read(&self->len); 90 | } 91 | 92 | hq_t *hq_new (void); 93 | hq_t *hq_new_size (uint64_t); 94 | void hq_init (hq_t *); 95 | void hq_init_size (hq_t *, uint64_t); 96 | void hq_cleanup (hq_t *); 97 | void hq_delete (hq_t *); 98 | void hq_enqueue (hq_t *, void *); 99 | void *hq_dequeue (hq_t *, bool *); 100 | hq_view_t *hq_view (hq_t *); 101 | void *hq_view_next (hq_view_t *, bool *); 102 | void hq_view_delete(hq_view_t *); 103 | 104 | static inline bool 105 | hq_cell_too_slow(hq_item_t item) 106 | { 107 | return (bool)(item.state & HQ_TOOSLOW); 108 | } 109 | 110 | static inline uint64_t 111 | hq_set_used(uint64_t ix) 112 | { 113 | return HQ_USED | ix; 114 | } 115 | 116 | static inline bool 117 | hq_is_moving(uint64_t state) 118 | { 119 | return state & HQ_MOVING; 120 | } 121 | 122 | static inline bool 123 | hq_is_moved(uint64_t state) 124 | { 125 | return state & HQ_MOVED; 126 | } 127 | 128 | static inline bool 129 | hq_is_queued(uint64_t state) 130 | { 131 | return state & HQ_USED; 132 | } 133 | 134 | static inline uint64_t 135 | hq_add_moving(uint64_t state) 136 | { 137 | return state | HQ_MOVING; 138 | } 139 | 140 | static inline uint64_t 141 | hq_add_moved(uint64_t state) 142 | { 143 | return state | HQ_MOVED | HQ_MOVING; 144 | } 145 | 146 | static inline uint64_t 147 | hq_extract_epoch(uint64_t state) 148 | { 149 | return state & ~(HQ_FLAG_MASK); 150 | } 151 | 152 | static inline bool 153 | hq_can_enqueue(uint64_t state) 154 | { 155 | return !(state & HQ_FLAG_MASK); 156 | } 157 | 158 | static inline uint64_t 159 | hq_ix(uint64_t seq, uint64_t sz) 160 | { 161 | return seq & (sz-1); 162 | } 163 | 164 | #endif 165 | -------------------------------------------------------------------------------- /include/hatrack/llstack.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: llstack.c 17 | * 18 | * Description: A lock-free, linked-list based stack, primarily for 19 | * reference. 20 | * 21 | * Author: John Viega, john@zork.org 22 | */ 23 | 24 | #ifndef __LLSTACK_H__ 25 | #define __LLSTACK_H__ 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | typedef struct llstack_node_t llstack_node_t; 33 | 34 | struct llstack_node_t { 35 | llstack_node_t *next; 36 | void *item; 37 | }; 38 | 39 | typedef struct { 40 | _Atomic (llstack_node_t *) head; 41 | } llstack_t; 42 | 43 | // clang-format off 44 | llstack_t *llstack_new (void); 45 | void llstack_init (llstack_t *); 46 | void llstack_cleanup(llstack_t *); 47 | void llstack_delete (llstack_t *); 48 | void llstack_push (llstack_t *, void *); 49 | void *llstack_pop (llstack_t *, bool *); 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /include/hatrack/lohat_common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: lohat_common.h 17 | * Description: Data structures and constants shared across the 18 | * lohat family. 19 | * 20 | * This is a bit vestigial; most algorithms ended up 21 | * mostly independent file-wise. But since the 22 | * algorithms share a name, I think it's reasonable 23 | * for the shared data structures and constants to 24 | * stay here. 25 | * 26 | * Author: John Viega, john@zork.org 27 | */ 28 | 29 | #ifndef __LOHAT_COMMON_H__ 30 | #define __LOHAT_COMMON_H__ 31 | 32 | #include 33 | 34 | /* We use an "epoch" counter that is incremented with every write 35 | * committment, that gives us an insertion order that we can sort on, 36 | * when proper ordering is desired. We can also use a second array to 37 | * store key/value pairs and index into it from the unordered array. 38 | * When we do that, there will be a natural ordering, but it will be 39 | * the order in which buckets are "reserved" for writing, and not 40 | * (necessarily) the order in which writes were COMMITTED. Generally, 41 | * the write order is more desirable than the committment order, but, 42 | * if we choose to, we can leverage the committment order to speed up 43 | * calculating the write order. Here are some options: 44 | * 45 | * 1) We can just have typical unordered buckets, and only sort the 46 | * contents at a linearization point, when required (i.e., by the 47 | * unique "epoch" associated with the write at the time the write 48 | * was committed). The list is not really being kept in any natual 49 | * "order" per se, but the insertion order is retained, and thus we 50 | * can recover it, even though the expected computational 51 | * complexity of an appropriate sort would be about O(n log n). 52 | * 53 | * 2) We can just care about the bucket reservation order, and call it 54 | * "close enough". 55 | * 56 | * However, in this scenario, the bucket reservation ordering 57 | * drifts from the write committment ordering in multiple 58 | * situations: 59 | * 60 | * a) If two writers are racing, and the writer who reserves 61 | * the first ordered bucket is the last to commit their 62 | * write. 63 | * b) If we DELETE an key/value pair from the table, then use 64 | * the same key for a re-insertion, the second insertion will 65 | * be in the same place in the table as the ORIGINAL item 66 | * (unless the table expanded after deletion, but before 67 | * re-insertion). 68 | * 69 | * 3) We can ignore the write committment order, as with option 2, but 70 | * keep the reservation order closer to it by requiring REINSERT 71 | * operations to get a new bucket reservation, to make the 72 | * reservation ordering more intuitive (and, in most cases with low 73 | * write contention, correct). In such a case, we can co-opt space 74 | * from the deletion record to point to the location of a new 75 | * reservation, IF ANY. 76 | * 77 | * 4) We can use approach 2 for storage, and then, when ordering is 78 | * important, sort the buckets by write committment / epoch. This 79 | * will use up much more space than option #1, and slow down writes 80 | * a little bit, but will drive down the expected time of sort 81 | * operations, when ordering is needed. 82 | * 83 | * The actual complexity will be impacted by the number of deletes, 84 | * but if the number is low, the complexity will approach O(n), the 85 | * same hit we have to pay to copy anyway. 86 | * 87 | * Note that ordering is important mainly when iterating over 88 | * dictionaries, at which point we will copy out the contents of a 89 | * linearized point, and sort that. We will never do an in-place 90 | * sort; it will always involve copying. 91 | * 92 | * 5) We can use approach 3) for storage, and then sort when needed, 93 | * as with #4. Here, the computational complexity of the sort will 94 | * generally be VERY close to O(n), but if there are a lot of 95 | * deletes, we will run out of buckets and need to migrate the 96 | * table much earlier (and more often) than we would have if we 97 | * weren't requiring new bucket reservations for re-inserts. 98 | * 99 | * We implement most of these options (though I skip #2, and have 100 | * temporarily taken out lohat-b which implemented approach #5), and 101 | * do some benchmarking of the tradeoffs. 102 | */ 103 | 104 | typedef struct lohat_record_st lohat_record_t; 105 | 106 | /* Our buckets must keep a "history" that consists of pending commits 107 | * and actual commits that might still be read by a current reader. 108 | * Older commits will be cleaned up automatically, based on epoch data 109 | * hidden in the allocation header. Specifically, the hidden header 110 | * has two fields, one for the commit epoch, and one for the retire 111 | * epoch. When a newer record comes in on top of us, once the newer 112 | * record is committed (meaning, its commit epoch is set), then it will 113 | * change our "retire epoch" to the same value as its commit epoch, 114 | * which we then use to ensure that the record does not have its 115 | * memory reclaimed until all reads that started before its retirement 116 | * epoch have finished their reads. 117 | * 118 | * The non-hidden fields are more or less what you'd expect to 119 | * see... a pointer to the next record associated with the bucket, and 120 | * a pointer the key/value pair (opaquely called item here-- if we are 121 | * using this implementation for sets, the data item might not have a 122 | * value at all. 123 | * 124 | * Note that we will, at table migration time, need to steal the least 125 | * significant two bits of the item pointer to assist with the 126 | * migration. This is discussed in a bit more detail below. 127 | */ 128 | struct lohat_record_st { 129 | bool deleted; 130 | void *item; 131 | lohat_record_t *next; 132 | }; 133 | 134 | /* These two flags are used in table migration. They are implemented 135 | * by stealing two bits from the 'head' field of the record linked 136 | * list (i.e., the value that holds a pointer to the first 137 | * lohat_record_t). This data structure changes slightly with each 138 | * lohat version, but they each use the same flag values. 139 | * 140 | * We *could* instead implement these flags by having the 'head' field 141 | * be 128 bits, a pointer to the top record and then flags. However, I 142 | * explicitly chose NOT to do this, because I want to keep CAS 143 | * operations to 64 bits wherever possible, for the sake of platforms 144 | * without a native 128-bit CAS. That way, on those deficient 145 | * architectures, their only expensive operation is for CASing of the 146 | * 128-bit hash values (specifically, C will generate per-bucket locks 147 | * on the 128-bit hash field, and with each hash write, lock it, 148 | * update the two halves, then unlock it. The penalty comes because 149 | * these locks get used on reads as well, to make sure no readers get 150 | * an inconsistent state). 151 | * 152 | * When the table is large enough that a resize is warranted, we pause 153 | * all writes as quickly as possible, by setting the LOHAT_F_MOVING 154 | * flag in each history bucket. This tells new writers to help migrate 155 | * the table before finishing their write, even if they are not adding 156 | * a new key (i.e., if they only had a modify operation to do). The 157 | * LOHAT_F_MOVED flag is used during the migration to tell other 158 | * threads they don't need to bother trying to migrate a bucket, as 159 | * the migration is already done. 160 | * 161 | * LOHAT_F_MOVED is not strictly necessary. We can let each thread 162 | * that aids in the migration determine on its own that the migration 163 | * of a bucket was successful. However, we use it to avoid unnecessary 164 | * cycles. 165 | * 166 | * Readers can safely ignore either of these flags. Even a late 167 | * arriving reader can ignore them-- let's say a reader shows up, 168 | * reserving an epoch when a table migration is nearly complete, and 169 | * writes are about to start up again. Any writes to the new table 170 | * will always be uninteresting to this reader, because they 171 | * necessarily will have a later epoch than the reader cares 172 | * about... even if the reader gets suspended. 173 | */ 174 | enum64(lohat_flag_t, 175 | LOHAT_F_MOVING = 0x0000000000000001, 176 | LOHAT_F_MOVED = 0x0000000000000002); 177 | 178 | #endif 179 | -------------------------------------------------------------------------------- /include/hatrack/oldhat.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: oldhat.c 17 | * Description: Old, Legacy, Dated Hardware-Acceptable Table 18 | * 19 | * This table stays away from 128-bit compare-and 20 | * swap operations. It does so by keeping all bucket 21 | * information in a single structure, and only ever 22 | * CASing a pointer to said structure. 23 | * 24 | * The net result is we require a lot of dynamic 25 | * memory allocation. 26 | * 27 | * Author: John Viega, john@zork.org 28 | */ 29 | 30 | #ifndef __OLDHAT_H__ 31 | #define __OLDHAT_H__ 32 | 33 | #include 34 | 35 | /* oldhat_record_t 36 | * 37 | * This is the representation of a bucket in oldhat. None of the 38 | * individual elements need to worry about atomicity; the entire 39 | * record is updated at once (by swapping the pointer to the record, 40 | * which is how we limit ourselves to a 64-bit CAS operation. 41 | * 42 | * The individual "hash buckets" only hold a pointer to a record of 43 | * this type; those bucket entries start out as the NULL pointer, and 44 | * as we mutate the table, we swap in oldhat_record_t objects (taking 45 | * care to properly dispose of swapped out objects when we're sure 46 | * that no thread has a reference to those objects). 47 | * 48 | * If we insert something into the table and then delete it, we do NOT 49 | * go back to a null pointer sitting in the bucket. Instead, we point 50 | * to a record that will indicate the bucket is empty... but reserved 51 | * for future re-insertions for items with the same hash value. These 52 | * records DO get cleared out if there's no insertion by the time we 53 | * begin migrating a new set of buckets (usually due to table 54 | * expansion, but sometimes to clean up if we have a lot of deleted 55 | * entries). 56 | * 57 | * We use several boolean fields in this structure that could easily 58 | * be moved into a bitmask, and could even steal those bits from the 59 | * hash value quite safely, if so desired. 60 | * 61 | * 62 | * hv -- The hash value associated with the contents / bucket, 63 | * if any. Note that, in this implementation, unlike all our 64 | * others, the all-zero value does not need to be an 65 | * indiciation that he bucket is empty. We have the "used" 66 | * flag for that. Not that it matters if you select a good 67 | * hash function! 68 | * 69 | * item -- The item passed to the hash table, usually a key : value 70 | * pair of some sort. 71 | * 72 | * moving -- We set this to true to indicate to writers that they 73 | * need to help us migrate the table. 74 | * 75 | * moved -- We set this to true to indicate to other threads helping 76 | * to migrate the table that the bucket in question is 77 | * fully migrated. 78 | * 79 | * used -- We set this to true when there is a value present. 80 | * 81 | */ 82 | // clang-format off 83 | typedef struct { 84 | hatrack_hash_t hv; 85 | void *item; 86 | bool moving; 87 | bool moved; 88 | bool used; 89 | } oldhat_record_t; 90 | 91 | typedef struct oldhat_store_st oldhat_store_t; 92 | 93 | 94 | /* oldhat_store_t 95 | * 96 | * The data type representing our current store. When we need to 97 | * resize or clean out our table, the top-level oldhat_t object will 98 | * stay the same; we instead replace the internal storage (we call 99 | * this migrating the table). 100 | * 101 | * All of our tables use the same metrics for when to perform a table 102 | * migration. We do it when approximately 3/4 of the total number of 103 | * buckets have a RECORD in them, even if that record corresponds to 104 | * an item that was deleted. 105 | * 106 | * We then use a different metric to figure out how big to make the 107 | * next store-- if about 25% of the current buckets (or fewer) have an 108 | * item in it, we will shrink the table size by 50%. If about 50% of 109 | * the current buckets (or more) have an item in it, we will double 110 | * the table size. Otherwise, we will use the same size, and just 111 | * clear out the dead entries, to make room for more inserts. 112 | * 113 | * last_slot -- The array index of the last bucket, so this will be 114 | * one less than the total number of buckets. We store 115 | * it this way, because we're going to use this value 116 | * far more frequently than the total number. 117 | * 118 | * threshold -- We use a simple metric to decide when we need to 119 | * migrate the hash table buckets to a different set of 120 | * buckets-- when an insertion would lead to 75% of the 121 | * buckets in the current table being used. This field 122 | * olds 75% of the total table size. Note that, when 123 | * we actually migrate the buckets, the allocated size 124 | * could grow, shrink or stay the same, depending on 125 | * how many removed items are cluttering up the table. 126 | * 127 | * used_count -- Indicates how many buckets in the table have a hash 128 | * value associated with it. This includes both items 129 | * currently in the table and buckets that are reserved, 130 | * because they have a hash value associated with them, 131 | * but the item has been removed since the last 132 | * resizing. 133 | * 134 | * store_next -- When writer threads realize it's time to migrate, 135 | * they will try to create the next store, if it hasn't 136 | * been put here by the time they read it. Once they 137 | * find the agreed upon store, they all race to migrate. 138 | * Only writers care about this variable, and only during 139 | * migration. 140 | * 141 | * buckets -- The actual bucket objects associated with this store. 142 | * Note that we use a variable-sized array here, and 143 | * dynamically allocate the store to the correct size, 144 | * so that we can avoid an extra indirection. 145 | */ 146 | // clang-format off 147 | struct oldhat_store_st { 148 | alignas(8) 149 | uint64_t last_slot; 150 | uint64_t threshold; 151 | _Atomic uint64_t used_count; 152 | _Atomic(oldhat_store_t *) store_next; 153 | _Atomic(oldhat_record_t *) buckets[]; 154 | }; 155 | 156 | /* oldhat_t 157 | * 158 | * The top-level oldhat object. 159 | * 160 | * store_current -- The current store to use. When we migrate the 161 | * table, this will change at the very end of the 162 | * migration process. Note that some readers might 163 | * still be reading from the old store after the 164 | * migration is completed, so we'll have to be sure 165 | * not to delete it prematurely. 166 | * 167 | * item_count -- The number of items in the table, approximately. 168 | * This value isn't used in anything critical, just 169 | * to return a result when querying the length. 170 | */ 171 | // clang-format off 172 | typedef struct { 173 | alignas(8) 174 | _Atomic(oldhat_store_t *) store_current; 175 | _Atomic uint64_t item_count; 176 | } oldhat_t; 177 | 178 | /* This API requires that you deal with hashing the key external to 179 | * the API. You might want to cache hash values, use different 180 | * functions for different data objects, etc. 181 | * 182 | * We do require 128-bit hash values, and require that the hash value 183 | * alone can stand in for object identity. One might, for instance, 184 | * choose a 3-universal keyed hash function, or if hash values need to 185 | * be consistent across runs, something fast and practical like XXH3. 186 | */ 187 | oldhat_t *oldhat_new (void); 188 | oldhat_t *oldhat_new_size (char); 189 | void oldhat_init (oldhat_t *); 190 | void oldhat_init_size(oldhat_t *, char); 191 | void oldhat_cleanup (oldhat_t *); 192 | void oldhat_delete (oldhat_t *); 193 | void *oldhat_get (oldhat_t *, hatrack_hash_t, bool *); 194 | void *oldhat_put (oldhat_t *, hatrack_hash_t, void *, bool *); 195 | void *oldhat_replace (oldhat_t *, hatrack_hash_t, void *, bool *); 196 | bool oldhat_add (oldhat_t *, hatrack_hash_t, void *); 197 | void *oldhat_remove (oldhat_t *, hatrack_hash_t, bool *); 198 | uint64_t oldhat_len (oldhat_t *); 199 | hatrack_view_t *oldhat_view (oldhat_t *, uint64_t *, bool); 200 | 201 | #endif 202 | -------------------------------------------------------------------------------- /include/hatrack/q64.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: q64.h 17 | * Description: A variant of our wait-free queue for x86 systems 18 | * lacking a 128-bit compare and swap. 19 | * 20 | * Author: John Viega, john@zork.org 21 | * 22 | * In the 128-bit version of the algorithm, fields get a full 64 bits 23 | * for data, and then a state field where most of the state is unused. 24 | * 25 | * In this version, we steal the two low bits for the state we 26 | * need. That means contents must either be pointers, or must fit in 27 | * 62 bits. 28 | * 29 | * In the case of pointers, they will be memory aligned, so stealing 30 | * the two bits is not going to impact anything. For non-pointer 31 | * values though, they should be shifted left a minimum of two bits. 32 | */ 33 | 34 | #ifndef __Q64_H__ 35 | #define __Q64_H__ 36 | 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | 43 | #define QUEUE_HELP_VALUE 1 << QUEUE_HELP_STEPS 44 | 45 | // clang-format off 46 | typedef uint64_t q64_item_t; 47 | typedef _Atomic q64_item_t q64_cell_t; 48 | 49 | typedef struct q64_segment_st q64_segment_t; 50 | 51 | /* If help_needed is non-zero, new segments get the default segement 52 | * size. Otherwise, we double the size of the queue, whatever it is. 53 | * 54 | * Combine this with writers (eventually) exponentially increasing the 55 | * number of cells they jump the counter when their enqueue attempts 56 | * fail, and this is guaranteed to be wait-free. 57 | */ 58 | 59 | struct q64_segment_st { 60 | alignas(64) 61 | _Atomic (q64_segment_t *)next; 62 | uint64_t size; 63 | _Atomic uint64_t enqueue_index; 64 | _Atomic uint64_t dequeue_index; 65 | q64_cell_t cells[]; 66 | }; 67 | 68 | typedef struct { 69 | q64_segment_t *enqueue_segment; 70 | q64_segment_t *dequeue_segment; 71 | } q64_seg_ptrs_t; 72 | 73 | typedef struct { 74 | alignas(16) 75 | _Atomic q64_seg_ptrs_t segments; 76 | uint64_t default_segment_size; 77 | _Atomic uint64_t help_needed; 78 | _Atomic uint64_t len; 79 | } q64_t; 80 | 81 | enum64(q64_cell_state_t, 82 | Q64_EMPTY = 0x00, 83 | Q64_TOOSLOW = 0x01, 84 | Q64_USED = 0x02); 85 | 86 | static inline uint64_t 87 | q64_len(q64_t *self) 88 | { 89 | return atomic_read(&self->len); 90 | } 91 | 92 | q64_t *q64_new (void); 93 | q64_t *q64_new_size (char); 94 | void q64_init (q64_t *); 95 | void q64_init_size(q64_t *, char); 96 | void q64_cleanup (q64_t *); 97 | void q64_delete (q64_t *); 98 | void q64_enqueue (q64_t *, void *); 99 | void *q64_dequeue (q64_t *, bool *); 100 | 101 | #endif 102 | -------------------------------------------------------------------------------- /include/hatrack/queue.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: queue.h 17 | * Description: A fast, wait-free queue implementation. 18 | * 19 | * Author: John Viega, john@zork.org 20 | * 21 | * Before building Hatrack, I'd implemented a version of the Michael 22 | * Scott lock-free queue. 23 | * 24 | * By the time I was done with Hatrack, I knew for sure that it'd be 25 | * easy to make something faster, due to all of the unnecessary 26 | * memory management in the algorithm, since every enqueue requires a 27 | * cell to be seprately malloc'd. 28 | * 29 | * I designed something in my head using a linked list of arrays into 30 | * which items get enqueued. But I looked around before building it, 31 | * and found that my idea was already improved upon in the 32 | * literature, particularly because my initial approach would have 33 | * left enqueues contending with each other. 34 | * 35 | * Morrison / Afek showed how to do a lock-free FIFO without 36 | * contention in the typical case, and Yang / Mellow-Crummey improved 37 | * the progress guarantee to wait-free. 38 | * 39 | * However, I did not like the Yang / Mellow-Crummey approach to 40 | * wait-freedom; the "helping" algorithm came across as overly 41 | * complicated on a number of dimensions. 42 | * 43 | * This queue uses my own mechanism to ensure wait-freedom. 44 | * Basically, the only time there's contention that could lead to 45 | * issues is when queues are nearly empty, such that dequeuers are 46 | * potentially interfering with enqueue operations. 47 | * 48 | * My approach to wait freedom here is twofold: 49 | * 50 | * 1) When an enqueue operation starts failing, the enqueue operation 51 | * attempts to create extra space bewtween the enqueuers and 52 | * dequeuers. It doubles the number it adds to the enqueue index 53 | * every time there's a successive failure. 54 | * 55 | * 2) Additionally, when there are a certain number of fails in a 56 | * row, the enqueuer registers for "help", and deregisters once 57 | * it enqueues successfully. 58 | * 59 | * Whenever "help" is requested, and a new segment is needed, threads 60 | * will double the size of the new segment. 61 | * 62 | * Because we're essentially adding an exponential amount of space 63 | * between the enqueues and the dequeues (who only ever advance 64 | * their index one item at a time), it will require a low, bounded 65 | * number of attempts before a successful enqueue. 66 | * 67 | * Future segments go back to the original size, as long as no 68 | * help is required at the time. 69 | * 70 | * This approach I would expect to be more efficient when helping is 71 | * actually required, and is certainly at least much simpler. 72 | * 73 | * Additionally, we use MMM for memory management, as opposed to the 74 | * custom hazard pointer hybrid. 75 | */ 76 | 77 | #ifndef __QUEUE_H__ 78 | #define __QUEUE_H__ 79 | 80 | #include 81 | #include 82 | #include 83 | #include 84 | 85 | 86 | #define QUEUE_HELP_VALUE 1 << QUEUE_HELP_STEPS 87 | 88 | // clang-format off 89 | typedef struct { 90 | void *item; 91 | uint64_t state; 92 | } queue_item_t; 93 | 94 | typedef _Atomic queue_item_t queue_cell_t; 95 | 96 | typedef struct queue_segment_st queue_segment_t; 97 | 98 | /* If help_needed is non-zero, new segments get the default segement 99 | * size. Otherwise, we double the size of the queue, whatever it is. 100 | * 101 | * Combine this with writers (eventually) exponentially increasing the 102 | * number of cells they jump the counter when their enqueue attempts 103 | * fail, and this is guaranteed to be wait-free. 104 | */ 105 | 106 | struct queue_segment_st { 107 | alignas(64) 108 | _Atomic (queue_segment_t *)next; 109 | uint64_t size; 110 | _Atomic uint64_t enqueue_index; 111 | _Atomic uint64_t dequeue_index; 112 | queue_cell_t cells[]; 113 | }; 114 | 115 | typedef struct { 116 | queue_segment_t *enqueue_segment; 117 | queue_segment_t *dequeue_segment; 118 | } queue_seg_ptrs_t; 119 | 120 | typedef struct { 121 | alignas(16) 122 | _Atomic queue_seg_ptrs_t segments; 123 | uint64_t default_segment_size; 124 | _Atomic uint64_t help_needed; 125 | _Atomic uint64_t len; 126 | } queue_t; 127 | 128 | enum64(queue_cell_state_t, 129 | QUEUE_EMPTY = 0x00, 130 | QUEUE_TOOSLOW = 0x01, 131 | QUEUE_USED = 0x02); 132 | 133 | static inline uint64_t 134 | queue_len(queue_t *self) 135 | { 136 | return atomic_read(&self->len); 137 | } 138 | 139 | queue_t *queue_new (void); 140 | queue_t *queue_new_size (char); 141 | void queue_init (queue_t *); 142 | void queue_init_size(queue_t *, char); 143 | void queue_cleanup (queue_t *); 144 | void queue_delete (queue_t *); 145 | void queue_enqueue (queue_t *, void *); 146 | void *queue_dequeue (queue_t *, bool *); 147 | 148 | #endif 149 | -------------------------------------------------------------------------------- /include/hatrack/refhat.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: refhat.h 17 | * Description: A reference hashtable that only works single-threaded. 18 | * 19 | * Author: John Viega, john@zork.org 20 | */ 21 | 22 | #ifndef __REFHAT_H__ 23 | #define __REFHAT_H__ 24 | 25 | #include 26 | 27 | // clang-format off 28 | 29 | /* refhat_bucket_t 30 | * 31 | * For consistency with our other (parallel) implementations, our 32 | * refhat hash table doesn't move things around the hash table on a 33 | * deletion. Instead, it marks buckets as "deleted". If the same key 34 | * gets reinserted before a table resize, the same bucket will be 35 | * reused. 36 | * 37 | * hv -- The hash value associated with a bucket, if any. 38 | * 39 | * item -- A pointer to the item being stored in the hash table, 40 | * which will generally be a key/value pair in the case 41 | * of dictionaries, or just a single value in the case of 42 | * sets. 43 | * 44 | * deleted -- Set to true if the item has been removed. 45 | * 46 | * epoch -- An indication of insertion time, which we will use to 47 | * sort items in the dictionary, when we produce a "view" 48 | * (views are intended for iteration or set operations). 49 | * The epoch number is chosen relative to other insertions, 50 | * and monotonically increases from 1 (see refhat_t below). 51 | * If an item is already in the table during a write 52 | * operation, where we end up replacing the stored item, 53 | * then the value of the epoch field is NOT updated, to 54 | * keep insertion order sorting consistent with how Python 55 | * does it. 56 | * 57 | * If this value is zero, it indicates there's no item in 58 | * the bucket. 59 | */ 60 | typedef struct { 61 | hatrack_hash_t hv; 62 | void *item; 63 | uint64_t epoch; 64 | } refhat_bucket_t; 65 | 66 | /* refhat_t 67 | * 68 | * The main type for our reference hash table; it contains any 69 | * information that persists across a table resize operation 70 | * (everything else lives in the refhat_bucket_t type). 71 | * 72 | * last_slot -- The array index of the last bucket, so this will be 73 | * one less than the total number of buckets. We store 74 | * it this way, because we're going to use this value 75 | * far more frequently than the total number. 76 | * 77 | * threshold -- We use a simple metric to decide when we need to 78 | * migrate the hash table buckets to a different set of 79 | * buckets-- when an insertion would lead to 75% of the 80 | * buckets in the current table being used. This field 81 | * olds 75% of the total table size. Note that, when 82 | * we actually migrate the buckets, the allocated size 83 | * could grow, shrink or stay the same, depending on 84 | * how many removed items are cluttering up the table. 85 | * 86 | * used_count -- Indicates how many buckets in the table have a hash 87 | * value associated with it. This includes both items 88 | * currently in the table and buckets that are reserved, 89 | * because they have a hash value associated with them, 90 | * but the item has been removed since the last 91 | * resizing. 92 | * 93 | * item_count -- The number of items in the table, NOT counting 94 | * deletion entries. 95 | * 96 | * buckets -- The current set of refhat_bucket_t objects. 97 | * 98 | * next_epoch -- The next epoch value to give to an insertion 99 | * operation, for the purposes of sort ordering. 100 | */ 101 | typedef struct { 102 | alignas(8) 103 | uint64_t last_slot; 104 | uint64_t threshold; 105 | uint64_t used_count; 106 | uint64_t item_count; 107 | refhat_bucket_t *buckets; 108 | uint64_t next_epoch; 109 | } refhat_t; 110 | 111 | 112 | 113 | /* This API requires that you deal with hashing the key external to 114 | * the API. You might want to cache hash values, use different 115 | * functions for different data objects, etc. 116 | * 117 | * We do require 128-bit hash values, and require that the hash value 118 | * alone can stand in for object identity. One might, for instance, 119 | * choose a 3-universal keyed hash function, or if hash values need to 120 | * be consistent across runs, something fast and practical like XXH3. 121 | */ 122 | refhat_t *refhat_new (void); 123 | refhat_t *refhat_new_size (char); 124 | void refhat_init (refhat_t *); 125 | void refhat_init_size(refhat_t *, char); 126 | void refhat_cleanup (refhat_t *); 127 | void refhat_delete (refhat_t *); 128 | void *refhat_get (refhat_t *, hatrack_hash_t, bool *); 129 | void *refhat_put (refhat_t *, hatrack_hash_t, void *, bool *); 130 | void *refhat_replace (refhat_t *, hatrack_hash_t, void *, bool *); 131 | bool refhat_add (refhat_t *, hatrack_hash_t, void *); 132 | void *refhat_remove (refhat_t *, hatrack_hash_t, bool *); 133 | uint64_t refhat_len (refhat_t *); 134 | hatrack_view_t *refhat_view (refhat_t *, uint64_t *, bool); 135 | 136 | //clang-format on 137 | 138 | #endif 139 | -------------------------------------------------------------------------------- /include/hatrack/set.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: set.h 17 | * Description: Higher level set interface based on woolhat. 18 | * 19 | * Author: John Viega, john@zork.org 20 | */ 21 | 22 | #ifndef __HATRACK_SET_H__ 23 | #define __HATRACK_SET_H__ 24 | 25 | #include 26 | #include 27 | 28 | 29 | typedef struct hatrack_set_st hatrack_set_t; 30 | 31 | struct hatrack_set_st { 32 | woolhat_t woolhat_instance; 33 | hatrack_hash_info_t hash_info; 34 | uint32_t item_type; 35 | hatrack_mem_hook_t pre_return_hook; 36 | hatrack_mem_hook_t free_handler; 37 | }; 38 | 39 | 40 | 41 | // clang-format off 42 | hatrack_set_t *hatrack_set_new (uint32_t); 43 | void hatrack_set_init (hatrack_set_t *, uint32_t); 44 | void hatrack_set_cleanup (hatrack_set_t *); 45 | void hatrack_set_delete (hatrack_set_t *); 46 | void hatrack_set_set_hash_offset (hatrack_set_t *, int32_t); 47 | void hatrack_set_set_cache_offset(hatrack_set_t *, int32_t); 48 | void hatrack_set_set_custom_hash (hatrack_set_t *, 49 | hatrack_hash_func_t); 50 | void hatrack_set_set_free_handler(hatrack_set_t *, 51 | hatrack_mem_hook_t); 52 | void hatrack_set_set_return_hook (hatrack_set_t *, 53 | hatrack_mem_hook_t); 54 | bool hatrack_set_contains (hatrack_set_t *, void *); 55 | bool hatrack_set_put (hatrack_set_t *, void *); 56 | bool hatrack_set_add (hatrack_set_t *, void *); 57 | bool hatrack_set_remove (hatrack_set_t *, void *); 58 | void *hatrack_set_items (hatrack_set_t *, uint64_t *); 59 | void *hatrack_set_items_sort (hatrack_set_t *, uint64_t *); 60 | bool hatrack_set_is_eq (hatrack_set_t *, hatrack_set_t *); 61 | bool hatrack_set_is_superset (hatrack_set_t *, hatrack_set_t *, 62 | bool); 63 | bool hatrack_set_is_subset (hatrack_set_t *, hatrack_set_t *, 64 | bool); 65 | bool hatrack_set_is_disjoint (hatrack_set_t *, hatrack_set_t *); 66 | hatrack_set_t *hatrack_set_difference (hatrack_set_t *, hatrack_set_t *); 67 | hatrack_set_t *hatrack_set_union (hatrack_set_t *, hatrack_set_t *); 68 | hatrack_set_t *hatrack_set_intersection (hatrack_set_t *, hatrack_set_t *); 69 | hatrack_set_t *hatrack_set_disjunction (hatrack_set_t *, hatrack_set_t *); 70 | 71 | 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /include/hatrack/stack.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: stack.h 17 | * Description: A faster stack implementation that avoids 18 | * using a linked list node for each item. 19 | * 20 | * We could devise something that is never going to 21 | * copy state when it needs to expand the underlying 22 | * store, breaking the stack up into linked 23 | * segments. For now, I'm not doing that, just to 24 | * keep things as simple as possible. 25 | * 26 | * 27 | * Author: John Viega, john@zork.org 28 | */ 29 | 30 | #ifndef __STACK_H__ 31 | #define __STACK_H__ 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | /* "Valid after" means that, in any epoch after the epoch stored in 39 | * this field, pushers that are assigned that slot are free to try 40 | * to write there. 41 | * 42 | * Slow pushers assigned this slot in or before the listed epoch 43 | * are not allowed to write here. 44 | * 45 | * Similarly, pushes add valid_after to tell (very) poppers whether 46 | * they're allowed to pop the item. As a pusher, if the operation 47 | * happens in epoch n, we'll actually write epoch-1 into the field, so 48 | * that the name "valid after" holds true. 49 | */ 50 | typedef struct { 51 | void *item; 52 | uint32_t state; 53 | uint32_t valid_after; 54 | } stack_item_t; 55 | 56 | typedef _Atomic stack_item_t stack_cell_t; 57 | typedef struct stack_store_t stack_store_t; 58 | 59 | typedef struct { 60 | uint64_t next_ix; 61 | stack_store_t *store; 62 | } stack_view_t; 63 | 64 | struct stack_store_t { 65 | alignas(8) 66 | uint64_t num_cells; 67 | _Atomic uint64_t head_state; 68 | _Atomic (stack_store_t *)next_store; 69 | _Atomic bool claimed; 70 | stack_cell_t cells[]; 71 | }; 72 | 73 | typedef struct { 74 | alignas(8) 75 | _Atomic (stack_store_t *)store; 76 | uint64_t compress_threshold; 77 | 78 | #ifdef HATSTACK_WAIT_FREE 79 | _Atomic int64_t push_help_shift; 80 | #endif 81 | 82 | } hatstack_t; 83 | 84 | 85 | hatstack_t *hatstack_new (uint64_t); 86 | void hatstack_init (hatstack_t *, uint64_t); 87 | void hatstack_cleanup (hatstack_t *); 88 | void hatstack_delete (hatstack_t *); 89 | void hatstack_push (hatstack_t *, void *); 90 | void *hatstack_pop (hatstack_t *, bool *); 91 | stack_view_t *hatstack_view (hatstack_t *); 92 | void *hatstack_view_next (stack_view_t *, bool *); 93 | void hatstack_view_delete(stack_view_t *); 94 | 95 | enum { 96 | HATSTACK_HEAD_MOVE_MASK = 0x80000000ffffffff, 97 | HATSTACK_HEAD_EPOCH_BUMP = 0x0000000100000000, 98 | HATSTACK_HEAD_INDEX_MASK = 0x00000000ffffffff, 99 | HATSTACK_HEAD_EPOCH_MASK = 0x7fffffff00000000, 100 | HATSTACK_HEAD_INITIALIZING = 0xffffffffffffffff, 101 | }; 102 | 103 | static inline bool 104 | head_is_moving(uint64_t n, uint64_t store_size) 105 | { 106 | return (n & HATSTACK_HEAD_INDEX_MASK) >= store_size; 107 | } 108 | 109 | static inline uint32_t 110 | head_get_epoch(uint64_t n) 111 | { 112 | return (n >> 32); 113 | } 114 | 115 | static inline uint32_t 116 | head_get_index(uint64_t n) 117 | { 118 | return n & HATSTACK_HEAD_INDEX_MASK; 119 | } 120 | 121 | static inline uint64_t 122 | head_candidate_new_epoch(uint64_t n, uint32_t ix) 123 | { 124 | return ((n & HATSTACK_HEAD_EPOCH_MASK) | ix) + HATSTACK_HEAD_EPOCH_BUMP; 125 | } 126 | 127 | // These flags / constants are used in stack_item_t's state. 128 | enum { 129 | HATSTACK_PUSHED = 0x00000001, // Cell is full. 130 | HATSTACK_POPPED = 0x00000002, // Cell was full, is empty. 131 | HATSTACK_MOVING = 0x00000004, 132 | HATSTACK_MOVED = 0x00000008 133 | }; 134 | 135 | static inline uint32_t 136 | state_add_moved(uint32_t old) 137 | { 138 | return old | HATSTACK_MOVING | HATSTACK_MOVED; 139 | } 140 | 141 | static inline uint32_t 142 | state_add_moving(uint32_t old) 143 | { 144 | return old | HATSTACK_MOVING; 145 | } 146 | 147 | static inline bool 148 | state_is_pushed(uint32_t state) 149 | { 150 | return (bool)(state & HATSTACK_PUSHED); 151 | } 152 | 153 | static inline bool 154 | state_is_popped(uint32_t state) 155 | { 156 | return (bool)(state & HATSTACK_POPPED); 157 | } 158 | 159 | static inline bool 160 | state_is_moving(uint32_t state) 161 | { 162 | return (bool)(state & HATSTACK_MOVING); 163 | } 164 | 165 | static inline bool 166 | state_is_moved(uint32_t state) 167 | { 168 | return (bool)(state & HATSTACK_MOVED); 169 | } 170 | 171 | static inline bool 172 | cell_can_push(stack_item_t item, uint32_t epoch) 173 | { 174 | if (item.valid_after >= epoch) { 175 | return false; 176 | } 177 | 178 | return true; 179 | } 180 | 181 | #endif 182 | -------------------------------------------------------------------------------- /include/hatrack/tiara.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: tiara.h 17 | * 18 | * Description: This Is A Rediculous Acronym. 19 | * 20 | * This is roughly in the hihat family, but uses 21 | * 64-bit hash values, which we do not generally 22 | * recommend. However, it allows us to show off an 23 | * algorithm that requires only a single 24 | * compare-and-swap per core operation. 25 | * 26 | * There are a few other differences between this and 27 | * the rest of the hihat family: 28 | * 29 | * 1) We do NOT keep an epoch value. It would 30 | * require more space, which defeats the purpose. 31 | * 32 | * 2) We still need two status bits, which normally 33 | * we'd steal from the epoch, or, in the lohat 34 | * family, from a pointer to a dynamically 35 | * allocated record. We've already weakened the 36 | * collision resistance of the hash to an amount 37 | * that I'm not truly comfortable with, so instead 38 | * we steal it from the item field, meaning that 39 | * you CANNOT store integers in here, without 40 | * shifting them up at least two bits. 41 | * 42 | * We obviously could do a lot better if we could CAS 43 | * more at once, and it's not particularly clear to 44 | * me why modern architectures won't just let you do 45 | * atomic loads and CASs on entire cache lines. 46 | * 47 | * Unless there's a good reason, hopefully we'll see 48 | * that happen some day! 49 | * 50 | * Author: John Viega, john@zork.org 51 | */ 52 | 53 | #ifndef __TIARA_H__ 54 | #define __TIARA_H__ 55 | 56 | #include 57 | 58 | enum64(tiara_flag_t, 59 | TIARA_F_MOVING = 0x0000000000000001, 60 | TIARA_F_MOVED = 0x0000000000000002, 61 | TIARA_F_USED = 0x0000000000000004, 62 | TIARA_F_ALL = TIARA_F_MOVING | TIARA_F_MOVED | TIARA_F_USED); 63 | 64 | typedef struct { 65 | uint64_t hv; 66 | void *item; 67 | } tiara_record_t; 68 | 69 | typedef _Atomic(tiara_record_t) tiara_bucket_t; 70 | 71 | typedef struct tiara_store_st tiara_store_t; 72 | 73 | struct tiara_store_st { 74 | alignas(8) uint64_t last_slot; 75 | uint64_t threshold; 76 | _Atomic uint64_t used_count; 77 | _Atomic(tiara_store_t *) store_next; 78 | alignas(16) tiara_bucket_t buckets[]; 79 | }; 80 | 81 | typedef struct { 82 | alignas(8) _Atomic(tiara_store_t *) store_current; 83 | _Atomic uint64_t item_count; 84 | } tiara_t; 85 | 86 | tiara_t *tiara_new(void); 87 | tiara_t *tiara_new_size(char); 88 | void tiara_init(tiara_t *); 89 | void tiara_init_size(tiara_t *, char); 90 | void tiara_cleanup(tiara_t *); 91 | void tiara_delete(tiara_t *); 92 | void *tiara_get(tiara_t *, uint64_t); 93 | void *tiara_put(tiara_t *, uint64_t, void *); 94 | void *tiara_replace(tiara_t *, uint64_t, void *); 95 | bool tiara_add(tiara_t *, uint64_t, void *); 96 | void *tiara_remove(tiara_t *, uint64_t); 97 | uint64_t tiara_len(tiara_t *); 98 | hatrack_view_t *tiara_view(tiara_t *, uint64_t *, bool); 99 | 100 | #endif 101 | -------------------------------------------------------------------------------- /include/hatrack/vector.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: vector.h 17 | * Description: A wait-free vector, complete w/ push/pop/peek. 18 | * 19 | * Author: John Viega, john@zork.org 20 | */ 21 | 22 | #ifndef __VECTOR_H__ 23 | #define __VECTOR_H__ 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | 31 | #define VECTOR_MIN_STORE_SZ_LOG 4 32 | 33 | // clang-format off 34 | typedef void (*vector_callback_t)(void *); 35 | 36 | typedef struct { 37 | void *item; 38 | int64_t state; 39 | } vector_item_t; 40 | 41 | typedef _Atomic vector_item_t vector_cell_t; 42 | 43 | typedef struct vector_store_t vector_store_t; 44 | 45 | typedef struct { 46 | int64_t next_ix; 47 | int64_t size; 48 | vector_store_t *contents; 49 | vector_callback_t eject_callback; 50 | } vector_view_t; 51 | 52 | typedef struct { 53 | int64_t array_size; 54 | int64_t job_id; 55 | } vec_size_info_t; 56 | 57 | struct vector_store_t { 58 | alignas(8) 59 | int64_t store_size; 60 | _Atomic vec_size_info_t array_size_info; 61 | _Atomic (vector_store_t *)next; 62 | _Atomic bool claimed; 63 | vector_cell_t cells[]; 64 | }; 65 | 66 | typedef struct { 67 | vector_callback_t ret_callback; 68 | vector_callback_t eject_callback; 69 | _Atomic (vector_store_t *)store; 70 | help_manager_t help_manager; 71 | } vector_t; 72 | 73 | vector_t *vector_new (int64_t); 74 | void vector_init (vector_t *, int64_t, bool); 75 | void vector_set_ret_callback (vector_t *, vector_callback_t); 76 | void vector_set_eject_callback(vector_t *, vector_callback_t); 77 | void vector_cleanup (vector_t *); 78 | void vector_delete (vector_t *); 79 | void *vector_get (vector_t *, int64_t, int *); 80 | bool vector_set (vector_t *, int64_t, void *); 81 | void vector_grow (vector_t *, int64_t); 82 | void vector_shrink (vector_t *, int64_t); 83 | uint32_t vector_len (vector_t *); 84 | void vector_push (vector_t *, void *); 85 | void *vector_pop (vector_t *, bool *); 86 | void *vector_peek (vector_t *, bool *); 87 | vector_view_t *vector_view (vector_t *); 88 | void *vector_view_next (vector_view_t *, bool *); 89 | void vector_view_delete (vector_view_t *); 90 | 91 | enum { 92 | VECTOR_POPPED = 0x8000000000000000, 93 | VECTOR_USED = 0x4000000000000000, 94 | VECTOR_MOVING = 0x2000000000000000, 95 | VECTOR_MOVED = 0x1000000000000000, 96 | VECTOR_JOB_MASK = 0x0fffffffffffffff 97 | }; 98 | 99 | 100 | enum { 101 | VECTOR_OK, 102 | VECTOR_OOB, 103 | VECTOR_UNINITIALIZED 104 | }; 105 | 106 | enum { 107 | VECTOR_OP_PUSH = 0, 108 | VECTOR_OP_POP, 109 | VECTOR_OP_PEEK, 110 | VECTOR_OP_GROW, 111 | VECTOR_OP_SHRINK, 112 | VECTOR_OP_SLOW_SET, 113 | VECTOR_OP_VIEW 114 | }; 115 | 116 | #endif 117 | -------------------------------------------------------------------------------- /include/hatrack/witchhat.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: witchhat.h 17 | * Description: Waiting I Trully Cannot Handle 18 | * 19 | * This is a lock-free, and wait freehash table, 20 | * without consistency / full ordering. 21 | * 22 | * Note that witchhat is based on hihat1, with a 23 | * helping mechanism in place to ensure wait freedom. 24 | * There are only a few places in hihat1 where we 25 | * need such a mechanism, so we will only comment on 26 | * those places. 27 | * 28 | * Refer to hihat.h and hihat.c for more detail on 29 | * the core algorithm, as here, we only comment on 30 | * the things that are different about witchhat. 31 | * 32 | * Author: John Viega, john@zork.org 33 | */ 34 | 35 | #ifndef __WITCHHAT_H__ 36 | #define __WITCHHAT_H__ 37 | 38 | #include 39 | 40 | typedef struct { 41 | void *item; 42 | uint64_t info; 43 | } witchhat_record_t; 44 | 45 | enum64(witchhat_flag_t, 46 | WITCHHAT_F_MOVING = 0x8000000000000000, 47 | WITCHHAT_F_MOVED = 040000000000000000, 48 | WITCHHAT_F_INITED = 0x2000000000000000, 49 | WITCHHAT_EPOCH_MASK = 0x1fffffffffffffff); 50 | 51 | typedef struct { 52 | _Atomic hatrack_hash_t hv; 53 | _Atomic witchhat_record_t record; 54 | } witchhat_bucket_t; 55 | 56 | typedef struct witchhat_store_st witchhat_store_t; 57 | 58 | // clang-format off 59 | struct witchhat_store_st { 60 | alignas(8) 61 | uint64_t last_slot; 62 | uint64_t threshold; 63 | _Atomic uint64_t used_count; 64 | _Atomic(witchhat_store_t *) store_next; 65 | alignas(16) 66 | witchhat_bucket_t buckets[]; 67 | }; 68 | 69 | typedef struct { 70 | alignas(8) 71 | _Atomic(witchhat_store_t *) store_current; 72 | _Atomic uint64_t item_count; 73 | _Atomic uint64_t help_needed; 74 | uint64_t next_epoch; 75 | 76 | } witchhat_t; 77 | 78 | 79 | witchhat_t *witchhat_new (void); 80 | witchhat_t *witchhat_new_size (char); 81 | void witchhat_init (witchhat_t *); 82 | void witchhat_init_size (witchhat_t *, char); 83 | void witchhat_cleanup (witchhat_t *); 84 | void witchhat_delete (witchhat_t *); 85 | void *witchhat_get (witchhat_t *, hatrack_hash_t, bool *); 86 | void *witchhat_put (witchhat_t *, hatrack_hash_t, void *, 87 | bool *); 88 | void *witchhat_replace (witchhat_t *, hatrack_hash_t, void *, 89 | bool *); 90 | bool witchhat_add (witchhat_t *, hatrack_hash_t, void *); 91 | void *witchhat_remove (witchhat_t *, hatrack_hash_t, bool *); 92 | uint64_t witchhat_len (witchhat_t *); 93 | hatrack_view_t *witchhat_view (witchhat_t *, uint64_t *, bool); 94 | hatrack_view_t *witchhat_view_no_mmm(witchhat_t *, uint64_t *, bool); 95 | 96 | /* These need to be non-static because tophat and hatrack_dict both 97 | * need them, so that they can call in without a second call to 98 | * MMM. But, they should be considered "friend" functions, and not 99 | * part of the public API. 100 | * 101 | * Actually, hatrack_dict no longer uses Witchhat, it uses Crown, but 102 | * I'm going to explicitly leave these here, instead of going back to 103 | * making them static. 104 | */ 105 | witchhat_store_t *witchhat_store_new (uint64_t); 106 | void *witchhat_store_get (witchhat_store_t *, hatrack_hash_t, 107 | bool *); 108 | void *witchhat_store_put (witchhat_store_t *, witchhat_t *, 109 | hatrack_hash_t, void *, bool *, 110 | uint64_t); 111 | void *witchhat_store_replace(witchhat_store_t *, witchhat_t *, 112 | hatrack_hash_t, void *, bool *, 113 | uint64_t); 114 | bool witchhat_store_add (witchhat_store_t *, witchhat_t *, 115 | hatrack_hash_t, void *, uint64_t); 116 | void *witchhat_store_remove (witchhat_store_t *, witchhat_t *, 117 | hatrack_hash_t, bool *, uint64_t); 118 | 119 | #endif 120 | -------------------------------------------------------------------------------- /include/hatrack/woolhat.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: woolhat.h 17 | * Description: Wait-free Operations, Orderable, Linearizable HAsh Table 18 | * This version keeps unordered buckets, and sorts 19 | * by epoch when needed. Views are fully consistent. 20 | * 21 | * Author: John Viega, john@zork.org 22 | */ 23 | 24 | #ifndef __WOOLHAT_H__ 25 | #define __WOOLHAT_H__ 26 | 27 | #include 28 | 29 | typedef struct woolhat_record_st woolhat_record_t; 30 | 31 | struct woolhat_record_st { 32 | woolhat_record_t *next; 33 | void *item; 34 | bool deleted; 35 | }; 36 | 37 | enum { 38 | WOOLHAT_F_MOVING = 0x0000000000000001, 39 | WOOLHAT_F_MOVED = 0x0000000000000002, 40 | WOOLHAT_F_DELETE_HELP = 0x0000000000000004 41 | }; 42 | 43 | // clang-format off 44 | 45 | typedef struct { 46 | woolhat_record_t *head; 47 | uint64_t flags; 48 | } woolhat_state_t; 49 | 50 | typedef struct { 51 | alignas(16) 52 | _Atomic hatrack_hash_t hv; 53 | _Atomic woolhat_state_t state; 54 | } woolhat_history_t; 55 | 56 | typedef struct woolhat_store_st woolhat_store_t; 57 | 58 | struct woolhat_store_st { 59 | alignas(8) 60 | uint64_t last_slot; 61 | uint64_t threshold; 62 | _Atomic uint64_t used_count; 63 | _Atomic(woolhat_store_t *) store_next; 64 | woolhat_history_t hist_buckets[]; 65 | }; 66 | 67 | typedef struct woolhat_st { 68 | alignas(8) 69 | _Atomic(woolhat_store_t *) store_current; 70 | _Atomic uint64_t item_count; 71 | _Atomic uint64_t help_needed; 72 | mmm_cleanup_func cleanup_func; 73 | void *cleanup_aux; 74 | } woolhat_t; 75 | 76 | 77 | /* This is a special type of view result that includes the hash 78 | * value, intended for set operations. Currently, it is only in use 79 | * by woolhat (and by hatrack_set, which is built on woolhat). 80 | */ 81 | 82 | typedef struct { 83 | hatrack_hash_t hv; 84 | void *item; 85 | int64_t sort_epoch; 86 | } hatrack_set_view_t; 87 | 88 | woolhat_t *woolhat_new (void); 89 | woolhat_t *woolhat_new_size (char); 90 | void woolhat_init (woolhat_t *); 91 | void woolhat_init_size (woolhat_t *, char); 92 | void woolhat_cleanup (woolhat_t *); 93 | void woolhat_delete (woolhat_t *); 94 | void woolhat_set_cleanup_func(woolhat_t *, mmm_cleanup_func, void *); 95 | void *woolhat_get (woolhat_t *, hatrack_hash_t, bool *); 96 | void *woolhat_put (woolhat_t *, hatrack_hash_t, void *, 97 | bool *); 98 | void *woolhat_replace (woolhat_t *, hatrack_hash_t, void *, 99 | bool *); 100 | bool woolhat_add (woolhat_t *, hatrack_hash_t, void *); 101 | void *woolhat_remove (woolhat_t *, hatrack_hash_t, bool *); 102 | uint64_t woolhat_len (woolhat_t *); 103 | 104 | hatrack_view_t *woolhat_view (woolhat_t *, uint64_t *, bool); 105 | hatrack_set_view_t *woolhat_view_epoch (woolhat_t *, uint64_t *, uint64_t); 106 | 107 | #endif 108 | -------------------------------------------------------------------------------- /scripts/config-debug: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | 4 | if [ ! -d scripts ] 5 | then 6 | cd .. 7 | if [ ! -d scripts ] 8 | then 9 | echo "Must be run from the scripts directory or the toplevel directory" 10 | exit 11 | fi 12 | fi 13 | ./configure CFLAGS="-O0 -g -DHATRACK_MMM_DEBUG -DHATRACK_COUNTERS" 14 | -------------------------------------------------------------------------------- /scripts/config-opt: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ ! -d scripts ] 4 | then 5 | cd .. 6 | if [ ! -d scripts ] 7 | then 8 | echo "Must be run from the scripts directory or the toplevel directory" 9 | exit 10 | fi 11 | fi 12 | ./configure CFLAGS="-Ofast -flto" 13 | -------------------------------------------------------------------------------- /src/hash/xxhash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * xxHash - Extremely Fast Hash algorithm 3 | * Copyright (C) 2012-2020 Yann Collet 4 | * 5 | * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are 9 | * met: 10 | * 11 | * * Redistributions of source code must retain the above copyright 12 | * notice, this list of conditions and the following disclaimer. 13 | * * Redistributions in binary form must reproduce the above 14 | * copyright notice, this list of conditions and the following disclaimer 15 | * in the documentation and/or other materials provided with the 16 | * distribution. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | * 30 | * You can contact the author at: 31 | * - xxHash homepage: https://www.xxhash.com 32 | * - xxHash source repository: https://github.com/Cyan4973/xxHash 33 | */ 34 | 35 | /* 36 | * xxhash.c instantiates functions defined in xxhash.h 37 | */ 38 | 39 | #include 40 | 41 | #ifdef HATRACK_SKIP_XXH_INLINING 42 | 43 | #define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ 44 | #define XXH_IMPLEMENTATION /* access definitions */ 45 | 46 | #include "xxhash.h" 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /src/queue/debug.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: debug.c 17 | * Description: Debugging via in-memory ring buffer, for use when 18 | * HATRACK_DEBUG is on. 19 | * 20 | * Author: John Viega, john@zork.org 21 | */ 22 | 23 | #include 24 | 25 | #ifdef HATRACK_DEBUG 26 | 27 | #include 28 | 29 | hatrack_debug_record_t __hatrack_debug[HATRACK_DEBUG_RING_SIZE] = {}; 30 | 31 | _Atomic uint64_t __hatrack_debug_sequence = 0; 32 | const char __hatrack_hex_conversion_table[] = "0123456789abcdef"; 33 | 34 | /* debug_dump() 35 | * 36 | * Prints the most recent records in the ring buffer to stderr, up to 37 | * the specified amount. 38 | */ 39 | void 40 | debug_dump(uint64_t max_msgs) 41 | { 42 | int64_t oldest_sequence; 43 | int64_t cur_sequence; 44 | int64_t i; 45 | 46 | if (!max_msgs || max_msgs > HATRACK_DEBUG_RING_SIZE) { 47 | max_msgs = HATRACK_DEBUG_RING_SIZE; 48 | } 49 | 50 | cur_sequence = atomic_load(&__hatrack_debug_sequence); 51 | oldest_sequence = cur_sequence - max_msgs; 52 | 53 | if (oldest_sequence < 0) { 54 | oldest_sequence = 0; 55 | } 56 | 57 | oldest_sequence &= HATRACK_DEBUG_RING_LAST_SLOT; 58 | cur_sequence &= HATRACK_DEBUG_RING_LAST_SLOT; 59 | 60 | if (oldest_sequence >= cur_sequence) { 61 | for (i = oldest_sequence; i < HATRACK_DEBUG_RING_SIZE; i++) { 62 | fprintf(stderr, 63 | "%06llu: (tid %ld) %s\n", 64 | (unsigned long long)__hatrack_debug[i].sequence, 65 | (long)__hatrack_debug[i].thread, 66 | __hatrack_debug[i].msg); 67 | } 68 | 69 | i = 0; 70 | } 71 | else { 72 | i = oldest_sequence; 73 | } 74 | 75 | for (; i < cur_sequence; i++) { 76 | fprintf(stderr, 77 | "%06llu: (tid %ld) %s\n", 78 | (unsigned long long)__hatrack_debug[i].sequence, 79 | (long)__hatrack_debug[i].thread, 80 | __hatrack_debug[i].msg); 81 | } 82 | 83 | return; 84 | } 85 | 86 | /* debug_thread() 87 | * 88 | * Prints (to stderr) all messages current in the ring buffer that 89 | * were written by the current thread. 90 | */ 91 | void 92 | debug_thread(void) 93 | { 94 | debug_other_thread(mmm_mytid); 95 | 96 | return; 97 | } 98 | 99 | /* debug_thread() 100 | * 101 | * Prints (to stderr) all messages current in the ring buffer that 102 | * were written by the thread with the given id. Note that we use the 103 | * thread IDs assigned by mmm for the purposes of thread identification. 104 | */ 105 | void 106 | debug_other_thread(int64_t tid) 107 | { 108 | int64_t start; 109 | int64_t i; 110 | 111 | start = atomic_load(&__hatrack_debug_sequence); 112 | start &= HATRACK_DEBUG_RING_LAST_SLOT; 113 | 114 | for (i = start; i < HATRACK_DEBUG_RING_SIZE; i++) { 115 | if (tid == __hatrack_debug[i].thread) { 116 | fprintf(stderr, 117 | "%06llu: (tid %ld) %s\n", 118 | (unsigned long long)__hatrack_debug[i].sequence, 119 | (long)__hatrack_debug[i].thread, 120 | __hatrack_debug[i].msg); 121 | } 122 | } 123 | 124 | for (i = 0; i < start; i++) { 125 | if (tid == __hatrack_debug[i].thread) { 126 | fprintf(stderr, 127 | "%06llu: (tid %ld) %s\n", 128 | (unsigned long long)__hatrack_debug[i].sequence, 129 | (long)__hatrack_debug[i].thread, 130 | __hatrack_debug[i].msg); 131 | } 132 | } 133 | 134 | return; 135 | } 136 | 137 | /* debug_grep() 138 | * 139 | * Okay, this doesn't really "grep", but it searches the message field 140 | * of all ring buffer entries (from oldest to newest), looking for the 141 | * given substring. 142 | * 143 | * The searching is implemented using strstr, so definitely no regexps 144 | * work. 145 | */ 146 | void 147 | debug_grep(char *s) 148 | { 149 | int64_t start; 150 | int64_t i; 151 | 152 | start = atomic_load(&__hatrack_debug_sequence); 153 | start &= HATRACK_DEBUG_RING_LAST_SLOT; 154 | 155 | for (i = start; i < HATRACK_DEBUG_RING_SIZE; i++) { 156 | if (strstr(__hatrack_debug[i].msg, s)) { 157 | fprintf(stderr, 158 | "%06llu: (tid %ld) %s\n", 159 | (unsigned long long)__hatrack_debug[i].sequence, 160 | (long)__hatrack_debug[i].thread, 161 | __hatrack_debug[i].msg); 162 | } 163 | } 164 | 165 | for (i = 0; i < start; i++) { 166 | if (strstr(__hatrack_debug[i].msg, s)) { 167 | fprintf(stderr, 168 | "%06llu: (tid %ld) %s\n", 169 | (unsigned long long)__hatrack_debug[i].sequence, 170 | (long)__hatrack_debug[i].thread, 171 | __hatrack_debug[i].msg); 172 | } 173 | } 174 | 175 | return; 176 | } 177 | 178 | /* debug_pgrep() 179 | * 180 | * If you pass in a pointer's value as an integer, constructs the 181 | * appropriate string for that pointer, and then calls debug_grep() 182 | * for you. 183 | */ 184 | void 185 | debug_pgrep(uintptr_t n) 186 | { 187 | char s[HATRACK_PTR_CHRS + 1] = {}; 188 | char *p = s + HATRACK_PTR_CHRS; 189 | int64_t i; 190 | 191 | for (i = 0; i < HATRACK_PTR_CHRS; i++) { 192 | *--p = __hatrack_hex_conversion_table[n & 0xf]; 193 | n >>= 4; 194 | } 195 | 196 | debug_grep(s); 197 | 198 | return; 199 | } 200 | 201 | #endif 202 | -------------------------------------------------------------------------------- /src/queue/llstack.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: llstack.c 17 | * Description: A lock-free, linked-list based stack, primarily for 18 | * reference. 19 | * 20 | * This is basically the "classic" lock-free 21 | * construction, except that we do not need to have 22 | * an ABA field, due to our use of MMM. 23 | * 24 | * Author: John Viega, john@zork.org 25 | */ 26 | 27 | #include 28 | 29 | llstack_t * 30 | llstack_new(void) 31 | { 32 | llstack_t *ret; 33 | 34 | ret = (llstack_t *)malloc(sizeof(llstack_t)); 35 | 36 | llstack_init(ret); 37 | 38 | return ret; 39 | } 40 | 41 | void 42 | llstack_init(llstack_t *self) 43 | { 44 | atomic_store(&self->head, NULL); 45 | } 46 | 47 | /* You're better off emptying the stack manually to do memory management 48 | * on the contents. But if you didn't, we'll still clean up the records 49 | * we allocated, at least! 50 | */ 51 | void 52 | llstack_cleanup(llstack_t *self) 53 | { 54 | bool found; 55 | 56 | do { 57 | llstack_pop(self, &found); 58 | } while(found); 59 | 60 | return; 61 | } 62 | 63 | void 64 | llstack_delete(llstack_t *self) 65 | { 66 | llstack_cleanup(self); 67 | free(self); 68 | 69 | return; 70 | } 71 | 72 | void 73 | llstack_push(llstack_t *self, void *item) 74 | { 75 | llstack_node_t *node; 76 | llstack_node_t *head; 77 | 78 | mmm_start_basic_op(); 79 | 80 | node = mmm_alloc_committed(sizeof(llstack_node_t)); 81 | head = atomic_read(&self->head); 82 | node->item = item; 83 | 84 | do { 85 | node->next = head; 86 | } while (!CAS(&self->head, &head, node)); 87 | 88 | mmm_end_op(); 89 | 90 | return; 91 | } 92 | 93 | void * 94 | llstack_pop(llstack_t *self, bool *found) 95 | { 96 | llstack_node_t *old_head; 97 | void *ret; 98 | 99 | mmm_start_basic_op(); 100 | 101 | old_head = atomic_read(&self->head); 102 | 103 | while (old_head && !CAS(&self->head, &old_head, old_head->next)); 104 | 105 | if (!old_head) { 106 | if (found) { 107 | *found = false; 108 | } 109 | 110 | mmm_end_op(); 111 | return NULL; 112 | } 113 | 114 | if (found) { 115 | *found = true; 116 | } 117 | 118 | ret = old_head->item; 119 | 120 | mmm_retire(old_head); 121 | mmm_end_op(); 122 | 123 | return ret; 124 | } 125 | -------------------------------------------------------------------------------- /src/queue/q64.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2024 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: q64.c 17 | * Description: A variant of our wait-free queue for x86 systems 18 | * lacking a 128-bit compare and swap. 19 | * 20 | * Author: John Viega, john@zork.org 21 | */ 22 | 23 | #include 24 | 25 | static const q64_item_t empty_cell = Q64_EMPTY; 26 | static const q64_item_t too_slow_marker = Q64_TOOSLOW; 27 | static const q64_item_t value_mask = ~(Q64_TOOSLOW|Q64_USED); 28 | 29 | static q64_segment_t * 30 | q64_new_segment(uint64_t num_cells) 31 | { 32 | q64_segment_t *ret; 33 | uint64_t len; 34 | 35 | len = sizeof(q64_segment_t) + sizeof(q64_item_t) * num_cells; 36 | ret = mmm_alloc_committed(len); 37 | ret->size = num_cells; 38 | 39 | return ret; 40 | } 41 | 42 | void 43 | q64_init(q64_t *self) 44 | { 45 | return q64_init_size(self, QSIZE_LOG_DEFAULT); 46 | } 47 | 48 | void 49 | q64_init_size(q64_t *self, char size_log) 50 | { 51 | q64_seg_ptrs_t segments; 52 | q64_segment_t *initial_segment; 53 | uint64_t seg_cells; // Number of cells per segment 54 | 55 | if (!size_log) { 56 | size_log = QSIZE_LOG_DEFAULT; 57 | } else { 58 | if (size_log < QSIZE_LOG_MIN || size_log > QSIZE_LOG_MAX) { 59 | abort(); 60 | } 61 | } 62 | 63 | seg_cells = 1 << size_log; 64 | self->default_segment_size = seg_cells; 65 | initial_segment = q64_new_segment(seg_cells); 66 | segments.enqueue_segment = initial_segment; 67 | segments.dequeue_segment = initial_segment; 68 | 69 | atomic_store(&self->segments, segments); 70 | atomic_store(&self->help_needed, 0); 71 | atomic_store(&self->len, 0); 72 | 73 | return; 74 | } 75 | 76 | q64_t * 77 | q64_new(void) 78 | { 79 | return q64_new_size(QSIZE_LOG_DEFAULT); 80 | } 81 | 82 | q64_t * 83 | q64_new_size(char size) 84 | { 85 | q64_t *ret; 86 | 87 | ret = (q64_t *)malloc(sizeof(q64_t)); 88 | q64_init_size(ret, size); 89 | 90 | return ret; 91 | } 92 | 93 | /* We assume here that this is only going to get called when there are 94 | * definitely no more enqueuers/dequeuers in the queue. If you need 95 | * to decref or free any remaining contents, drain the queue before 96 | * calling cleanup. 97 | */ 98 | void 99 | q64_cleanup(q64_t *self) 100 | { 101 | q64_seg_ptrs_t segments; 102 | q64_segment_t *cur; 103 | q64_segment_t *next; 104 | 105 | segments = atomic_load(&self->segments); 106 | cur = segments.dequeue_segment; 107 | 108 | while (cur) { 109 | next = atomic_load(&cur->next); 110 | 111 | mmm_retire_unused(cur); 112 | 113 | cur = next; 114 | } 115 | 116 | return; 117 | } 118 | 119 | void 120 | q64_delete(q64_t *self) 121 | { 122 | q64_cleanup(self); 123 | free(self); 124 | 125 | return; 126 | } 127 | 128 | /* q64_enqueue is pretty simple in the average case. It only gets 129 | * complicated when the segment we're working in runs out of cells 130 | * in which we're allowed to enqueue. Otherwise, we're just 131 | * using FAA to get a new slot to write into, and if it fails, 132 | * it's because a dequeue thinks we're too slow, so we start 133 | * increasing the "step" value exponentially (dequeue ops only 134 | * ever increase in steps of 1). 135 | */ 136 | void 137 | q64_enqueue(q64_t *self, void *item) 138 | { 139 | q64_seg_ptrs_t segments; 140 | q64_seg_ptrs_t candidate_segments; 141 | q64_segment_t *segment; 142 | q64_item_t expected; 143 | q64_item_t candidate; 144 | uint64_t end_size; 145 | uint64_t cur_ix; 146 | uint64_t step; 147 | bool need_help; 148 | bool need_to_enqueue; 149 | q64_segment_t *new_segment; 150 | q64_segment_t *expected_segment; 151 | uint64_t new_size; 152 | 153 | step = 1; 154 | 155 | mmm_start_basic_op(); 156 | 157 | need_help = false; 158 | segments = atomic_read(&self->segments); 159 | segment = segments.enqueue_segment; 160 | end_size = segment->size; 161 | cur_ix = atomic_fetch_add(&segment->enqueue_index, step); 162 | candidate = (((q64_item_t) item) & value_mask) | Q64_USED; 163 | 164 | try_again: 165 | while (cur_ix < end_size) { 166 | expected = empty_cell; 167 | if (CAS(&segment->cells[cur_ix], &expected, candidate)) { 168 | 169 | if (need_help) { 170 | atomic_fetch_sub(&self->help_needed, 1); 171 | } 172 | mmm_end_op(); 173 | 174 | atomic_fetch_add(&self->len, 1); 175 | return; 176 | } 177 | step <<= 1; 178 | cur_ix = atomic_fetch_add(&segment->enqueue_index, step); 179 | } 180 | 181 | if (step >= QUEUE_HELP_VALUE && !need_help) { 182 | need_help = true; 183 | atomic_fetch_add(&self->help_needed, 1); 184 | 185 | segments = atomic_read(&self->segments); 186 | 187 | if (segments.enqueue_segment != segment) { 188 | segment = segments.enqueue_segment; 189 | cur_ix = atomic_fetch_add(&segment->enqueue_index, step); 190 | goto try_again; 191 | } 192 | 193 | new_size = segment->size << 1; 194 | } 195 | else { 196 | segments = atomic_read(&self->segments); 197 | 198 | if (segments.enqueue_segment != segment) { 199 | segment = segments.enqueue_segment; 200 | cur_ix = atomic_fetch_add(&segment->enqueue_index, step); 201 | goto try_again; 202 | } 203 | 204 | if (atomic_read(&self->help_needed)) { 205 | new_size = segment->size << 1; 206 | } 207 | else { 208 | new_size = self->default_segment_size; 209 | } 210 | } 211 | 212 | new_segment = q64_new_segment(new_size); 213 | new_segment->enqueue_index = 1; 214 | expected_segment = NULL; 215 | 216 | atomic_store(&new_segment->cells[0], candidate); 217 | 218 | 219 | if (!CAS(&segment->next, &expected_segment, new_segment)) { 220 | mmm_retire_unused(new_segment); 221 | new_segment = expected_segment; 222 | need_to_enqueue = true; 223 | } 224 | else { 225 | need_to_enqueue = false; 226 | } 227 | 228 | candidate_segments.enqueue_segment = new_segment; 229 | candidate_segments.dequeue_segment = segments.dequeue_segment; 230 | 231 | while (!CAS(&self->segments, &segments, candidate_segments)) { 232 | if (segments.enqueue_segment != segment) { 233 | break; 234 | } 235 | candidate_segments.dequeue_segment = segments.dequeue_segment; 236 | } 237 | 238 | if (!need_to_enqueue) { 239 | if (need_help) { 240 | atomic_fetch_sub(&self->help_needed, 1); 241 | } 242 | 243 | mmm_end_op(); 244 | atomic_fetch_add(&self->len, 1); 245 | 246 | return; 247 | } 248 | segment = new_segment; 249 | cur_ix = atomic_fetch_add(&segment->enqueue_index, step); 250 | 251 | goto try_again; 252 | } 253 | 254 | void * 255 | q64_dequeue(q64_t *self, bool *found) 256 | { 257 | q64_seg_ptrs_t segments; 258 | q64_seg_ptrs_t candidate_segments; 259 | q64_segment_t *segment; 260 | q64_segment_t *new_segment; 261 | q64_item_t cell_contents; 262 | uint64_t cur_ix; 263 | uint64_t head_ix; 264 | void *ret; 265 | 266 | mmm_start_basic_op(); 267 | 268 | segments = atomic_read(&self->segments); 269 | segment = segments.dequeue_segment; 270 | 271 | retry_dequeue: 272 | 273 | while (true) { 274 | cur_ix = atomic_load(&segment->dequeue_index); 275 | head_ix = atomic_load(&segment->enqueue_index); 276 | 277 | if (cur_ix >= segment->size) { 278 | break; 279 | } 280 | 281 | if (cur_ix >= head_ix) { 282 | return hatrack_not_found_w_mmm(found); 283 | } 284 | 285 | cur_ix = atomic_fetch_add(&segment->dequeue_index, 1); 286 | if (cur_ix >= segment->size) { 287 | break; 288 | } 289 | 290 | cell_contents = empty_cell; 291 | 292 | if (CAS(&segment->cells[cur_ix], &cell_contents, too_slow_marker)) { 293 | continue; 294 | } 295 | 296 | ret = (void *)(cell_contents & value_mask); 297 | 298 | atomic_fetch_sub(&self->len, 1); 299 | 300 | return hatrack_found_w_mmm(found, ret); 301 | } 302 | 303 | new_segment = atomic_read(&segment->next); 304 | if (!new_segment) { 305 | /* The enqueuer threads have not completed setting up a new segment 306 | * yet, so the queue is officially empty. 307 | * 308 | * Some future dequeuer will be back here to change the 309 | * dequeue segment pointer. 310 | */ 311 | return hatrack_not_found_w_mmm(found); 312 | } 313 | 314 | candidate_segments.enqueue_segment = segments.enqueue_segment; 315 | candidate_segments.dequeue_segment = new_segment; 316 | 317 | while (!CAS(&self->segments, &segments, candidate_segments)) { 318 | /* If we fail, and someone else updated the dequeue segment, 319 | * then we try again in that new segment. 320 | */ 321 | if (segments.dequeue_segment != segment) { 322 | // We must be way behind. 323 | segment = segments.dequeue_segment; 324 | 325 | goto retry_dequeue; 326 | } 327 | 328 | /* Otherwise, the enqueue segment was updated, and 329 | * we should try again w/ the proper enqueue segment. 330 | */ 331 | candidate_segments.enqueue_segment = segments.enqueue_segment; 332 | } 333 | 334 | mmm_retire(segment); 335 | segments = candidate_segments; 336 | segment = new_segment; 337 | 338 | goto retry_dequeue; 339 | } 340 | -------------------------------------------------------------------------------- /src/queue/queue.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License atn 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: queue.c 17 | * Description: A fast, wait-free queue implementation. 18 | * 19 | * Author: John Viega, john@zork.org 20 | */ 21 | 22 | #include 23 | 24 | static const queue_item_t empty_cell = { NULL, QUEUE_EMPTY }; 25 | static const queue_item_t too_slow_marker = { NULL, QUEUE_TOOSLOW }; 26 | 27 | static queue_segment_t * 28 | queue_new_segment(uint64_t num_cells) 29 | { 30 | queue_segment_t *ret; 31 | uint64_t len; 32 | 33 | len = sizeof(queue_segment_t) + sizeof(queue_item_t) * num_cells; 34 | ret = mmm_alloc_committed(len); 35 | ret->size = num_cells; 36 | 37 | return ret; 38 | } 39 | 40 | void 41 | queue_init(queue_t *self) 42 | { 43 | return queue_init_size(self, QSIZE_LOG_DEFAULT); 44 | } 45 | 46 | void 47 | queue_init_size(queue_t *self, char size_log) 48 | { 49 | queue_seg_ptrs_t segments; 50 | queue_segment_t *initial_segment; 51 | uint64_t seg_cells; // Number of cells per segment 52 | 53 | if (!size_log) { 54 | size_log = QSIZE_LOG_DEFAULT; 55 | } else { 56 | if (size_log < QSIZE_LOG_MIN || size_log > QSIZE_LOG_MAX) { 57 | abort(); 58 | } 59 | } 60 | 61 | seg_cells = 1 << size_log; 62 | self->default_segment_size = seg_cells; 63 | initial_segment = queue_new_segment(seg_cells); 64 | segments.enqueue_segment = initial_segment; 65 | segments.dequeue_segment = initial_segment; 66 | 67 | atomic_store(&self->segments, segments); 68 | atomic_store(&self->help_needed, 0); 69 | atomic_store(&self->len, 0); 70 | 71 | return; 72 | } 73 | 74 | queue_t * 75 | queue_new(void) 76 | { 77 | return queue_new_size(QSIZE_LOG_DEFAULT); 78 | } 79 | 80 | queue_t * 81 | queue_new_size(char size) 82 | { 83 | queue_t *ret; 84 | 85 | ret = (queue_t *)malloc(sizeof(queue_t)); 86 | queue_init_size(ret, size); 87 | 88 | return ret; 89 | } 90 | 91 | /* We assume here that this is only going to get called when there are 92 | * definitely no more enqueuers/dequeuers in the queue. If you need 93 | * to decref or free any remaining contents, drain the queue before 94 | * calling cleanup. 95 | */ 96 | void 97 | queue_cleanup(queue_t *self) 98 | { 99 | queue_seg_ptrs_t segments; 100 | queue_segment_t *cur; 101 | queue_segment_t *next; 102 | 103 | segments = atomic_load(&self->segments); 104 | cur = segments.dequeue_segment; 105 | 106 | while (cur) { 107 | next = atomic_load(&cur->next); 108 | 109 | mmm_retire_unused(cur); 110 | 111 | cur = next; 112 | } 113 | 114 | return; 115 | } 116 | 117 | void 118 | queue_delete(queue_t *self) 119 | { 120 | queue_cleanup(self); 121 | free(self); 122 | 123 | return; 124 | } 125 | 126 | /* queue_enqueue is pretty simple in the average case. It only gets 127 | * complicated when the segment we're working in runs out of cells 128 | * in which we're allowed to enqueue. Otherwise, we're just 129 | * using FAA to get a new slot to write into, and if it fails, 130 | * it's because a dequeue thinks we're too slow, so we start 131 | * increasing the "step" value exponentially (dequeue ops only 132 | * ever increase in steps of 1). 133 | */ 134 | void 135 | queue_enqueue(queue_t *self, void *item) 136 | { 137 | queue_seg_ptrs_t segments; 138 | queue_seg_ptrs_t candidate_segments; 139 | queue_segment_t *segment; 140 | queue_item_t expected; 141 | queue_item_t candidate; 142 | uint64_t end_size; 143 | uint64_t cur_ix; 144 | uint64_t step; 145 | bool need_help; 146 | bool need_to_enqueue; 147 | queue_segment_t *new_segment; 148 | queue_segment_t *expected_segment; 149 | uint64_t new_size; 150 | 151 | step = 1; 152 | 153 | mmm_start_basic_op(); 154 | 155 | need_help = false; 156 | segments = atomic_read(&self->segments); 157 | segment = segments.enqueue_segment; 158 | end_size = segment->size; 159 | cur_ix = atomic_fetch_add(&segment->enqueue_index, step); 160 | candidate.state = QUEUE_USED; 161 | candidate.item = item; 162 | 163 | try_again: 164 | while (cur_ix < end_size) { 165 | expected = empty_cell; 166 | if (CAS(&segment->cells[cur_ix], &expected, candidate)) { 167 | 168 | if (need_help) { 169 | atomic_fetch_sub(&self->help_needed, 1); 170 | } 171 | mmm_end_op(); 172 | 173 | atomic_fetch_add(&self->len, 1); 174 | return; 175 | } 176 | step <<= 1; 177 | cur_ix = atomic_fetch_add(&segment->enqueue_index, step); 178 | } 179 | 180 | if (step >= QUEUE_HELP_VALUE && !need_help) { 181 | need_help = true; 182 | atomic_fetch_add(&self->help_needed, 1); 183 | 184 | segments = atomic_read(&self->segments); 185 | 186 | if (segments.enqueue_segment != segment) { 187 | segment = segments.enqueue_segment; 188 | cur_ix = atomic_fetch_add(&segment->enqueue_index, step); 189 | goto try_again; 190 | } 191 | 192 | new_size = segment->size << 1; 193 | } 194 | else { 195 | segments = atomic_read(&self->segments); 196 | 197 | if (segments.enqueue_segment != segment) { 198 | segment = segments.enqueue_segment; 199 | cur_ix = atomic_fetch_add(&segment->enqueue_index, step); 200 | goto try_again; 201 | } 202 | 203 | if (atomic_read(&self->help_needed)) { 204 | new_size = segment->size << 1; 205 | } 206 | else { 207 | new_size = self->default_segment_size; 208 | } 209 | } 210 | 211 | new_segment = queue_new_segment(new_size); 212 | new_segment->enqueue_index = 1; 213 | expected_segment = NULL; 214 | 215 | atomic_store(&new_segment->cells[0], candidate); 216 | 217 | 218 | if (!CAS(&segment->next, &expected_segment, new_segment)) { 219 | mmm_retire_unused(new_segment); 220 | new_segment = expected_segment; 221 | need_to_enqueue = true; 222 | } 223 | else { 224 | need_to_enqueue = false; 225 | } 226 | 227 | candidate_segments.enqueue_segment = new_segment; 228 | candidate_segments.dequeue_segment = segments.dequeue_segment; 229 | 230 | while (!CAS(&self->segments, &segments, candidate_segments)) { 231 | if (segments.enqueue_segment != segment) { 232 | break; 233 | } 234 | candidate_segments.dequeue_segment = segments.dequeue_segment; 235 | } 236 | 237 | if (!need_to_enqueue) { 238 | if (need_help) { 239 | atomic_fetch_sub(&self->help_needed, 1); 240 | } 241 | 242 | mmm_end_op(); 243 | atomic_fetch_add(&self->len, 1); 244 | 245 | return; 246 | } 247 | segment = new_segment; 248 | cur_ix = atomic_fetch_add(&segment->enqueue_index, step); 249 | 250 | goto try_again; 251 | } 252 | 253 | void * 254 | queue_dequeue(queue_t *self, bool *found) 255 | { 256 | queue_seg_ptrs_t segments; 257 | queue_seg_ptrs_t candidate_segments; 258 | queue_segment_t *segment; 259 | queue_segment_t *new_segment; 260 | queue_item_t cell_contents; 261 | uint64_t cur_ix; 262 | uint64_t head_ix; 263 | void *ret; 264 | 265 | mmm_start_basic_op(); 266 | 267 | segments = atomic_read(&self->segments); 268 | segment = segments.dequeue_segment; 269 | 270 | retry_dequeue: 271 | 272 | while (true) { 273 | cur_ix = atomic_load(&segment->dequeue_index); 274 | head_ix = atomic_load(&segment->enqueue_index); 275 | 276 | if (cur_ix >= segment->size) { 277 | break; 278 | } 279 | 280 | if (cur_ix >= head_ix) { 281 | return hatrack_not_found_w_mmm(found); 282 | } 283 | 284 | cur_ix = atomic_fetch_add(&segment->dequeue_index, 1); 285 | if (cur_ix >= segment->size) { 286 | break; 287 | } 288 | 289 | cell_contents = empty_cell; 290 | 291 | if (CAS(&segment->cells[cur_ix], &cell_contents, too_slow_marker)) { 292 | continue; 293 | } 294 | 295 | ret = cell_contents.item; 296 | 297 | atomic_fetch_sub(&self->len, 1); 298 | 299 | return hatrack_found_w_mmm(found, ret); 300 | } 301 | 302 | new_segment = atomic_read(&segment->next); 303 | if (!new_segment) { 304 | /* The enqueuer threads have not completed setting up a new segment 305 | * yet, so the queue is officially empty. 306 | * 307 | * Some future dequeuer will be back here to change the 308 | * dequeue segment pointer. 309 | */ 310 | return hatrack_not_found_w_mmm(found); 311 | } 312 | 313 | candidate_segments.enqueue_segment = segments.enqueue_segment; 314 | candidate_segments.dequeue_segment = new_segment; 315 | 316 | while (!CAS(&self->segments, &segments, candidate_segments)) { 317 | /* If we fail, and someone else updated the dequeue segment, 318 | * then we try again in that new segment. 319 | */ 320 | if (segments.dequeue_segment != segment) { 321 | // We must be way behind. 322 | segment = segments.dequeue_segment; 323 | 324 | goto retry_dequeue; 325 | } 326 | 327 | /* Otherwise, the enqueue segment was updated, and 328 | * we should try again w/ the proper enqueue segment. 329 | */ 330 | candidate_segments.enqueue_segment = segments.enqueue_segment; 331 | } 332 | 333 | mmm_retire(segment); 334 | segments = candidate_segments; 335 | segment = new_segment; 336 | 337 | goto retry_dequeue; 338 | } 339 | -------------------------------------------------------------------------------- /src/support/counters.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: counters.c 17 | * Description: In-memory counters for performance monitoring, 18 | * when HATRACK_DEBUG is on. 19 | * 20 | * Author: John Viega, john@zork.org 21 | */ 22 | 23 | #include 24 | 25 | #ifdef HATRACK_COUNTERS 26 | 27 | #include 28 | #include 29 | 30 | // clang-format off 31 | _Atomic uint64_t hatrack_counters[HATRACK_COUNTERS_NUM] = {}; 32 | uint64_t hatrack_last_counters[HATRACK_COUNTERS_NUM] = {}; 33 | 34 | _Atomic uint64_t hatrack_yn_counters[HATRACK_YN_COUNTERS_NUM][2] = {}; 35 | uint64_t hatrack_last_yn_counters[HATRACK_YN_COUNTERS_NUM][2] = {}; 36 | 37 | char *hatrack_counter_names[HATRACK_COUNTERS_NUM] = { 38 | "mmm alloc calls", 39 | "mmm used retires", 40 | "mmm unused retires", 41 | "stores shrunk", 42 | "hi-a sleep 1 worked", 43 | "hi-a sleep 1 failed", 44 | "hi-a sleep 2 worked", 45 | "hi-a sleep 2 failed", 46 | "wh help requests" 47 | }; 48 | 49 | char *hatrack_yn_counter_names[HATRACK_YN_COUNTERS_NUM] = { 50 | "linearize epoch eq", // 0 51 | "mmm write commits", // 1 52 | "mmm commit helps", // 2 53 | "lh bucket acquires", // 3 54 | "lh record installs", // 4 55 | "lh record delete", // 5 56 | "lh store creates", // 6 57 | "lh F_MOVING set", // 7 58 | "lh F_MOVED (empty)", // 8 59 | "lh F_MOVED (deleted)", // 9 60 | "lh migrate hash", // 10 61 | "lh migrate record", // 11 62 | "lh F_MOVED (migrate)", // 12 63 | "lh len installed", // 13 64 | "lh store installs", // 14 65 | "lh-a bucket acquires", // 15 66 | "lh-a ptr installs", // 16 67 | "lh-a hist hash installs", // 17 68 | "lh-a record installs", // 18 69 | "lh-a record delete", // 19 70 | "lh-a store creates", // 20 71 | "lh-a F_MOVING set", // 21 72 | "lh-a F_MOVED (empty)", // 22 73 | "lh-a F_MOVED (deleted)", // 23 74 | "lh-a migrate hash", // 24 75 | "lh-a migrate record", // 25 76 | "lh-a move other hash", // 26 77 | "lh-a install new ptr", // 27 78 | "lh-a F_MOVED (migrate)", // 28 79 | "lh-a hist ptr installed", // 29 80 | "lh-a store installs", // 30 81 | "lh-b bucket acquires", // 31 82 | "lh-b ptr installs", // 32 83 | "lh-b hist hash installs", // 33 84 | "lh-b forward installed", // 34 85 | "lh-b record installs", // 35 86 | "lh-b record delete", // 36 87 | "lh-b store creates", // 37 88 | "lh-b F_MOVING set", // 38 89 | "lh-b F_MOVED (empty)", // 39 90 | "lh-b F_MOVED (deleted)", // 40 91 | "lh-b migrate hash", // 41 92 | "lh-b migrate record", // 42 93 | "lh-b move other hash", // 43 94 | "lh-b install new ptr", // 44 95 | "lh-b F_MOVED (migrate)", // 45 96 | "lh-b hist ptr installed", // 46 97 | "lh-b store installs", // 47 98 | "hih bucket acquires", // 48 99 | "hih record installs", // 49 100 | "hih record delete", // 50 101 | "hih store creates", // 51 102 | "hih F_MOVING set", // 52 103 | "hih F_MOVED (empty)", // 53 104 | "hih migrate hash", // 54 105 | "hih migrate record", // 55 106 | "hih F_MOVED (migrate)", // 56 107 | "hih len installed", // 57 108 | "hih store installs", // 58 109 | "hiha woke up to no job", // 59 110 | "wh bucket acquires", // 60 111 | "wh record installs", // 61 112 | "wh record delete", // 62 113 | "wh store creates", // 63 114 | "wh F_MOVING set", // 64 115 | "wh F_MOVED (empty)", // 65 116 | "wh migrate hash", // 66 117 | "wh migrate record", // 67 118 | "wh F_MOVED (migrate)", // 68 119 | "wh len installed", // 69 120 | "wh store installs", // 70 121 | "wool bucket acquires", // 71 122 | "wool record installs", // 72 123 | "wool record delete", // 73 124 | "wool store creates", // 74 125 | "wool F_MOVING set", // 75 126 | "wool F_MOVED (empty)", // 76 127 | "wool F_MOVED (deleted)", // 77 128 | "wool migrate hash", // 78 129 | "wool migrate record", // 79 130 | "wool F_MOVED (migrate)", // 70 131 | "wool len installed", // 81 132 | "wool store installs", // 82 133 | }; 134 | 135 | // clang-format on 136 | 137 | /* 138 | * Used to output (to stderr) the difference between counters, from 139 | * the last time counters_output_delta() was called, until now. 140 | */ 141 | void 142 | counters_output_delta(void) 143 | { 144 | uint64_t i; 145 | uint64_t total; 146 | uint64_t y_cur, n_cur, y_last, n_last; 147 | uint64_t ydelta, ndelta; 148 | double percent; 149 | 150 | fprintf(stderr, "----------- Counter Deltas --------------\n"); 151 | for (i = 0; i < HATRACK_COUNTERS_NUM; i++) { 152 | if (hatrack_counters[i] == hatrack_last_counters[i]) { 153 | continue; 154 | } 155 | 156 | ydelta = hatrack_counters[i] - hatrack_last_counters[i]; 157 | fprintf(stderr, 158 | "%s:\t %llu\n", 159 | hatrack_counter_names[i], 160 | (unsigned long long)ydelta); 161 | 162 | hatrack_last_counters[i] = hatrack_counters[i]; 163 | } 164 | 165 | for (i = 0; i < HATRACK_YN_COUNTERS_NUM; i++) { 166 | y_cur = hatrack_yn_counters[i][0]; 167 | n_cur = hatrack_yn_counters[i][1]; 168 | y_last = hatrack_last_yn_counters[i][0]; 169 | n_last = hatrack_last_yn_counters[i][1]; 170 | ydelta = y_cur - y_last; 171 | ndelta = n_cur - n_last; 172 | total = ydelta + ndelta; 173 | percent = (((double)ydelta) / (double)total) * 100.0; 174 | 175 | hatrack_last_yn_counters[i][0] = y_cur; 176 | hatrack_last_yn_counters[i][1] = n_cur; 177 | 178 | if (!total) { 179 | continue; 180 | } 181 | 182 | fprintf(stderr, 183 | "%s:\t %llu y, %llu n of %llu (%.2f%% y)\n", 184 | hatrack_yn_counter_names[i], 185 | (unsigned long long)ydelta, 186 | (unsigned long long)ndelta, 187 | (unsigned long long)total, 188 | percent); 189 | } 190 | 191 | return; 192 | } 193 | 194 | void 195 | counters_output_alltime(void) 196 | { 197 | uint64_t i; 198 | uint64_t total; 199 | bool unused_counters = false; 200 | bool print_comma = false; 201 | 202 | fprintf(stderr, "----------- Counter TOTALS --------------\n"); 203 | 204 | for (i = 0; i < HATRACK_COUNTERS_NUM; i++) { 205 | if (!hatrack_counters[i]) { 206 | unused_counters = true; 207 | continue; 208 | } 209 | 210 | fprintf(stderr, 211 | "%s:\t %llu\n", 212 | hatrack_counter_names[i], 213 | (unsigned long long)hatrack_counters[i]); 214 | } 215 | 216 | for (i = 0; i < HATRACK_YN_COUNTERS_NUM; i++) { 217 | total = hatrack_yn_counters[i][0] + hatrack_yn_counters[i][1]; 218 | 219 | if (!total) { 220 | unused_counters = true; 221 | continue; 222 | } 223 | 224 | fprintf(stderr, 225 | "%s:\t %llu y, %llu n of %llu (%.2f%% y)\n", 226 | hatrack_yn_counter_names[i], 227 | (unsigned long long)hatrack_yn_counters[i][0], 228 | (unsigned long long)hatrack_yn_counters[i][1], 229 | (unsigned long long)total, 230 | (double)100.0 231 | * (((double)hatrack_yn_counters[i][0]) / (double)total)); 232 | } 233 | 234 | if (unused_counters) { 235 | fprintf(stderr, "\nUnused counters: "); 236 | 237 | for (i = 0; i < HATRACK_COUNTERS_NUM; i++) { 238 | if (hatrack_counters[i]) { 239 | continue; 240 | } 241 | 242 | if (print_comma) { 243 | fprintf(stderr, ", %s", hatrack_counter_names[i]); 244 | } 245 | else { 246 | fprintf(stderr, "%s", hatrack_counter_names[i]); 247 | } 248 | } 249 | 250 | for (i = 0; i < HATRACK_YN_COUNTERS_NUM; i++) { 251 | if (hatrack_yn_counters[i][0] || hatrack_yn_counters[i][1]) { 252 | continue; 253 | } 254 | 255 | if (print_comma) { 256 | fprintf(stderr, ", %s", hatrack_yn_counter_names[i]); 257 | } 258 | else { 259 | fprintf(stderr, "%s", hatrack_yn_counter_names[i]); 260 | } 261 | } 262 | fprintf(stderr, "\n"); 263 | } 264 | 265 | return; 266 | } 267 | 268 | #endif 269 | -------------------------------------------------------------------------------- /src/support/hatrack_common.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: hatrack_common.c 17 | * Description: Functionality shared across all our default hash tables. 18 | * Most of it consists of short inlined functions, which 19 | * live in hatrack_common.h 20 | * 21 | * Author: John Viega, john@zork.org 22 | */ 23 | 24 | #include 25 | 26 | /* Used when using quicksort to sort the contents of a hash table 27 | * 'view' by insertion time (the sort_epoch field). 28 | */ 29 | int 30 | hatrack_quicksort_cmp(const void *bucket1, const void *bucket2) 31 | { 32 | hatrack_view_t *item1; 33 | hatrack_view_t *item2; 34 | 35 | item1 = (hatrack_view_t *)bucket1; 36 | item2 = (hatrack_view_t *)bucket2; 37 | 38 | return item1->sort_epoch - item2->sort_epoch; 39 | } 40 | -------------------------------------------------------------------------------- /src/support/helpmanager.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: helpmanager.c 17 | * Description: Support for data-structure specific linearization 18 | * through a wait-free CAPQ (compare-and-pop queue), 19 | * 20 | * 21 | * Author: John Viega, john@zork.org 22 | * 23 | */ 24 | 25 | #include 26 | #include 27 | 28 | void 29 | hatrack_help_init(help_manager_t *manager, 30 | void *parent, 31 | helper_func *vtable, 32 | bool zero) 33 | { 34 | if (zero) { 35 | bzero(manager, sizeof(help_manager_t)); 36 | } 37 | 38 | manager->parent = parent; 39 | manager->vtable = vtable; 40 | 41 | capq_init(&manager->capq); 42 | 43 | return; 44 | } 45 | 46 | static const help_cell_t empty_cell = { 47 | .data = NULL, 48 | .jobid = -1 49 | }; 50 | 51 | void * 52 | hatrack_perform_wf_op(help_manager_t *manager, 53 | uint64_t op, 54 | void *data, 55 | void *aux, 56 | bool *foundret) 57 | { 58 | help_record_t *my_record; 59 | help_record_t *other_record; 60 | capq_top_t qtop; 61 | int64_t my_jobid; 62 | int64_t other_jobid; 63 | bool found; 64 | helper_func f; 65 | help_cell_t retcell; 66 | help_cell_t foundcell; 67 | 68 | my_record = &thread_records[mmm_mytid]; 69 | my_record->op = op; 70 | my_record->input = data; 71 | my_record->aux = aux; 72 | my_record->success = empty_cell; 73 | my_record->retval = empty_cell; 74 | 75 | my_jobid = capq_enqueue(&manager->capq, my_record); 76 | 77 | do { 78 | qtop = capq_top(&manager->capq, &found); 79 | if (!found) { 80 | break; 81 | } 82 | other_jobid = qtop.state; 83 | other_record = qtop.item; 84 | f = manager->vtable[other_record->op]; 85 | 86 | (*f)(manager, other_record, other_jobid); 87 | } while (other_jobid < my_jobid); 88 | 89 | retcell = atomic_load(&my_record->retval); 90 | 91 | if (foundret) { 92 | foundcell = atomic_load(&my_record->success); 93 | *foundret = (bool)foundcell.data; 94 | } 95 | 96 | return retcell.data; 97 | } 98 | 99 | void 100 | hatrack_complete_help(help_manager_t *manager, 101 | help_record_t *record, 102 | int64_t jobid, 103 | void *result, 104 | bool success) 105 | { 106 | help_cell_t candidate; 107 | help_cell_t expected; 108 | 109 | expected = atomic_load(&record->retval); 110 | 111 | if (expected.jobid < jobid) { 112 | candidate.data = result; 113 | candidate.jobid = jobid; 114 | 115 | CAS(&record->retval, &expected, candidate); 116 | } 117 | 118 | expected = atomic_load(&record->success); 119 | 120 | if (expected.jobid < jobid) { 121 | candidate.data = (void *)success; 122 | candidate.jobid = jobid; 123 | 124 | CAS(&record->success, &expected, candidate); 125 | } 126 | 127 | capq_cap(&manager->capq, jobid); 128 | 129 | return; 130 | } 131 | -------------------------------------------------------------------------------- /src/support/mmm.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: mmm.c 17 | * Description: Miniature memory manager: a malloc wrapper to support 18 | * linearization and safe reclaimation for my hash tables. 19 | * 20 | * Author: John Viega, john@zork.org 21 | * 22 | */ 23 | 24 | #include 25 | 26 | // clang-format off 27 | __thread mmm_header_t *mmm_retire_list = NULL; 28 | __thread pthread_once_t mmm_inited = PTHREAD_ONCE_INIT; 29 | _Atomic uint64_t mmm_epoch = HATRACK_EPOCH_FIRST; 30 | _Atomic uint64_t mmm_nexttid = 0; 31 | __thread int64_t mmm_mytid = -1; 32 | __thread uint64_t mmm_retire_ctr = 0; 33 | 34 | uint64_t mmm_reservations[HATRACK_THREADS_MAX] = { 0, }; 35 | 36 | //clang-format on 37 | 38 | 39 | static void mmm_empty(void); 40 | 41 | 42 | /* 43 | * We want to avoid overrunning the reservations array that our memory 44 | * management system uses. 45 | * 46 | * In all cases, the first HATRACK_THREADS_MAX threads to register 47 | * with us get a steadily increasing id. Also, threads will yield 48 | * their TID altogether if and when they call the mmm thread cleanup 49 | * routine-- mmm_clean_up_before_exit(). 50 | * 51 | * The unused ID then goes on a linked list (mmm_free_tids), and 52 | * doesn't get touched until we run out of TIDs to give out. 53 | * 54 | */ 55 | 56 | static _Atomic (mmm_free_tids_t *) mmm_free_tids; 57 | 58 | /* This grabs an mmm-specific threadid and stashes it in the 59 | * thread-local variable mmm_mytid. 60 | * 61 | * We have a fixed number of TIDS to give out though (controlled by 62 | * the preprocessor variable, HATRACK_THREADS_MAX). We give them out 63 | * sequentially till they're done, and then we give out ones that have 64 | * been "given back", which are stored on a stack (mmm_free_tids). 65 | * 66 | * If we finally run out, we abort. 67 | */ 68 | void 69 | mmm_register_thread(void) 70 | { 71 | mmm_free_tids_t *head; 72 | 73 | if (mmm_mytid != -1) { 74 | return; 75 | } 76 | mmm_mytid = atomic_fetch_add(&mmm_nexttid, 1); 77 | 78 | if (mmm_mytid >= HATRACK_THREADS_MAX) { 79 | head = atomic_load(&mmm_free_tids); 80 | 81 | do { 82 | if (!head) { 83 | abort(); 84 | } 85 | } while (!CAS(&mmm_free_tids, &head, head->next)); 86 | 87 | mmm_mytid = head->tid; 88 | mmm_retire(head); 89 | } 90 | 91 | mmm_reservations[mmm_mytid] = HATRACK_EPOCH_UNRESERVED; 92 | 93 | return; 94 | } 95 | 96 | // Call when a thread exits to add to the free TID stack. 97 | static void 98 | mmm_tid_giveback(void) 99 | { 100 | mmm_free_tids_t *new_head; 101 | mmm_free_tids_t *old_head; 102 | 103 | new_head = mmm_alloc(sizeof(mmm_free_tids_t)); 104 | new_head->tid = mmm_mytid; 105 | old_head = atomic_load(&mmm_free_tids); 106 | 107 | do { 108 | new_head->next = old_head; 109 | } while (!CAS(&mmm_free_tids, &old_head, new_head)); 110 | 111 | return; 112 | } 113 | 114 | // This is here for convenience of testing; generally this 115 | // is not the way to handle tid recyling! 116 | void mmm_reset_tids(void) 117 | { 118 | atomic_store(&mmm_nexttid, 0); 119 | 120 | return; 121 | } 122 | 123 | /* For now, our cleanup function spins until it is able to retire 124 | * everything on its list. Soon, when we finally get around to 125 | * worrying about thread kills, we will change this to add its 126 | * contents to an "ophan" list. 127 | */ 128 | void 129 | mmm_clean_up_before_exit(void) 130 | { 131 | if (mmm_mytid == -1) { 132 | return; 133 | } 134 | 135 | mmm_end_op(); 136 | 137 | while (mmm_retire_list) { 138 | mmm_empty(); 139 | } 140 | 141 | mmm_tid_giveback(); 142 | 143 | return; 144 | } 145 | 146 | /* Sets the retirement epoch on the pointer, and adds it to the 147 | * thread-local retirement list. 148 | * 149 | * We also keep a thread-local counter of how many times we've called 150 | * this function, and every HATRACK_RETIRE_FREQ calls, we run 151 | * mmm_empty(), which walks our thread-local retirement list, looking 152 | * for items to free. 153 | */ 154 | void 155 | mmm_retire(void *ptr) 156 | { 157 | mmm_header_t *cell; 158 | 159 | cell = mmm_get_header(ptr); 160 | 161 | #ifdef HATRACK_MMM_DEBUG 162 | // Don't need this check when not debugging algorithms. 163 | if (!cell->write_epoch) { 164 | DEBUG_MMM_INTERNAL(ptr, "No write epoch??"); 165 | abort(); 166 | } 167 | // Algorithms that steal bits from pointers might steal up to 168 | // three bits, thus the mask of 0x07. 169 | if (hatrack_pflag_test(ptr, 0x07)) { 170 | DEBUG_MMM_INTERNAL(ptr, "Bad alignment on retired pointer."); 171 | abort(); 172 | } 173 | 174 | /* Detect multiple threads adding this to their retire list. 175 | * Generally, you should be able to avoid this, but with 176 | * HATRACK_MMM_DEBUG on we explicitly check for it. 177 | */ 178 | if (cell->retire_epoch) { 179 | DEBUG_MMM_INTERNAL(ptr, "Double free"); 180 | DEBUG_PTR((void *)atomic_load(&mmm_epoch), "epoch of double free"); 181 | 182 | abort(); 183 | 184 | return; 185 | } 186 | #endif 187 | 188 | cell->retire_epoch = atomic_load(&mmm_epoch); 189 | cell->next = mmm_retire_list; 190 | mmm_retire_list = cell; 191 | 192 | DEBUG_MMM_INTERNAL(cell->data, "mmm_retire"); 193 | 194 | if (++mmm_retire_ctr & HATRACK_RETIRE_FREQ) { 195 | mmm_retire_ctr = 0; 196 | mmm_empty(); 197 | } 198 | 199 | return; 200 | } 201 | 202 | /* The basic gist of this algorithm is that we're going to look at 203 | * every reservation we can find, identifying the oldest reservation 204 | * in the list. 205 | * 206 | * Then, we can then safely free anything in the list with an earlier 207 | * retirement epoch than the reservation time. Since the items in the 208 | * stack were pushed on in order of their retirement epoch, it 209 | * suffices to find the first item that is lower than the target, 210 | * and free everything else. 211 | */ 212 | static void 213 | mmm_empty(void) 214 | { 215 | mmm_header_t *tmp; 216 | mmm_header_t *cell; 217 | uint64_t lowest; 218 | uint64_t reservation; 219 | uint64_t lasttid; 220 | uint64_t i; 221 | 222 | /* We don't have to search the whole array, just the items assigned 223 | * to active threads. Even if a new thread comes along, it will 224 | * not be able to reserve something that's already been retired 225 | * by the time we call this. 226 | */ 227 | lasttid = atomic_load(&mmm_nexttid); 228 | 229 | if (lasttid > HATRACK_THREADS_MAX) { 230 | lasttid = HATRACK_THREADS_MAX; 231 | } 232 | 233 | /* We start out w/ the "lowest" reservation we've seen as 234 | * HATRACK_EPOCH_MAX. If this value never changes, then it 235 | * means no epochs were reserved, and we can safely 236 | * free every record in our stack. 237 | */ 238 | lowest = HATRACK_EPOCH_MAX; 239 | 240 | for (i = 0; i < lasttid; i++) { 241 | reservation = mmm_reservations[i]; 242 | 243 | if (reservation < lowest) { 244 | lowest = reservation; 245 | } 246 | } 247 | 248 | /* The list here is ordered by retire epoch, with most recent on 249 | * top. Go down the list until the NEXT cell is the first item we 250 | * should delete. 251 | * 252 | * Then, set the current cell's next pointer to NULL (since 253 | * it's the new end of the list), and then place the pointer at 254 | * the top of the list of cells to delete. 255 | * 256 | * Note that this function is only called if there's something 257 | * something on the retire list, so cell will never start out 258 | * empty. 259 | */ 260 | cell = mmm_retire_list; 261 | 262 | // Special-case this, in case we have to delete the head cell, 263 | // to make sure we reinitialize the linked list right. 264 | if (mmm_retire_list->retire_epoch < lowest) { 265 | mmm_retire_list = NULL; 266 | } else { 267 | while (true) { 268 | // We got to the end of the list, and didn't 269 | // find one we should bother deleting. 270 | if (!cell->next) { 271 | return; 272 | } 273 | 274 | if (cell->next->retire_epoch < lowest) { 275 | tmp = cell; 276 | cell = cell->next; 277 | tmp->next = NULL; 278 | break; 279 | } 280 | 281 | cell = cell->next; 282 | } 283 | } 284 | 285 | // Now cell and everything below it can be freed. 286 | while (cell) { 287 | tmp = cell; 288 | cell = cell->next; 289 | HATRACK_FREE_CTR(); 290 | DEBUG_MMM_INTERNAL(tmp->data, "mmm_empty::free"); 291 | 292 | // Call the cleanup handler, if one exists. 293 | if (tmp->cleanup) { 294 | (*tmp->cleanup)(&tmp->data, tmp->cleanup_aux); 295 | } 296 | 297 | free(tmp); 298 | } 299 | 300 | return; 301 | } 302 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # The Hatrack test suite 2 | 3 | Build this from the top level directory using: 4 | ``` 5 | make check 6 | ``` 7 | 8 | That will add an executable names `test` to this directory, which will 9 | run some very basic functionality tests, and then run a ton of timing 10 | tests. 11 | 12 | If you'd like to see counters for most of the lock-free 13 | implementations, to see how often compare-and-swap applications fail, 14 | then compile with `-DHATRACK_COUNTERS` on. This will slow down the 15 | algorithms that use it, considerably. See the `config-debug` script 16 | in the `scripts` directory, which configures without optimization, 17 | with counters, and with additional debugging turned on. 18 | 19 | So far, I've run these tests in the following environments: 20 | 21 | 1) An M1 Mac laptop, using clang. 22 | 23 | 2) An x86 without a 128-bit compare-and-swap, using `clang`. 24 | 25 | 3) The same x86, using `gcc`. 26 | 27 | I've tested with a few different memory managers as well. 28 | 29 | In my initial testing, the lock-free algorithms usually performs about 30 | as well-as their locking counterparts, or even better. That's true 31 | even in my environments where there's no 128-bit compare-and-swap 32 | operation. 33 | 34 | I have not yet put together real scaling data to show how each 35 | algorithm performs as the number of cores goes up, mainly because I've 36 | not yet run on a machine with a high number of cores. 37 | 38 | But I expect the lock-free algorithms to NOT be any sort of bottleneck 39 | to scaling. The `malloc` implementation is more likely. In fact, I've 40 | switched out `malloc`s using `LD_PRELOAD`, and on the Mac, the system 41 | allocator definitely seems to be a bottleneck, as with `hoard`, 42 | performance scales much more linearly. 43 | 44 | You can run custom tests from the command line; `test --help` should 45 | get you started. -------------------------------------------------------------------------------- /tests/default.c: -------------------------------------------------------------------------------- 1 | /* Copyright © 2021-2022 John Viega 2 | * 3 | * See LICENSE.txt for licensing info. 4 | * 5 | * Name: default.c 6 | * 7 | * Description: The test executable runs the below tests by default, 8 | * unless you specify other testing. It leverages the 9 | * performance.c to do the actual testing, this just 10 | * picks out the parameters. 11 | * 12 | * Author: John Viega, john@zork.org 13 | */ 14 | 15 | #include "testhat.h" 16 | 17 | #define basictest(name, g, p, a, r, d, v, o, sz, pf, range, num, ops) \ 18 | { \ 19 | name, g, p, a, r, d, v, o, sz, pf, range, num, ops, true, 0, NULL \ 20 | } 21 | 22 | #define threadset(name, g, p, a, r, d, v, o, sz, pf, range, ops) \ 23 | basictest(name, g, p, a, r, d, v, o, sz, pf, range, 1, ops), \ 24 | basictest(name, g, p, a, r, d, v, o, sz, pf, range, 2, ops), \ 25 | basictest(name, g, p, a, r, d, v, o, sz, pf, range, 3, ops), \ 26 | basictest(name, g, p, a, r, d, v, o, sz, pf, range, 4, ops), \ 27 | basictest(name, g, p, a, r, d, v, o, sz, pf, range, 8, ops), \ 28 | basictest(name, g, p, a, r, d, v, o, sz, pf, range, 20, ops), \ 29 | basictest(name, g, p, a, r, d, v, o, sz, pf, range, 100, ops) 30 | 31 | #define viewset(name, g, p, a, r, d, v, o, pf, num) \ 32 | basictest(name, g, p, a, r, d, v, o, 5, pf, 10, num, 5000000), \ 33 | basictest(name, g, p, a, r, d, v, o, 8, pf, 100, num, 1500000), \ 34 | basictest(name, g, p, a, r, d, v, o, 11, pf, 1000, num, 250000), \ 35 | basictest(name, g, p, a, r, d, v, o, 15, pf, 10000, num, 10000), \ 36 | basictest(name, g, p, a, r, d, v, o, 18, pf, 100000, num, 1000) 37 | 38 | #define sortset(name, g, p, a, r, d, v, o, pf, num) \ 39 | basictest(name, g, p, a, r, d, v, o, 5, pf, 10, num, 5000000), \ 40 | basictest(name, g, p, a, r, d, v, o, 8, pf, 100, num, 1000000), \ 41 | basictest(name, g, p, a, r, d, v, o, 11, pf, 1000, num, 50000), \ 42 | basictest(name, g, p, a, r, d, v, o, 15, pf, 10000, num, 3000), \ 43 | basictest(name, g, p, a, r, d, v, o, 18, pf, 100000, num, 300) 44 | 45 | /* Right now, I've picked all op counts so that most tests take around 46 | * a second (plus or minus a bit) w/ 1 thread, when compiled 47 | * w/ optimization and w/o debug. 48 | * 49 | * Note that, duncecap's locking strategy often makes it a huge, huge 50 | * outlier, where it can take 10x the time of other algorithms as you 51 | * add threads. 52 | */ 53 | // clang-format off 54 | benchmark_t default_tests[] = { 55 | threadset("big read", 100, 0, 0, 0, 0, 0, 0, 17, 100, 100000, 10000000), 56 | threadset("big put", 0, 100, 0, 0, 0, 0, 0, 4, 0, 100000, 10000000), 57 | threadset("big add", 0, 0, 100, 0, 0, 0, 0, 4, 0, 100000, 10000000), 58 | threadset("big replace", 0, 0, 0, 100, 0, 0, 0, 17, 75, 100000, 10000000), 59 | threadset("big remove", 0, 0, 0, 0, 100, 0, 0, 17, 100, 100000, 10000000), 60 | threadset("small read", 100, 0, 0, 0, 0, 0, 0, 6, 100, 64, 25000000), 61 | threadset("small put", 0, 100, 0, 0, 0, 0, 0, 6, 0, 64, 15000000), 62 | threadset("med. read", 100, 0, 0, 0, 0, 0, 0, 12, 100, 2048, 50000000), 63 | threadset("med. put", 0, 100, 0, 0, 0, 0, 0, 12, 0, 2048, 25000000), 64 | viewset("view speed", 0, 0, 0, 0, 0, 100, 0, 100, 1), 65 | sortset("sort speed", 0, 0, 0, 0, 0, 0, 100, 100, 1), 66 | threadset("grow", 0, 0, 100, 10, 0, 0, 0, 4, 0, 2500000, 2500000), 67 | threadset("big cache", 98, 0, 1, 0, 1, 0, 0, 23, 75, 8388608, 5000000), 68 | threadset("data xch", 10, 0, 40, 10, 40, 0, 0, 17, 75, 100000, 15000000), 69 | threadset("contend", 0, 100, 0, 0, 0, 0, 0, 20, 0, 10, 25000000), 70 | threadset("|| sort", 60, 20, 0, 5, 5, 0, 10, 17, 50, 100000, 2000), 71 | { 72 | 0, 73 | } 74 | }; 75 | 76 | void 77 | run_default_tests(config_info_t *config) 78 | { 79 | int i = 0; 80 | 81 | while (default_tests[i].total_ops) { 82 | default_tests[i].hat_list = (char **)config->hat_list; 83 | 84 | run_performance_test(&default_tests[i]); 85 | counters_output_delta(); 86 | i++; 87 | } 88 | 89 | return; 90 | } 91 | -------------------------------------------------------------------------------- /tests/rand.c: -------------------------------------------------------------------------------- 1 | /* Copyright © 2021-2022 John Viega 2 | * 3 | * See LICENSE.txt for licensing info. 4 | * 5 | * Name: rand.c 6 | * 7 | * Description: A random number generation API that keeps RNGs 8 | * per-thread to minimize contention. 9 | * 10 | * Note that this interface isn't particularly high 11 | * level: 12 | * 13 | * 1) You need to do the hashing yourself, and pass in 14 | * the value. 15 | * 16 | * 2) You just pass in a pointer to an "item" that's 17 | * expected to represent the key/item pair. 18 | * 19 | * 3) You need to do your own memory management for 20 | * the key / item pairs, if appropriate. 21 | * 22 | * Most of the implementation is inlined in the header 23 | * file, since it merely dispatches to individual 24 | * implementations. 25 | * 26 | * To try to make algorithm comparisons as fair as 27 | * possible, I try to do everything I can to 28 | * eliminate places where the OS might use a mutex, 29 | * where there might be contention among threads. 30 | * 31 | * Top of that list is malloc() -- where I recommend 32 | * addressing by linking in the hoard malloc 33 | * implementation. 34 | * 35 | * Second on that list is the random number 36 | * generator, since we use a lot of random numbers in 37 | * our testing, and would like to avoid several 38 | * things: 39 | * 40 | * 1) Calling into the kernel more than we need to 41 | * (e.g., if we were to read from /dev/urandom). 42 | * 43 | * 2) Any locks around RNG APIs. For instance, I'm 44 | * pretty sure arc4random() has such a lock on my 45 | * machine. 46 | * 47 | * 3) Holding on to too much memory. 48 | * 49 | * My basic approach is to implement ARC4 ourselves, 50 | * and keep the state on a per-thread basis, with the 51 | * seed xor'd with the bottom byte of the thread's 52 | * pthread id (just to get some variance in the 53 | * number streams; multiple threads can definitely 54 | * end up with identical streams of numbers). We 55 | * read the seed once, at initialization time, from 56 | * /dev/urandom. 57 | * 58 | * Note that ARC4 is very broken, cryptographically 59 | * (it was once considered a cryptographic RNG, but 60 | * now certainly is not), but we don't need 61 | * cryptographically strong random numbers for our 62 | * purposes. This just gets the job done with a quick 63 | * algorithm, that can be done without hitting the 64 | * kernel, after initialization. 65 | * 66 | * Author: John Viega, john@zork.org 67 | */ 68 | 69 | #include "testhat.h" 70 | 71 | #include // For open 72 | #include // For close and read 73 | #include // For memcpy 74 | 75 | static int rand_fd = 0; 76 | 77 | typedef struct { 78 | uint32_t S[256]; 79 | uint32_t x, y; 80 | } arc4_ctx; 81 | 82 | __thread arc4_ctx rng_ctx; 83 | __thread bool rand_inited = false; 84 | 85 | static void 86 | system_random(char *buf, size_t num) 87 | { 88 | if (!rand_fd) { 89 | rand_fd = open("/dev/urandom", O_RDONLY); 90 | } 91 | 92 | read(rand_fd, buf, num); 93 | 94 | return; 95 | } 96 | 97 | /* Given a "seed", initialize the per-thread rng. The seed is 98 | * effectively an RC4 key; we initialize the stream cipher, then use 99 | * the bytes as output. 100 | */ 101 | static void 102 | test_thread_init_rand(char *seed_buf) 103 | { 104 | uint32_t a, i, j = 0, k = 0; 105 | 106 | rng_ctx.x = 1; 107 | rng_ctx.y = 0; 108 | 109 | for (i = 0; i < 256; i++) { 110 | rng_ctx.S[i] = i; 111 | } 112 | 113 | for (i = 0; i < 256; i++) { 114 | a = rng_ctx.S[i]; 115 | j = (j + (seed_buf[k]) + a) & 0xff; 116 | rng_ctx.S[i] = rng_ctx.S[j]; 117 | rng_ctx.S[j] = a; 118 | ++k; 119 | if (k == HATRACK_RAND_SEED_SIZE) { 120 | k = 0; 121 | } 122 | } 123 | 124 | rand_inited = true; 125 | 126 | return; 127 | } 128 | 129 | /* Used for initializing the main thread's random number generator, 130 | * either via a 128-bit seed passed on the command line (for 131 | * repeatability purposes), or via HATRACK_RAND_SEED_SIZE random 132 | * bytes. 133 | * 134 | * Subsequent threads will ALWAYS get a new seed from the system. 135 | */ 136 | void 137 | test_init_rand(__int128_t seed) 138 | { 139 | char seed_buf[HATRACK_RAND_SEED_SIZE] = { 140 | 0, 141 | }; 142 | 143 | if (!seed) { 144 | system_random(seed_buf, HATRACK_SEED_SIZE); 145 | } 146 | else { 147 | memcpy(seed_buf, &seed, sizeof(seed)); 148 | } 149 | 150 | test_thread_init_rand(seed_buf); 151 | 152 | return; 153 | } 154 | 155 | uint32_t 156 | test_rand(void) 157 | { 158 | uint32_t out; 159 | uint8_t *p = (uint8_t *)&out; 160 | uint32_t a, b, ta, tb, ty, i; 161 | 162 | if (!rand_inited) { 163 | char seed_buf[HATRACK_RAND_SEED_SIZE]; 164 | 165 | system_random(seed_buf, HATRACK_SEED_SIZE); 166 | test_thread_init_rand(seed_buf); 167 | } 168 | 169 | a = rng_ctx.S[rng_ctx.x]; 170 | rng_ctx.y = (rng_ctx.y + a) & 0xff; 171 | b = rng_ctx.S[rng_ctx.y]; 172 | 173 | for (i = 0; i < 4; i++) { 174 | rng_ctx.S[rng_ctx.y] = a; 175 | a = (a + b) & 0xff; 176 | rng_ctx.S[rng_ctx.x] = b; 177 | rng_ctx.x = (rng_ctx.x + 1) & 0xff; 178 | ta = rng_ctx.S[rng_ctx.x]; 179 | ty = (rng_ctx.y + ta) & 0xff; 180 | tb = rng_ctx.S[ty]; 181 | *p++ = rng_ctx.S[a]; 182 | rng_ctx.y = ty; 183 | a = ta; 184 | b = tb; 185 | } 186 | 187 | return out; 188 | } 189 | 190 | void 191 | test_shuffle_array(void *arr, uint32_t num_items, uint32_t item_size) 192 | { 193 | uint8_t *first; 194 | uint8_t *p; 195 | uint8_t *swap_start; 196 | uint32_t n; 197 | uint32_t i; 198 | 199 | first = (uint8_t *)arr; 200 | p = first + (num_items * item_size); 201 | 202 | while (num_items > 0) { 203 | p = p - item_size; 204 | n = test_rand() % num_items--; 205 | 206 | swap_start = first + (n * item_size); 207 | if (swap_start == p) { 208 | continue; 209 | } 210 | 211 | /* For now, swap item_size bytes, byte by byte. Ideally, we'd 212 | * swap 128 bits at a time, until we have less than 128 bits 213 | * left. But speed isn't critical in our instance, and I 214 | * don't want to over-complicate things right now. 215 | */ 216 | for (i = 0; i < item_size; i++) { 217 | p[i] ^= swap_start[i]; 218 | swap_start[i] ^= p[i]; 219 | p[i] ^= swap_start[i]; 220 | } 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /tests/test.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright © 2021-2022 John Viega 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Name: test.c 17 | * 18 | * Description: Main file for test case running, along with 19 | * the function that pre-computes hash keys. 20 | * 21 | * 22 | * Author: John Viega, john@zork.org 23 | * 24 | */ 25 | 26 | #include "testhat.h" // Will NOT be installed, so leave in quotes. 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | 33 | hatrack_hash_t *precomputed_hashes = NULL; 34 | static uint64_t num_hashes = 0; 35 | 36 | // This is obviously meant to be run single-threaded. 37 | void 38 | precompute_hashes(uint64_t max_range) 39 | { 40 | size_t alloc_size; 41 | 42 | if (num_hashes > max_range) { 43 | return; 44 | } 45 | 46 | alloc_size = sizeof(hatrack_hash_t) * max_range; 47 | 48 | if (!precomputed_hashes) { 49 | precomputed_hashes = (hatrack_hash_t *)malloc(alloc_size); 50 | } 51 | else { 52 | precomputed_hashes 53 | = (hatrack_hash_t *)realloc(precomputed_hashes, alloc_size); 54 | } 55 | 56 | for (; num_hashes < max_range; num_hashes++) { 57 | precomputed_hashes[num_hashes] = hash_int(num_hashes); 58 | } 59 | 60 | return; 61 | } 62 | 63 | int 64 | main(int argc, char *argv[]) 65 | { 66 | config_info_t *config; 67 | 68 | config = parse_args(argc, argv); 69 | 70 | mmm_register_thread(); 71 | 72 | if (config->run_custom_test) { 73 | run_performance_test(&config->custom); 74 | } 75 | 76 | if (config->run_func_tests) { 77 | run_functional_tests(config); 78 | } 79 | 80 | if (config->run_default_tests) { 81 | run_default_tests(config); 82 | } 83 | 84 | counters_output_alltime(); 85 | 86 | printf("Press to exit.\n"); 87 | getc(stdin); 88 | 89 | return 0; 90 | } 91 | --------------------------------------------------------------------------------