├── .gitignore ├── README.md ├── build.sh ├── install_deps_aws.sh ├── meson.build └── src ├── lf_bcast.c ├── lf_bcast.h ├── lf_machine_assumptions.h ├── lf_pool.c ├── lf_pool.h ├── lf_queue.c ├── lf_queue.h ├── lf_shm.c ├── lf_shm.h ├── lf_util.h ├── meson.build ├── test_bcast_functional.c ├── test_bcast_shm.h ├── test_bcast_shm_create.c ├── test_bcast_shm_run.c ├── test_bcast_stress.c ├── test_header.h ├── test_pool_stress.c └── test_queue_functional.c /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | \#*# 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lockfree 2 | 3 | A collection of some lockfree datastructures 4 | 5 | ## Datastructures 6 | 7 | - Pool: A simple lockfree memory/object pool based on the Treiber Stack 8 | - Queue: A simple lockfree MPMC queue based very roughly on Michael&Scott queues 9 | - Bcast: A simple lockfree MPMC "broadcast-y" fan-out pub-sub queue based even more roughly on Michael&Scott queues 10 | 11 | ## Properties 12 | 13 | - Lockfree 14 | - Non-allocating (after init) 15 | - Fixed-size 16 | - Cache-efficent flat structures 17 | - Agnostic to in-process memory or shared-memory 18 | - Crash-safe (kill -9 cannot cause deadlocks, livelocks, corruption, etc) 19 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | THISDIR=$(dirname $(realpath $0)) 4 | cd $THISDIR 5 | 6 | if [ ! -d build ]; then 7 | meson build 8 | fi 9 | 10 | (cd build && ninja $@) 11 | -------------------------------------------------------------------------------- /install_deps_aws.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo yum install meson gcc 3 | -------------------------------------------------------------------------------- /meson.build: -------------------------------------------------------------------------------- 1 | project('lockfree', 'c') 2 | 3 | ################################## 4 | ## Common flags 5 | 6 | flags_c = [ 7 | '-std=c11', 8 | #'-fvisibility=hidden', 9 | ] 10 | 11 | flags_cpp = [ 12 | '-std=c++14', 13 | '-fno-exceptions', '-fno-rtti', 14 | #'-fvisibility=hidden', '-fvisibility-inlines-hidden', 15 | ] 16 | 17 | flags_common = [ 18 | '-fdiagnostics-color=always', 19 | '-D_GNU_SOURCE', 20 | '-Dtypeof=__typeof__', 21 | '-I' + meson.source_root() + '/src', 22 | '-g', '-fPIC', 23 | '-ffast-math', '-fno-associative-math', '-fno-reciprocal-math', 24 | '-fno-strict-aliasing', 25 | ] 26 | 27 | flags_warn = [ 28 | ## Warning enables 29 | '-Wall', '-Werror', '-Wextra', 30 | 31 | ## Warning disables: annoying 32 | '-Wno-unused-function', '-Wno-unused-parameter', 33 | 34 | ## Warning disables: Complains about 'Type t[1] = {{0}}' 35 | '-Wno-missing-field-initializers', 36 | 37 | ## Warning disables: Complains about 'if (val < 0 || val >= LIMIT)' when val is unsigned 38 | '-Wno-type-limits', 39 | 40 | ## Warning disables: Complains about macros that expand to empty if-else bodies 41 | '-Wno-empty-body', 42 | ] 43 | 44 | flags_release = [ 45 | ## Optimization 46 | '-O2', 47 | 48 | ## Warning disables: 49 | '-Wno-unused-variable', '-Wno-unused-but-set-variable', 50 | 51 | ## Warning disables: Seriously, no comments in comments? 52 | '-Wno-comments', 53 | ] 54 | 55 | flags_debug = [ 56 | ] 57 | 58 | flags_special = [ 59 | ] 60 | 61 | link_flags_common = [ 62 | '-fdiagnostics-color=always', 63 | #'-fvisibility=hidden', '-fvisibility-inlines-hidden', 64 | '-lpthread', 65 | ] 66 | 67 | add_global_arguments(flags_c + flags_common + flags_warn + flags_special, language : 'c') 68 | add_global_arguments(flags_cpp + flags_common + flags_warn + flags_special, language : 'cpp') 69 | 70 | add_global_link_arguments(link_flags_common, language : 'c') 71 | add_global_link_arguments(link_flags_common, language : 'cpp') 72 | 73 | ## Release 74 | add_global_arguments(flags_release, language : 'c') 75 | add_global_arguments(flags_release, language : 'cpp') 76 | 77 | ## Debug 78 | # add_global_arguments(flags_debug, language : 'c') 79 | # add_global_arguments(flags_debug, language : 'cpp') 80 | 81 | ################################## 82 | ## Definitions 83 | subdir('src') 84 | -------------------------------------------------------------------------------- /src/lf_bcast.c: -------------------------------------------------------------------------------- 1 | #include "lf_bcast.h" 2 | #include "lf_pool.h" 3 | #include "lf_util.h" 4 | 5 | #define ESTIMATED_PUBLISHERS 16 6 | 7 | struct __attribute__((aligned(CACHE_LINE_SZ))) lf_bcast 8 | { 9 | u64 depth_mask; 10 | u64 max_msg_sz; 11 | u64 head_idx; 12 | u64 tail_idx; 13 | size_t pool_off; 14 | char _pad_2[CACHE_LINE_SZ - 5*sizeof(u64) - sizeof(size_t)]; 15 | 16 | lf_ref_t slots[]; 17 | }; 18 | static_assert(sizeof(lf_bcast_t) == CACHE_LINE_SZ, ""); 19 | static_assert(alignof(lf_bcast_t) == CACHE_LINE_SZ, ""); 20 | 21 | typedef struct msg msg_t; 22 | struct __attribute__((aligned(alignof(u128)))) msg 23 | { 24 | u64 size; 25 | u8 payload[]; 26 | }; 27 | 28 | static inline lf_pool_t *get_pool(lf_bcast_t *b) { return (lf_pool_t*)((char*)b + b->pool_off); } 29 | 30 | lf_bcast_t * lf_bcast_new(size_t depth, size_t max_msg_sz) 31 | { 32 | size_t mem_sz, mem_align; 33 | lf_bcast_footprint(depth, max_msg_sz, &mem_sz, &mem_align); 34 | 35 | void *mem = NULL; 36 | int ret = posix_memalign(&mem, mem_align, mem_sz); 37 | if (ret != 0) return NULL; 38 | 39 | lf_bcast_t *bcast = lf_bcast_mem_init(mem, depth, max_msg_sz); 40 | if (!bcast) { 41 | free(mem); 42 | return NULL; 43 | } 44 | 45 | return bcast; 46 | } 47 | 48 | void lf_bcast_delete(lf_bcast_t *bcast) 49 | { 50 | free(bcast); 51 | } 52 | 53 | static void try_drop_head(lf_bcast_t *b, u64 head_idx) 54 | { 55 | // TODO AUDIT 56 | lf_ref_t head_cur = b->slots[head_idx & b->depth_mask]; 57 | if (!LF_U64_CAS(&b->head_idx, head_idx, head_idx+1)) return; 58 | 59 | // FIXME: THIS IS WHERE WE MIGHT LOSE SHARED RESOURCES 60 | u64 msg_off = head_cur.val; 61 | msg_t *msg = (msg_t*)((char*)b + msg_off); 62 | lf_pool_release(get_pool(b), msg); 63 | } 64 | 65 | bool lf_bcast_pub(lf_bcast_t *b, void * msg_buf, size_t msg_sz) 66 | { 67 | msg_t *msg = (msg_t*)lf_pool_acquire(get_pool(b)); 68 | if (!msg) return false; // out of elements 69 | u64 msg_off = (char*)msg - (char*)b; 70 | 71 | msg->size = msg_sz; 72 | memcpy(msg->payload, msg_buf, msg_sz); 73 | 74 | while (1) { 75 | u64 head_idx = b->head_idx; 76 | u64 tail_idx = b->tail_idx; 77 | lf_ref_t * tail_ptr = &b->slots[tail_idx & b->depth_mask]; 78 | lf_ref_t tail_cur = *tail_ptr; 79 | LF_BARRIER_ACQUIRE(); 80 | 81 | // Stale tail pointer? Try to advance it.. 82 | if (tail_cur.tag == tail_idx) { 83 | LF_U64_CAS(&b->tail_idx, tail_idx, tail_idx+1); 84 | LF_PAUSE(); 85 | continue; 86 | } 87 | 88 | // Stale tail_idx? Try again.. 89 | if (tail_cur.tag >= tail_idx) { 90 | LF_PAUSE(); 91 | continue; 92 | } 93 | 94 | // Slot currently used.. full.. roll off the head 95 | if (head_idx <= tail_cur.tag) { 96 | assert(head_idx == tail_cur.tag); 97 | try_drop_head(b, head_idx); 98 | LF_PAUSE(); 99 | continue; 100 | } 101 | 102 | // Otherwise, try to append the tail 103 | lf_ref_t tail_next = LF_REF_MAKE(tail_idx, msg_off); 104 | if (!LF_REF_CAS(tail_ptr, tail_cur, tail_next)) { 105 | LF_PAUSE(); 106 | continue; 107 | } 108 | 109 | // Success, try to update the tail. If we fail, it's okay. 110 | LF_U64_CAS(&b->tail_idx, tail_idx, tail_idx+1); 111 | return true; 112 | } 113 | } 114 | 115 | typedef struct sub_impl sub_impl_t; 116 | struct __attribute__((aligned(16))) sub_impl 117 | { 118 | lf_bcast_t * bcast; 119 | u64 idx; 120 | char _extra[16]; 121 | }; 122 | static_assert(sizeof(sub_impl_t) == sizeof(lf_bcast_sub_t), ""); 123 | static_assert(alignof(sub_impl_t) == alignof(lf_bcast_sub_t), ""); 124 | 125 | void lf_bcast_sub_begin(lf_bcast_sub_t *_sub, lf_bcast_t *b) 126 | { 127 | sub_impl_t *sub = (sub_impl_t*)_sub; 128 | sub->bcast = b; 129 | sub->idx = b->head_idx; 130 | } 131 | 132 | bool lf_bcast_sub_next(lf_bcast_sub_t *_sub, void * msg_buf, size_t * _out_msg_sz, size_t *_out_drops) 133 | { 134 | sub_impl_t *sub = (sub_impl_t*)_sub; 135 | lf_bcast_t *b = sub->bcast; 136 | size_t drops = 0; 137 | 138 | while (1) { 139 | if (sub->idx == b->tail_idx) return false; 140 | 141 | lf_ref_t * ref_ptr = &b->slots[sub->idx & b->depth_mask]; 142 | lf_ref_t ref = *ref_ptr; 143 | 144 | LF_BARRIER_ACQUIRE(); 145 | 146 | if (ref.tag != sub->idx) { // We've fallen behind and the message we wanted was dropped? 147 | sub->idx++; 148 | drops++; 149 | LF_PAUSE(); 150 | continue; 151 | } 152 | u64 msg_off = ref.val; 153 | msg_t *msg = (msg_t*)((char*)b + msg_off); 154 | size_t msg_sz = msg->size; 155 | if (msg_sz > b->max_msg_sz) { // Size doesn't make sense.. inconsistent 156 | LF_PAUSE(); 157 | continue; 158 | } 159 | memcpy(msg_buf, msg->payload, msg_sz); 160 | 161 | LF_BARRIER_ACQUIRE(); 162 | 163 | lf_ref_t ref2 = *ref_ptr; 164 | if (!LF_REF_EQUAL(ref, ref2)) { // Data changed while reading? Drop it. 165 | sub->idx++; 166 | drops++; 167 | LF_PAUSE(); 168 | continue; 169 | } 170 | 171 | sub->idx++; 172 | *_out_msg_sz = msg_sz; 173 | *_out_drops = drops; 174 | return true; 175 | } 176 | } 177 | 178 | void lf_bcast_footprint(size_t depth, size_t max_msg_sz, size_t *_size, size_t *_align) 179 | { 180 | size_t elt_sz = LF_ALIGN_UP(sizeof(msg_t) + max_msg_sz, alignof(u128)); 181 | size_t pool_elts = depth + ESTIMATED_PUBLISHERS; 182 | 183 | size_t pool_size, pool_align; 184 | lf_pool_footprint(pool_elts, elt_sz, &pool_size, &pool_align); 185 | 186 | size_t size = sizeof(lf_bcast_t); 187 | size += depth * sizeof(lf_ref_t); 188 | size = LF_ALIGN_UP(size, pool_align); 189 | size += pool_size; 190 | 191 | if (_size) *_size = size; 192 | if (_align) *_align = alignof(lf_bcast_t); 193 | } 194 | 195 | lf_bcast_t * lf_bcast_mem_init(void *mem, size_t depth, size_t max_msg_sz) 196 | { 197 | if (!LF_IS_POW2(depth)) return NULL; 198 | if (max_msg_sz == 0) return NULL; 199 | 200 | size_t elt_sz = LF_ALIGN_UP(sizeof(msg_t) + max_msg_sz, alignof(u128)); 201 | size_t pool_elts = depth + ESTIMATED_PUBLISHERS; 202 | 203 | size_t pool_size, pool_align; 204 | lf_pool_footprint(pool_elts, elt_sz, &pool_size, &pool_align); 205 | 206 | size_t size = sizeof(lf_bcast_t) + depth * sizeof(lf_ref_t); 207 | size_t pool_off = LF_ALIGN_UP(size, pool_align); 208 | void * pool_mem = (char*)mem + pool_off; 209 | 210 | lf_bcast_t *b = (lf_bcast_t*)mem; 211 | b->depth_mask = depth-1; 212 | b->max_msg_sz = max_msg_sz; 213 | b->head_idx = 1; /* Start from 1 because we use 0 to mean "unused" */ 214 | b->tail_idx = 1; /* Start from 1 because we use 0 to mean "unused" */ 215 | b->pool_off = pool_off; 216 | 217 | memset(b->slots, 0, depth * sizeof(lf_ref_t)); 218 | 219 | lf_pool_t *pool = lf_pool_mem_init(pool_mem, pool_elts, elt_sz); 220 | if (!pool) return NULL; 221 | assert(pool == pool_mem); 222 | 223 | return b; 224 | } 225 | 226 | lf_bcast_t * lf_bcast_mem_join(void *mem, size_t depth, size_t max_msg_sz) 227 | { 228 | lf_bcast_t *bcast = (lf_bcast_t *)mem; 229 | if (depth-1 != bcast->depth_mask) return NULL; 230 | if (max_msg_sz != bcast->max_msg_sz) return NULL; 231 | 232 | size_t elt_sz = LF_ALIGN_UP(sizeof(msg_t) + max_msg_sz, alignof(u128)); 233 | size_t pool_elts = depth + ESTIMATED_PUBLISHERS; 234 | 235 | void * pool_mem = (char*)mem + bcast->pool_off; 236 | lf_pool_t * pool = lf_pool_mem_join(pool_mem, pool_elts, elt_sz); 237 | if (!pool) return NULL; 238 | assert(pool == pool_mem); 239 | 240 | return bcast; 241 | } 242 | 243 | void lf_bcast_mem_leave(lf_bcast_t *b) 244 | { 245 | lf_pool_t *pool = get_pool(b); 246 | lf_pool_mem_leave(pool); 247 | } 248 | -------------------------------------------------------------------------------- /src/lf_bcast.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | /* Lockfree Bcast: multi-publisher broadcast to multi-consumer */ 6 | 7 | typedef struct lf_bcast lf_bcast_t; 8 | typedef struct lf_bcast_sub lf_bcast_sub_t; 9 | struct __attribute__((aligned(16))) lf_bcast_sub { char _opaque[32]; }; 10 | 11 | /* Basic API */ 12 | lf_bcast_t * lf_bcast_new(size_t depth, size_t max_msg_sz); 13 | void lf_bcast_delete(lf_bcast_t *bcast); 14 | bool lf_bcast_pub(lf_bcast_t *b, void * msg, size_t msg_sz); 15 | void lf_bcast_sub_begin(lf_bcast_sub_t *sub, lf_bcast_t *b); 16 | bool lf_bcast_sub_next(lf_bcast_sub_t *sub, void * msg_buf, size_t * _out_msg_sz, size_t *_out_drops); 17 | 18 | /* Advanced API */ 19 | void lf_bcast_footprint(size_t depth, size_t max_msg_sz, size_t *size, size_t *align); 20 | lf_bcast_t * lf_bcast_mem_init(void *mem, size_t depth, size_t max_msg_sz); 21 | lf_bcast_t * lf_bcast_mem_join(void *mem, size_t depth, size_t max_msg_sz); 22 | void lf_bcast_mem_leave(lf_bcast_t *lf_bcast); 23 | -------------------------------------------------------------------------------- /src/lf_machine_assumptions.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | // Machine is byte-addressable 6 | static_assert(CHAR_BIT == 8, ""); 7 | 8 | // Sizes / Offsets / Alignments are 64-bit 9 | static_assert(sizeof(size_t) == 8, ""); 10 | 11 | // Assuming we have 64-bit pointers 12 | static_assert(sizeof(void*) == 8, ""); 13 | 14 | // Long and Long Long are the same and 64-bits 15 | static_assert(sizeof(long) == sizeof(long long), ""); 16 | static_assert(sizeof(long) == 8, ""); 17 | -------------------------------------------------------------------------------- /src/lf_pool.c: -------------------------------------------------------------------------------- 1 | #include "lf_pool.h" 2 | #include "lf_util.h" 3 | 4 | struct __attribute__((aligned(CACHE_LINE_SZ))) lf_pool 5 | { 6 | size_t num_elts; 7 | size_t elt_sz; 8 | u64 tag_next; 9 | u64 _pad1; 10 | lf_ref_t head; 11 | char _pad2[CACHE_LINE_SZ - 2*sizeof(size_t) - 2*sizeof(u64) - sizeof(lf_ref_t)]; 12 | 13 | char mem[]; 14 | }; 15 | static_assert(sizeof(lf_pool_t) == CACHE_LINE_SZ, ""); 16 | static_assert(alignof(lf_pool_t) == CACHE_LINE_SZ, ""); 17 | 18 | lf_pool_t * lf_pool_new(size_t num_elts, size_t elt_sz) 19 | { 20 | size_t mem_sz, mem_align; 21 | lf_pool_footprint(num_elts, elt_sz, &mem_sz, &mem_align); 22 | 23 | void *mem = NULL; 24 | int ret = posix_memalign(&mem, mem_align, mem_sz); 25 | if (ret != 0) return NULL; 26 | 27 | lf_pool_t *pool = lf_pool_mem_init(mem, num_elts, elt_sz); 28 | if (!pool) { 29 | free(mem); 30 | return NULL; 31 | } 32 | 33 | return pool; 34 | } 35 | 36 | void lf_pool_delete(lf_pool_t *pool) 37 | { 38 | free(pool); 39 | } 40 | 41 | void *lf_pool_acquire(lf_pool_t *pool) 42 | { 43 | while (1) { 44 | lf_ref_t cur = pool->head; 45 | if (LF_REF_IS_NULL(cur)) return NULL; 46 | 47 | u64 elt_off = cur.val; 48 | lf_ref_t * elt = (lf_ref_t*)((char*)pool + elt_off); 49 | lf_ref_t next = *elt; 50 | 51 | if (!LF_REF_CAS(&pool->head, cur, next)) { 52 | LF_PAUSE(); 53 | continue; 54 | } 55 | return elt; 56 | } 57 | } 58 | 59 | void lf_pool_release(lf_pool_t *pool, void *elt) 60 | { 61 | u64 elt_off = (u64)((char*)elt - (char*)pool); 62 | u64 tag = LF_ATOMIC_INC(&pool->tag_next); 63 | lf_ref_t next = LF_REF_MAKE(tag, elt_off); 64 | 65 | while (1) { 66 | lf_ref_t cur = pool->head; 67 | *(lf_ref_t*)elt = cur; 68 | 69 | if (!LF_REF_CAS(&pool->head, cur, next)) { 70 | LF_PAUSE(); 71 | continue; 72 | } 73 | return; 74 | } 75 | } 76 | 77 | void lf_pool_footprint(size_t num_elts, size_t elt_sz, size_t *_size, size_t *_align) 78 | { 79 | elt_sz = LF_ALIGN_UP(elt_sz, alignof(u128)); 80 | 81 | if (_size) *_size = sizeof(lf_pool_t) + elt_sz * num_elts; 82 | if (_align) *_align = alignof(lf_pool_t); 83 | } 84 | 85 | lf_pool_t * lf_pool_mem_init(void *mem, size_t num_elts, size_t elt_sz) 86 | { 87 | if (elt_sz == 0) return NULL; 88 | elt_sz = LF_ALIGN_UP(elt_sz, alignof(u128)); 89 | 90 | lf_pool_t *lf_pool = (lf_pool_t *)mem; 91 | lf_pool->num_elts = num_elts; 92 | lf_pool->elt_sz = elt_sz; 93 | lf_pool->tag_next = 0; 94 | lf_pool->head = LF_REF_NULL; 95 | 96 | char *ptr = lf_pool->mem + num_elts * elt_sz; 97 | for (size_t i = num_elts; i > 0; i--) { 98 | ptr -= elt_sz; 99 | lf_pool_release(lf_pool, ptr); 100 | } 101 | 102 | return lf_pool; 103 | } 104 | 105 | lf_pool_t * lf_pool_mem_join(void *mem, size_t num_elts, size_t elt_sz) 106 | { 107 | lf_pool_t *pool = (lf_pool_t *)mem; 108 | if (num_elts != pool->num_elts) return NULL; 109 | if (elt_sz != pool->elt_sz) return NULL; 110 | return pool; 111 | } 112 | 113 | void lf_pool_mem_leave(lf_pool_t *lf_pool) 114 | { 115 | /* no-op at the moment */ 116 | } 117 | -------------------------------------------------------------------------------- /src/lf_pool.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | /* Lockfree Pool */ 5 | 6 | typedef struct lf_pool lf_pool_t; 7 | 8 | /* Basic API */ 9 | lf_pool_t * lf_pool_new(size_t num_elts, size_t elt_sz); 10 | void lf_pool_delete(lf_pool_t *lf_pool); 11 | void * lf_pool_acquire(lf_pool_t *lf_pool); 12 | void lf_pool_release(lf_pool_t *lf_pool, void *elt); 13 | 14 | /* Advanced API */ 15 | void lf_pool_footprint(size_t num_elts, size_t elt_sz, size_t *size, size_t *align); 16 | lf_pool_t * lf_pool_mem_init(void *mem, size_t num_elts, size_t elt_sz); 17 | lf_pool_t * lf_pool_mem_join(void *mem, size_t num_elts, size_t elt_sz); 18 | void lf_pool_mem_leave(lf_pool_t *lf_pool); 19 | -------------------------------------------------------------------------------- /src/lf_queue.c: -------------------------------------------------------------------------------- 1 | #include "lf_queue.h" 2 | #include "lf_util.h" 3 | 4 | /* TODO: Explain the design of this queue */ 5 | 6 | struct __attribute__((aligned(CACHE_LINE_SZ))) lf_queue 7 | { 8 | u64 depth_mask; 9 | u64 head_idx; 10 | u64 tail_idx; 11 | char _pad_2[CACHE_LINE_SZ - 3*sizeof(u64)]; 12 | 13 | lf_ref_t slots[]; 14 | }; 15 | static_assert(sizeof(lf_queue_t) == CACHE_LINE_SZ, ""); 16 | static_assert(alignof(lf_queue_t) == CACHE_LINE_SZ, ""); 17 | 18 | lf_queue_t * lf_queue_new(size_t depth) 19 | { 20 | size_t mem_sz, mem_align; 21 | lf_queue_footprint(depth, &mem_sz, &mem_align); 22 | 23 | void *mem = NULL; 24 | int ret = posix_memalign(&mem, mem_align, mem_sz); 25 | if (ret != 0) return NULL; 26 | 27 | lf_queue_t *queue = lf_queue_mem_init(mem, depth); 28 | if (!queue) { 29 | free(mem); 30 | return NULL; 31 | } 32 | 33 | return queue; 34 | } 35 | 36 | void lf_queue_delete(lf_queue_t *q) 37 | { 38 | free(q); 39 | } 40 | 41 | bool lf_queue_enqueue(lf_queue_t *q, uint64_t val) 42 | { 43 | while (1) { 44 | u64 head_idx = q->head_idx; 45 | u64 tail_idx = q->tail_idx; 46 | lf_ref_t * tail_ptr = &q->slots[tail_idx & q->depth_mask]; 47 | lf_ref_t tail_cur = *tail_ptr; 48 | 49 | // Stale tail pointer? Try to advance it.. 50 | if (tail_cur.tag == tail_idx) { 51 | LF_U64_CAS(&q->tail_idx, tail_idx, tail_idx+1); 52 | LF_PAUSE(); 53 | continue; 54 | } 55 | 56 | // Slot currently used? 57 | if (head_idx <= tail_cur.tag) return false; // Full! 58 | 59 | // Otherwise, try to append the tail 60 | lf_ref_t tail_next = LF_REF_MAKE(tail_idx, val); 61 | if (!LF_REF_CAS(tail_ptr, tail_cur, tail_next)) { 62 | continue; 63 | LF_PAUSE(); 64 | } 65 | 66 | // Success, try to update the tail. If we fail, it's okay. 67 | LF_U64_CAS(&q->tail_idx, tail_idx, tail_idx+1); 68 | return true; 69 | } 70 | } 71 | 72 | bool lf_queue_dequeue(lf_queue_t *q, uint64_t *_val) 73 | { 74 | while (1) { 75 | u64 head_idx = q->head_idx; 76 | u64 tail_idx = q->tail_idx; 77 | lf_ref_t * head_ptr = &q->slots[head_idx & q->depth_mask]; 78 | lf_ref_t head_cur = *head_ptr; 79 | 80 | // When head reaches tail, we either have an empty queue or a stale tail 81 | if (head_idx == tail_idx) { 82 | if (head_cur.tag != head_idx) return false; // Empty! 83 | // Head seems valid, but the tail is stale.. try to update it 84 | LF_U64_CAS(&q->tail_idx, tail_idx, tail_idx); 85 | LF_PAUSE(); 86 | continue; 87 | } 88 | 89 | // Try to pop head 90 | // IMPORTANT: On success, the head node is considered "deallocated" 91 | // which means it is invalid for us to access it in any way. Another 92 | // operation may reuse it before our access and we could read stale data 93 | u64 val = head_cur.val; 94 | if (!LF_U64_CAS(&q->head_idx, head_idx, head_idx+1)) { 95 | LF_PAUSE(); 96 | continue; 97 | } 98 | 99 | // Success! 100 | *_val = val; 101 | return true; 102 | } 103 | } 104 | 105 | void lf_queue_footprint(size_t depth, size_t *_size, size_t *_align) 106 | { 107 | if (_size) *_size = sizeof(lf_queue_t) + depth * sizeof(lf_ref_t); 108 | if (_align) *_align = alignof(lf_queue_t); 109 | } 110 | 111 | lf_queue_t *lf_queue_mem_init(void *mem, size_t depth) 112 | { 113 | if (!LF_IS_POW2(depth)) return NULL; 114 | 115 | lf_queue_t *queue = (lf_queue_t *)mem; 116 | queue->depth_mask = depth-1; 117 | queue->head_idx = 1; /* Start from 1 because we use 0 to mean "unused" */ 118 | queue->tail_idx = 1; /* Start from 1 because we use 0 to mean "unused" */ 119 | 120 | memset(queue->slots, 0, depth * sizeof(lf_ref_t)); 121 | 122 | return queue; 123 | } 124 | 125 | lf_queue_t *lf_queue_mem_join(void *mem, size_t depth) 126 | { 127 | lf_queue_t *queue = (lf_queue_t *)mem; 128 | if (depth-1 != queue->depth_mask) return NULL; 129 | return queue; 130 | } 131 | 132 | void lf_queue_mem_leave(lf_queue_t *lf_queue) 133 | { 134 | /* no-op at the moment */ 135 | } 136 | -------------------------------------------------------------------------------- /src/lf_queue.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | 6 | /* Lockfree Queue: MPMC */ 7 | 8 | typedef struct lf_queue lf_queue_t; 9 | 10 | /* Basic API */ 11 | lf_queue_t * lf_queue_new(size_t depth); 12 | void lf_queue_delete(lf_queue_t *q); 13 | bool lf_queue_enqueue(lf_queue_t *q, uint64_t val); 14 | bool lf_queue_dequeue(lf_queue_t *q, uint64_t *_val); 15 | 16 | /* Advanced API */ 17 | void lf_queue_footprint(size_t depth, size_t *size, size_t *align); 18 | lf_queue_t * lf_queue_mem_init(void *mem, size_t depth); 19 | lf_queue_t * lf_queue_mem_join(void *mem, size_t depth); 20 | void lf_queue_mem_leave(lf_queue_t *lf_queue); 21 | -------------------------------------------------------------------------------- /src/lf_shm.c: -------------------------------------------------------------------------------- 1 | #include "lf_shm.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | bool lf_shm_create(const char *name, size_t size) 8 | { 9 | int shm_fd = shm_open(name, O_CREAT|O_RDWR, 0644); 10 | if (shm_fd < 0) return false; 11 | ftruncate(shm_fd, 0); 12 | ftruncate(shm_fd, size); 13 | close(shm_fd); 14 | 15 | return true; 16 | } 17 | 18 | void lf_shm_remove(const char *name) 19 | { 20 | shm_unlink(name); 21 | } 22 | 23 | void * lf_shm_open(const char *name, size_t * _opt_size) 24 | { 25 | int shm_fd = shm_open(name, O_CREAT|O_RDWR, 0755); 26 | if (shm_fd < 0) return NULL; 27 | 28 | struct stat st[1]; 29 | if (0 != fstat(shm_fd, st)) return NULL; 30 | size_t size = st->st_size; 31 | 32 | void *mem = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED, shm_fd, 0); 33 | if (mem == MAP_FAILED) return NULL; 34 | 35 | close(shm_fd); 36 | if (_opt_size) *_opt_size = size; 37 | return mem; 38 | } 39 | 40 | void lf_shm_close(void *shm, size_t size) 41 | { 42 | munmap(shm, size); 43 | } 44 | -------------------------------------------------------------------------------- /src/lf_shm.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | 5 | bool lf_shm_create(const char *name, size_t size); 6 | void lf_shm_remove(const char *name); 7 | void * lf_shm_open(const char *name, size_t * _opt_size); 8 | void lf_shm_close(void *shm, size_t size); 9 | -------------------------------------------------------------------------------- /src/lf_util.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "lf_machine_assumptions.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #define CACHE_LINE_SZ 128 10 | 11 | typedef uint8_t u8; 12 | typedef uint16_t u16; 13 | typedef uint32_t u32; 14 | typedef uint64_t u64; 15 | typedef unsigned __int128 u128; 16 | typedef int8_t i8; 17 | typedef int16_t i16; 18 | typedef int32_t i32; 19 | typedef int64_t i64; 20 | typedef __int128 i128; 21 | 22 | typedef struct lf_ref lf_ref_t; 23 | struct lf_ref 24 | { 25 | union { 26 | struct { 27 | u64 val; 28 | u64 tag; 29 | }; 30 | unsigned __int128 u128; 31 | }; 32 | }; 33 | static_assert(sizeof(lf_ref_t) == sizeof(unsigned __int128), ""); 34 | static_assert(alignof(lf_ref_t) == alignof(unsigned __int128), ""); 35 | 36 | #define LF_REF_MAKE(_tag, _val) ({ lf_ref_t r; r.val = _val; r.tag = _tag; r; }) 37 | #define LF_REF_NULL ({ lf_ref_t r = {}; r; }) 38 | #define LF_REF_IS_NULL(ref) ((ref).u128 == 0) 39 | #define LF_REF_EQUAL(a, b) ((a).u128 == (b).u128) 40 | 41 | #define LF_REF_CAS(ptr, last, next) _impl_lf_ref_cas(ptr, last, next) 42 | #define LF_U64_CAS(ptr, last, next) _impl_lf_u64_cas(ptr, last, next) 43 | 44 | #define LF_ATOMIC_INC(ptr) __sync_add_and_fetch(ptr, 1); 45 | #define LF_ATOMIC_LOAD_ACQUIRE(ptr) ({ typeof(*ptr) ret; __atomic_load(ptr, &ret, __ATOMIC_ACQUIRE); ret; }) 46 | #define LF_BARRIER_ACQUIRE() __atomic_thread_fence(__ATOMIC_ACQUIRE) 47 | #define LF_BARRIER_RELEASE() __atomic_thread_fence(__ATOMIC_RELEASE) 48 | #define LF_BARRIER_FULL() __atomic_thread_fence(__ATOMIC_SEQ_CST) 49 | 50 | #define LF_ALIGN_UP(s, a) (((s)+((a)-1))&~((a)-1)) 51 | #define LF_IS_POW2(s) ((!!(s))&(!((s)&((s)-1)))) 52 | 53 | #if defined(__arm__) 54 | # define LF_PAUSE() __yield() 55 | #elif defined(__x86_64__) 56 | # define LF_PAUSE() __asm__ __volatile__("pause" ::: "memory") 57 | #else 58 | # define LF_PAUSE() 59 | #endif 60 | 61 | static inline __attribute__((always_inline)) bool _impl_lf_ref_cas(lf_ref_t *ptr, lf_ref_t prev, lf_ref_t next) 62 | { 63 | return __sync_bool_compare_and_swap(&ptr->u128, prev.u128, next.u128); 64 | } 65 | 66 | static inline __attribute__((always_inline)) bool _impl_lf_u64_cas(u64 *ptr, u64 prev, u64 next) 67 | { 68 | return __sync_bool_compare_and_swap(ptr, prev, next); 69 | } 70 | -------------------------------------------------------------------------------- /src/meson.build: -------------------------------------------------------------------------------- 1 | 2 | SRC = [ 3 | 'lf_pool.c', 4 | 'lf_queue.c', 5 | 'lf_bcast.c', 6 | 'lf_shm.c', 7 | ] 8 | 9 | liblockfree = static_library('lockfree', SRC) 10 | 11 | executable('test_pool_stress', 'test_pool_stress.c', link_with: liblockfree) 12 | executable('test_queue_functional', 'test_queue_functional.c', link_with: liblockfree) 13 | #executable('test_queue_stress', 'test_queue_stress.c', link_with: liblockfree) 14 | 15 | executable('test_bcast_functional', 'test_bcast_functional.c', link_with: liblockfree) 16 | executable('test_bcast_stress', 'test_bcast_stress.c', link_with: liblockfree) 17 | executable('test_bcast_shm_create', 'test_bcast_shm_create.c', link_with: liblockfree) 18 | executable('test_bcast_shm_run', 'test_bcast_shm_run.c', link_with: liblockfree) 19 | -------------------------------------------------------------------------------- /src/test_bcast_functional.c: -------------------------------------------------------------------------------- 1 | #include "lf_bcast.h" 2 | #include "test_header.h" 3 | 4 | static bool pub(lf_bcast_t *b, u64 val) 5 | { 6 | return lf_bcast_pub(b, &val, sizeof(val)); 7 | } 8 | 9 | static bool sub_next(lf_bcast_sub_t *sub, u64 *val) 10 | { 11 | size_t msg_sz; 12 | size_t drops; 13 | bool success = lf_bcast_sub_next(sub, val, &msg_sz, &drops); 14 | if (!success) return success; 15 | assert(msg_sz == sizeof(u64)); 16 | return true; 17 | } 18 | 19 | static void test_simple(void) 20 | { 21 | u64 val; 22 | 23 | lf_bcast_t *b = lf_bcast_new(2, sizeof(val)); 24 | if (!b) FAIL("Failed to allocate bcast"); 25 | 26 | REQUIRE(pub(b, 2)); 27 | REQUIRE(pub(b, 3)); 28 | 29 | lf_bcast_sub_t sub_1[1]; 30 | lf_bcast_sub_begin(sub_1, b); 31 | REQUIRE(sub_next(sub_1, &val) && val == 2); 32 | REQUIRE(sub_next(sub_1, &val) && val == 3); 33 | REQUIRE(!sub_next(sub_1, &val)); 34 | 35 | lf_bcast_sub_t sub_2[1]; 36 | lf_bcast_sub_begin(sub_2, b); 37 | REQUIRE(sub_next(sub_2, &val) && val == 2); 38 | REQUIRE(sub_next(sub_2, &val) && val == 3); 39 | REQUIRE(!sub_next(sub_2, &val)); 40 | 41 | REQUIRE(pub(b, 4)); 42 | 43 | REQUIRE(sub_next(sub_1, &val) && val == 4); 44 | REQUIRE(!sub_next(sub_1, &val)); 45 | REQUIRE(sub_next(sub_2, &val) && val == 4); 46 | REQUIRE(!sub_next(sub_2, &val)); 47 | 48 | lf_bcast_sub_t sub_3[1]; 49 | lf_bcast_sub_begin(sub_3, b); 50 | REQUIRE(sub_next(sub_3, &val) && val == 3); 51 | REQUIRE(sub_next(sub_3, &val) && val == 4); 52 | REQUIRE(!sub_next(sub_3, &val)); 53 | 54 | lf_bcast_delete(b); 55 | } 56 | 57 | int main() 58 | { 59 | test_simple(); 60 | return 0; 61 | } 62 | -------------------------------------------------------------------------------- /src/test_bcast_shm.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "lf_shm.h" 3 | #include "lf_bcast.h" 4 | #include "lf_util.h" 5 | #include "test_header.h" 6 | #include 7 | 8 | #define DEPTH 1024 9 | #define MAX_MSG_SZ 1024 10 | 11 | static inline size_t region_size(void) 12 | { 13 | size_t size, align; 14 | lf_bcast_footprint(DEPTH, MAX_MSG_SZ, &size, &align); 15 | 16 | size_t pagesize = getpagesize(); 17 | assert(LF_IS_POW2(pagesize)); 18 | return LF_ALIGN_UP(size, pagesize); 19 | } 20 | -------------------------------------------------------------------------------- /src/test_bcast_shm_create.c: -------------------------------------------------------------------------------- 1 | #include "test_bcast_shm.h" 2 | 3 | int main(int argc, char *argv[]) 4 | { 5 | if (argc != 2) { 6 | fprintf(stderr, "usage: %s \n", argv[0]); 7 | return 1; 8 | } 9 | const char *name = argv[1]; 10 | 11 | size_t size = region_size(); 12 | if (!lf_shm_create(name, size)) { 13 | fprintf(stderr, "ERROR: Failed to create shm region '%s'\n", name); 14 | return 2; 15 | } 16 | 17 | void *mem = lf_shm_open(name, NULL); 18 | if (!mem) { 19 | fprintf(stderr, "ERROR: Failed to open shm region '%s'\n", name); 20 | return 3; 21 | } 22 | 23 | if (!lf_bcast_mem_init(mem, DEPTH, MAX_MSG_SZ)) { 24 | fprintf(stderr, "ERROR: Failed to init shm region '%s'\n", name); 25 | return 4; 26 | } 27 | 28 | lf_shm_close(mem, size); 29 | return 0; 30 | } 31 | -------------------------------------------------------------------------------- /src/test_bcast_shm_run.c: -------------------------------------------------------------------------------- 1 | #include "test_bcast_shm.h" 2 | #include 3 | 4 | #define MAX_RECV 128 5 | 6 | typedef struct recv_state recv_state_t; 7 | struct recv_state 8 | { 9 | u32 cnt_start; 10 | u32 cnt_end; 11 | }; 12 | 13 | static void report_stats(recv_state_t *recv, size_t n) 14 | { 15 | printf("STATS:\n"); 16 | for (size_t i = 0; i < n; i++) { 17 | recv_state_t *r = &recv[i]; 18 | if (!r->cnt_start) continue; 19 | printf(" %5zu | [%u, %u] (%u)\n", i, r->cnt_start, r->cnt_end, r->cnt_end - r->cnt_start); 20 | r->cnt_start = 0; 21 | } 22 | } 23 | 24 | static void exec_operation_pub(lf_bcast_t *b, u32 id) 25 | { 26 | u64 count = (u64)id << 32; 27 | 28 | i64 dt_acc = 0; 29 | size_t dt_count = 0; 30 | 31 | while (1) { 32 | count++; 33 | i64 start_tm = wallclock(); 34 | bool success = lf_bcast_pub(b, &count, sizeof(count)); 35 | dt_acc += wallclock() - start_tm; 36 | dt_count++; 37 | if (!success) printf("WARN: Failed to publish!\n"); 38 | if (dt_count == 10000) { 39 | printf("Stats: %lu nanos / pub\n", (long)(dt_acc/dt_count)); 40 | dt_acc = 0; 41 | dt_count = 0; 42 | } 43 | //usleep(1e6); //100); 44 | usleep(100); 45 | } 46 | } 47 | 48 | static void exec_operation_sub(lf_bcast_t *b) 49 | { 50 | lf_bcast_sub_t sub[1]; 51 | lf_bcast_sub_begin(sub, b); 52 | 53 | recv_state_t recv[MAX_RECV] = {}; 54 | 55 | char msg[MAX_MSG_SZ]; 56 | size_t msg_sz; 57 | size_t drops; 58 | 59 | i64 start_tm = wallclock(); 60 | while (1) { 61 | i64 now = wallclock(); 62 | if (now - start_tm > (i64)1e9) { 63 | report_stats(recv, ARRAY_SIZE(recv)); 64 | start_tm = now; 65 | } 66 | 67 | if (!lf_bcast_sub_next(sub, msg, &msg_sz, &drops)) continue; 68 | assert(msg_sz == sizeof(u64)); 69 | 70 | u32 id, cnt; 71 | memcpy(&cnt, msg, sizeof(u32)); 72 | memcpy(&id, msg+sizeof(u32), sizeof(u32)); 73 | 74 | if (id < ARRAY_SIZE(recv)) { 75 | recv_state_t *r = &recv[id]; 76 | if (!r->cnt_start) r->cnt_start = cnt; 77 | r->cnt_end = cnt; 78 | } 79 | } 80 | } 81 | 82 | int main(int argc, char *argv[]) 83 | { 84 | if (argc < 3) { 85 | fprintf(stderr, "usage: %s []\n", argv[0]); 86 | return 1; 87 | } 88 | const char *name = argv[1]; 89 | const char *oper = argv[2]; 90 | u32 id = argc > 3 ? atoi(argv[3]) : 0; 91 | 92 | size_t size = 0; 93 | void *mem = lf_shm_open(name, &size); 94 | if (!mem) { 95 | fprintf(stderr, "ERROR: Failed to open shm region '%s'\n", name); 96 | return 2; 97 | } 98 | 99 | size_t exp_size = region_size(); 100 | if (size != exp_size) { 101 | fprintf(stderr, "ERROR: Region size mismatches expected size\n"); 102 | return 3; 103 | } 104 | 105 | lf_bcast_t * b = lf_bcast_mem_join(mem, DEPTH, MAX_MSG_SZ); 106 | if (!b) { 107 | fprintf(stderr, "ERROR: Failed to join shm region '%s'\n", name); 108 | return 4; 109 | } 110 | 111 | if (0) {} 112 | else if (0 == strcmp(oper, "pub")) exec_operation_pub(b, id); 113 | else if (0 == strcmp(oper, "sub")) exec_operation_sub(b); 114 | else { 115 | fprintf(stderr, "ERROR: Unknown operation: '%s'\n", oper); 116 | return 5; 117 | } 118 | 119 | lf_bcast_mem_leave(b); 120 | lf_shm_close(mem, size); 121 | return 0; 122 | } 123 | -------------------------------------------------------------------------------- /src/test_bcast_stress.c: -------------------------------------------------------------------------------- 1 | #include "lf_bcast.h" 2 | #include "test_header.h" 3 | 4 | #define MAX_THREADS 128 5 | 6 | typedef struct thread_state thread_state_t; 7 | struct thread_state 8 | { 9 | pthread_t thread; 10 | lf_bcast_t * bcast; 11 | 12 | bool pub; 13 | u32 pub_id; 14 | size_t pub_msgs; 15 | size_t sub_msgs; 16 | 17 | size_t n_msgs; 18 | size_t n_drops; 19 | i64 dt; 20 | }; 21 | 22 | static void thread_func_pub(thread_state_t *t) 23 | { 24 | lf_bcast_t *b = t->bcast; 25 | u64 msg = (u64)t->pub_id << 32; 26 | i64 start_time = wallclock(); 27 | 28 | for (size_t i = 0; i < t->pub_msgs; i++) { 29 | msg++; 30 | bool success = lf_bcast_pub(b, &msg, sizeof(msg)); 31 | assert(success); 32 | t->n_msgs++; 33 | } 34 | 35 | t->dt = wallclock() - start_time; 36 | } 37 | 38 | static void thread_func_sub(thread_state_t *t) 39 | { 40 | lf_bcast_t *b = t->bcast; 41 | i64 start_time = wallclock(); 42 | 43 | u64 last_msg[MAX_THREADS] = {}; 44 | 45 | lf_bcast_sub_t sub[1]; 46 | lf_bcast_sub_begin(sub, b); 47 | 48 | u64 msg; 49 | size_t msg_sz; 50 | size_t drops; 51 | for (size_t i = 0; i < (size_t)1e9; i++) { 52 | if (!lf_bcast_sub_next(sub, &msg, &msg_sz, &drops)) continue; 53 | assert(msg_sz == sizeof(msg)); 54 | 55 | u32 pub_id = msg>>32; 56 | assert(pub_id < MAX_THREADS); 57 | assert(msg > last_msg[pub_id]); 58 | last_msg[pub_id] = msg; 59 | 60 | t->n_msgs++; 61 | t->n_drops += drops; 62 | if (t->n_msgs + t->n_drops == t->sub_msgs) break; // DONE! 63 | } 64 | 65 | t->dt = wallclock() - start_time; 66 | } 67 | 68 | static void * thread_func(void *usr) 69 | { 70 | thread_state_t *t = (thread_state_t*)usr; 71 | if (t->pub) thread_func_pub(t); 72 | else thread_func_sub(t); 73 | return NULL; 74 | } 75 | 76 | static void run_test(const char *test_name, size_t num_pub, size_t num_sub, size_t num_elts) 77 | { 78 | lf_bcast_t *b = lf_bcast_new(num_elts, sizeof(u64)); 79 | if (!b) FAIL("Failed to create new bcast"); 80 | 81 | size_t pub_msgs = (size_t)1e5; 82 | size_t sub_msgs = num_pub * pub_msgs; 83 | 84 | assert(num_sub < MAX_THREADS); 85 | thread_state_t sub_threads[MAX_THREADS] = {{}}; 86 | for (size_t i = 0; i < num_sub; i++) { 87 | thread_state_t *t = &sub_threads[i]; 88 | t->bcast = b; 89 | t->pub = false; 90 | t->sub_msgs = sub_msgs; 91 | pthread_create(&t->thread, NULL, thread_func, t); 92 | } 93 | 94 | assert(num_pub < MAX_THREADS); 95 | thread_state_t pub_threads[MAX_THREADS] = {{}}; 96 | for (size_t i = 0; i < num_pub; i++) { 97 | thread_state_t *t = &pub_threads[i]; 98 | t->bcast = b; 99 | t->pub = true; 100 | t->pub_id = (u32)i; 101 | t->pub_msgs = pub_msgs; 102 | pthread_create(&t->thread, NULL, thread_func, t); 103 | } 104 | 105 | for (size_t i = 0; i < num_sub; i++) { 106 | pthread_join(sub_threads[i].thread, NULL); 107 | } 108 | for (size_t i = 0; i < num_pub; i++) { 109 | pthread_join(pub_threads[i].thread, NULL); 110 | } 111 | 112 | lf_bcast_delete(b); 113 | 114 | // Report stats: 115 | printf("Test: %s\n", test_name); 116 | for (size_t i = 0; i < num_sub; i++) { 117 | thread_state_t *t = &sub_threads[i]; 118 | printf(" Sub Thread %zu | n_msgs: %7zu n_drops: %7zu | %.f nanos/msg\n", i, t->n_msgs, t->n_drops, (double)t->dt/t->n_msgs); 119 | } 120 | for (size_t i = 0; i < num_pub; i++) { 121 | thread_state_t *t = &pub_threads[i]; 122 | printf(" Pub Thread %zu | n_msgs: %7zu n_drops: %7zu | %.f nanos/msg\n", i, t->n_msgs, t->n_drops, (double)t->dt/t->n_msgs); 123 | } 124 | } 125 | 126 | int main() 127 | { 128 | // Stress publishing, rolling around with lots of contention 129 | run_test("1pub0sub", 1, 0, 128); 130 | run_test("2pub0sub", 2, 0, 128); 131 | run_test("4pub0sub", 4, 0, 128); 132 | run_test("4pub0sub", 8, 0, 128); 133 | printf("\n"); 134 | 135 | // Stress publishing with 1 sub 136 | run_test("1pub1sub", 1, 1, 2048); 137 | run_test("2pub1sub", 2, 1, 2048); 138 | run_test("4pub1sub", 4, 1, 2048); 139 | run_test("8pub1sub", 8, 1, 2048); 140 | printf("\n"); 141 | 142 | // Stress publishing with 2 subs 143 | run_test("1pub2sub", 1, 2, 2048); 144 | run_test("2pub2sub", 2, 2, 2048); 145 | run_test("4pub2sub", 4, 2, 2048); 146 | run_test("8pub2sub", 8, 2, 2048); 147 | printf("\n"); 148 | 149 | // Stress subscribing 150 | run_test("1pub1sub", 1, 1, 2048); 151 | run_test("1pub2sub", 1, 2, 2048); 152 | run_test("1pub4sub", 1, 4, 2048); 153 | run_test("1pub8sub", 1, 8, 2048); 154 | printf("\n"); 155 | 156 | return 0; 157 | } 158 | -------------------------------------------------------------------------------- /src/test_header.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | typedef uint8_t u8; 10 | typedef uint16_t u16; 11 | typedef uint32_t u32; 12 | typedef uint64_t u64; 13 | typedef int8_t i8; 14 | typedef int16_t i16; 15 | typedef int32_t i32; 16 | typedef int64_t i64; 17 | 18 | static inline i64 wallclock(void) 19 | { 20 | struct timespec ts[1] = {{}}; 21 | clock_gettime(CLOCK_REALTIME, ts); 22 | return (i64)ts->tv_sec * 1000000000ul + (i64)ts->tv_nsec; 23 | } 24 | 25 | #define FAIL(...) do { fprintf(stderr, "FAIL: "); fprintf(stderr, __VA_ARGS__); fprintf(stderr, "\n"); abort(); } while(0) 26 | 27 | #define REQUIRE(expr) do {\ 28 | if (!(expr)) FAIL("REQUIRE FAILED AT %s:%d WITH '%s'", __FUNCTION__, __LINE__, #expr); \ 29 | } while(0) 30 | 31 | #define ARRAY_SIZE(arr) (sizeof(arr)/sizeof((arr)[0])) 32 | -------------------------------------------------------------------------------- /src/test_pool_stress.c: -------------------------------------------------------------------------------- 1 | #include "lf_pool.h" 2 | #include "test_header.h" 3 | 4 | typedef struct thread_state thread_state_t; 5 | struct thread_state 6 | { 7 | pthread_t thread; 8 | lf_pool_t *pool; 9 | size_t n_attempts; 10 | size_t n_success; 11 | i64 dt; 12 | }; 13 | 14 | static void * thread_func(void *usr) 15 | { 16 | thread_state_t *t = (thread_state_t*)usr; 17 | 18 | lf_pool_t *p = t->pool; 19 | i64 start_time = wallclock(); 20 | for (size_t i = 0; i < (size_t)1e5; i++) { 21 | void *elt = lf_pool_acquire(p); 22 | t->n_attempts++; 23 | if (elt) { 24 | t->n_success++; 25 | lf_pool_release(p, elt); 26 | } 27 | } 28 | t->dt = wallclock() - start_time; 29 | return NULL; 30 | } 31 | 32 | #define MAX_THREADS 128 33 | static void run_test(const char *test_name, size_t num_threads, size_t num_elts, size_t elt_sz) 34 | { 35 | lf_pool_t *p = lf_pool_new(num_elts, elt_sz); 36 | if (!p) FAIL("Failed to create new pool"); 37 | 38 | assert(num_threads < MAX_THREADS); 39 | thread_state_t threads[MAX_THREADS]; 40 | 41 | for (size_t i = 0; i < num_threads; i++) { 42 | thread_state_t *t = &threads[i]; 43 | memset(t, 0, sizeof(*t)); 44 | t->pool = p; 45 | pthread_create(&t->thread, NULL, thread_func, t); 46 | } 47 | for (size_t i = 0; i < num_threads; i++) { 48 | pthread_join(threads[i].thread, NULL); 49 | } 50 | 51 | lf_pool_delete(p); 52 | 53 | // Report stats: 54 | printf("Test: %s\n", test_name); 55 | for (size_t i = 0; i < num_threads; i++) { 56 | thread_state_t *t = &threads[i]; 57 | printf(" Thread %zu | n_attempts: %zu n_success: %zu %.f nanos/attempt\n", i, t->n_attempts, t->n_success, (double)t->dt/t->n_attempts); 58 | } 59 | } 60 | 61 | int main() 62 | { 63 | // Threads fighting over a single tiny element 64 | run_test("2thread_1elt", 2, 1, 1); 65 | run_test("8thread_1elt", 8, 1, 1); 66 | 67 | // Threads fighting over two medium-sized element 68 | run_test("2thread_2elt", 2, 2, 128); 69 | run_test("8thread_2elt", 8, 2, 128); 70 | 71 | // Plenty of elts 72 | run_test("2thread_2elt", 2, 64, 128); 73 | run_test("4thread_2elt", 4, 64, 128); 74 | run_test("5thread_2elt", 5, 64, 128); 75 | run_test("8thread_2elt", 8, 64, 128); 76 | 77 | return 0; 78 | } 79 | -------------------------------------------------------------------------------- /src/test_queue_functional.c: -------------------------------------------------------------------------------- 1 | #include "lf_queue.h" 2 | #include "test_header.h" 3 | 4 | static void test_simple(void) 5 | { 6 | lf_queue_t *q = lf_queue_new(1); 7 | if (!q) FAIL("Failed to allocate queue"); 8 | u64 val; 9 | 10 | REQUIRE(lf_queue_enqueue(q, 45)); 11 | REQUIRE(!lf_queue_enqueue(q, 65)); 12 | REQUIRE(lf_queue_dequeue(q, &val) && val == 45); 13 | REQUIRE(!lf_queue_dequeue(q, &val)); 14 | 15 | lf_queue_delete(q); 16 | } 17 | 18 | static void test_roll_over(void) 19 | { 20 | lf_queue_t *q = lf_queue_new(2); 21 | if (!q) FAIL("Failed to allocate queue"); 22 | u64 val; 23 | 24 | REQUIRE(lf_queue_enqueue(q, 45)); 25 | REQUIRE(lf_queue_dequeue(q, &val) && val == 45); 26 | 27 | REQUIRE(lf_queue_enqueue(q, 65)); 28 | REQUIRE(lf_queue_enqueue(q, 70)); 29 | REQUIRE(!lf_queue_enqueue(q, 80)); 30 | 31 | REQUIRE(lf_queue_dequeue(q, &val) && val == 65); 32 | REQUIRE(lf_queue_dequeue(q, &val) && val == 70); 33 | REQUIRE(!lf_queue_dequeue(q, &val)); 34 | 35 | lf_queue_delete(q); 36 | } 37 | 38 | int main() 39 | { 40 | test_simple(); 41 | test_roll_over(); 42 | return 0; 43 | } 44 | --------------------------------------------------------------------------------