├── LICENSE ├── Makefile ├── README-BENCH-HT.md ├── README-RLU.md ├── bench.c ├── bench_run.sh ├── hash-list.c ├── hash-list.h ├── hazard_ptrs.c ├── hazard_ptrs.h ├── init_env.sh ├── kernel-bench ├── Makefile ├── README.md ├── barrier.c ├── barrier.h ├── hash-list.h ├── intset.c ├── intset.h ├── rcu-hash-list.c ├── rcuintset.c ├── rlu-hash-list.c ├── rlu.c ├── rlu.h ├── rluintset.c ├── sync_test.c └── sync_test.h ├── new-urcu.c ├── new-urcu.h ├── rlu.c ├── rlu.h └── run_tests.py /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Read-Log-Update authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | URCUDIR ?= /usr/local 2 | 3 | CC := gcc 4 | LD := gcc 5 | 6 | CFLAGS += -I$(URCUDIR)/include 7 | CFLAGS += -D_REENTRANT 8 | CFLAGS += -Wall -Winline 9 | #CFLAGS += --param inline-unit-growth=1000 10 | CFLAGS += -mrtm 11 | 12 | ifdef DEBUG 13 | CFLAGS += -O0 -g3 14 | else 15 | CFLAGS += -DNDEBUG 16 | CFLAGS += -O3 17 | endif 18 | 19 | IS_HAZARD_PTRS_HARRIS = -DIS_HAZARD_PTRS_HARRIS 20 | IS_HARRIS = -DIS_HARRIS 21 | IS_RCU = -DIS_RCU 22 | IS_RLU = -DIS_RLU 23 | 24 | LDFLAGS += -L$(URCUDIR)/lib 25 | LDFLAGS += -lpthread 26 | 27 | BINS = bench-harris bench-hp-harris bench-rcu bench-rlu 28 | 29 | .PHONY: all clean 30 | 31 | all: $(BINS) 32 | 33 | rlu.o: rlu.c rlu.h 34 | $(CC) $(CFLAGS) $(DEFINES) -c -o $@ $< 35 | 36 | new-urcu.o: new-urcu.c 37 | $(CC) $(CFLAGS) $(DEFINES) -c -o $@ $< 38 | 39 | hazard_ptrs.o: hazard_ptrs.c 40 | $(CC) $(CFLAGS) $(DEFINES) -c -o $@ $< 41 | 42 | hash-list.o: hash-list.c 43 | $(CC) $(CFLAGS) $(DEFINES) -c -o $@ $< 44 | 45 | bench-harris.o: bench.c 46 | $(CC) $(CFLAGS) $(IS_HARRIS) $(DEFINES) -c -o $@ $< 47 | 48 | bench-hp-harris.o: bench.c 49 | $(CC) $(CFLAGS) $(IS_HAZARD_PTRS_HARRIS) $(DEFINES) -c -o $@ $< 50 | 51 | bench-rcu.o: bench.c 52 | $(CC) $(CFLAGS) $(IS_RCU) $(DEFINES) -c -o $@ $< 53 | 54 | bench-rlu.o: bench.c 55 | $(CC) $(CFLAGS) $(IS_RLU) $(DEFINES) -c -o $@ $< 56 | 57 | bench-harris: new-urcu.o hazard_ptrs.o rlu.o hash-list.o bench-harris.o 58 | $(LD) -o $@ $^ $(LDFLAGS) 59 | 60 | bench-hp-harris: new-urcu.o hazard_ptrs.o rlu.o hash-list.o bench-hp-harris.o 61 | $(LD) -o $@ $^ $(LDFLAGS) 62 | 63 | bench-rcu: new-urcu.o hazard_ptrs.o rlu.o hash-list.o bench-rcu.o 64 | $(LD) -o $@ $^ $(LDFLAGS) 65 | 66 | bench-rlu: new-urcu.o hazard_ptrs.o rlu.o hash-list.o bench-rlu.o 67 | $(LD) -o $@ $^ $(LDFLAGS) 68 | 69 | clean: 70 | rm -f $(BINS) *.o 71 | -------------------------------------------------------------------------------- /README-BENCH-HT.md: -------------------------------------------------------------------------------- 1 | 2 | Read-Log-Update: A Lightweight Synchronization Mechanism for Concurrent Programming 3 | =================================================================================== 4 | 5 | Authors 6 | ------- 7 | Alexander Matveev (MIT) 8 | 9 | Nir Shavit (MIT and Tel-Aviv University) 10 | 11 | Pascal Felber (University of Neuchatel) 12 | 13 | Patrick Marlier (University of Neuchatel) 14 | 15 | Code Maintainer 16 | --------------- 17 | Name: Alexander Matveev 18 | 19 | Email: amatveev@csail.mit.edu 20 | 21 | RLU-HT Benchmark 22 | ================ 23 | Our RLU-HT benchmark provides: 24 | 25 | (1) Linked-list implemented by: RCU, RLU, Harris-Michael, and Hazard-Pointers+Harris-Michael. 26 | 27 | (2) Hash-table implemented by: RCU, RLU. 28 | 29 | In this benchmark we use: 30 | 31 | (1) RLU fine-grained 32 | 33 | (2) Userspace RCU library of Arbel and Morrison (http://dl.acm.org/citation.cfm?id=2611471) 34 | 35 | (3) Harris-Michael list that is based on synchrobench (https://github.com/gramoli/synchrobench) 36 | 37 | (4) Hazard pointers that are based on thread-scan (https://github.com/Willtor/ThreadScan) and stack-track (https://github.com/armafire/stack_track) 38 | 39 | Compilation 40 | ----------- 41 | Execute "make" 42 | 43 | Execution Options 44 | ----------------- 45 | -h, --help 46 | 47 | Print this message 48 | 49 | -a, --do-not-alternate 50 | 51 | Do not alternate insertions and removals 52 | 53 | -w, --rlu-max-ws 54 | 55 | Maximum number of write-sets aggregated in RLU deferral (default=(1)) 56 | 57 | -b, --buckets 58 | 59 | Number of buckets (for linked-list use 1, default=(1)) 60 | 61 | -d, --duration 62 | 63 | Test duration in milliseconds (0=infinite, default=(10000)) 64 | 65 | -i, --initial-size 66 | 67 | Number of elements to insert before test (default=(256)) 68 | 69 | -r, --range 70 | 71 | Range of integer values inserted in set (default=((256) * 2)) 72 | 73 | -s, --seed 74 | 75 | RNG seed (0=time-based, default=(0)) 76 | 77 | -u, --update-rate 78 | 79 | Percentage of update transactions: 0-1000 (100%) (default=(200)) 80 | 81 | -n, --num-threads 82 | 83 | Number of threads (default=(1)) 84 | 85 | Example 86 | ------- 87 | ./bench-rlu -a -b1000 -d10000 -i100000 -r200000 -w10 -u200 -n16 88 | 89 | => Initializes a 100,000 items RLU hash-table with 1000 buckets (100 items per bucket). 90 | 91 | => The key range is 200,000, and the update ratio is 20% (10% inserts and 10% removes). 92 | 93 | => Has no alternation: completely randomized insert/remove. 94 | 95 | => Uses RLU deferral with maximum number of write-sets set to 10. 96 | 97 | => Executes 16 threads for 10 seconds. 98 | 99 | ./bench-rcu -a -b1000 -d10000 -i100000 -r200000 -w10 -u200 -n16 100 | 101 | => Works as the previous example but uses RCU instead. 102 | 103 | -------------------------------------------------------------------------------- /README-RLU.md: -------------------------------------------------------------------------------- 1 | 2 | Read-Log-Update: A Lightweight Synchronization Mechanism for Concurrent Programming 3 | =================================================================================== 4 | 5 | Authors 6 | ------- 7 | Alexander Matveev (MIT) 8 | 9 | Nir Shavit (MIT and Tel-Aviv University) 10 | 11 | Pascal Felber (University of Neuchatel) 12 | 13 | Patrick Marlier (University of Neuchatel) 14 | 15 | Code Maintainer 16 | ----------------- 17 | Name: Alexander Matveev 18 | 19 | Email: amatveev@csail.mit.edu 20 | 21 | RLU v1.0 (08/23/2015) 22 | ===================== 23 | 24 | General 25 | ------- 26 | This is a C implementation of Read-Log-Update (RLU). 27 | 28 | RLU is described in: http://people.csail.mit.edu/amatveev/RLU_SOSP_2015.pdf . 29 | 30 | RLU Types 31 | --------- 32 | Currently we provide two flavors of RLU: (1) coarse-grained and (2) fine-grained. 33 | The coarse-grained flavor has no support for RLU deferral and it provides writer 34 | locks that programmers can use to serialize and coordinate writers. In this way, the 35 | coarse-grained RLU is simpler to use since all operations take an immediate effect, 36 | and they execute once and never abort. In contrast, the fine-grained flavor has no 37 | support for writer locks. Instead it uses per-object locks of RLU to coordinate 38 | writers and does provide support for RLU deferral. As a result, in fine-grained RLU, 39 | writers can execute concurrently while avoiding RLU synchronize calls. 40 | 41 | Brief Tutorial 42 | -------------- 43 | (1) Add "rlu.c" and "rlu.h" to your project and include "rlu.h" 44 | 45 | (2) On program start call "RLU_INIT(type, max_write_sets)": 46 | 47 | => Set "type" to RLU_TYPE_FINE_GRAINED or RLU_TYPE_COARSE_GRAINED 48 | 49 | => For fine-grained, "max_write_sets" defines the number of write-sets that RLU deferral can aggregate. 50 | 51 | => For coarse-grained, "max_write_sets" must be 1 (no RLU deferral) 52 | 53 | (3) For each created thread: 54 | 55 | => On thread start call RLU_THREAD_INIT(self), where self is "rlu_thread_data_t *" 56 | 57 | => On thread finish call RLU_THREAD_FINISH(self) 58 | 59 | (4) Allocate RLU-protected objects with RLU_ALLOC(obj_size) 60 | 61 | (5) To execute an RLU protected section: 62 | 63 | => On section start call RLU_READER_LOCK(self) 64 | 65 | => On section finish call RLU_READER_UNLOCK(self) 66 | 67 | (6) Inside the section: 68 | 69 | => Use RLU_DEREF(self, p_obj) to dereference pointer "p_obj" 70 | 71 | => Before modifying to an object pointed by "p_obj", you need to lock (and log) this object. 72 | 73 | (A) If RLU is fine-grained, then use RLU_TRY_LOCK(self, p_p_obj), where "p_p_obj" is pointer to "p_obj". 74 | 75 | * After RLU_TRY_LOCK(self, p_p_obj) returns, "p_obj" will point to a copy that you can modify. 76 | 77 | * This call can fail, in which case you need to call RLU_ABORT(self) 78 | 79 | (B) If RLU is coarse-grained, then use RLU_LOCK(self, p_p_obj), where "p_p_obj" is pointer to "p_obj". 80 | 81 | * After RLU_LOCK(self, p_p_obj) returns, "p_obj" will point to a copy that you can modify. 82 | 83 | * This call never fails. To achieve this, RLU provides writer locks that you can use to serialize 84 | and coordinate writers. The API function is: 85 | RLU_TRY_WRITER_LOCK(self, writer_lock_id) 86 | 87 | * When RLU completes or aborts, it ensures to release all writer locks (no need to manage them). 88 | 89 | * This call can fail, in which case you need to call RLU_ABORT(self) to release all locks. 90 | 91 | => Use RLU_ASSIGN_PTR(self, p_ptr, p_obj) to assign p_obj to location pointed by p_ptr (*p_ptr = p_obj) 92 | 93 | => Use RLU_IS_SAME_PTRS(p_obj_1, p_obj_2) to compare two pointers 94 | 95 | => Use RLU_FREE(self, p_obj) to free "p_obj" 96 | 97 | Notes 98 | ----- 99 | (1) For an usage example of RLU fine-grained look in "README-BENCH-HT.md" of "rlu" git repository. 100 | 101 | (2) For an usage example of RLU coarse-grained look in "README-BENCH-RHT.md" of "rlu-rht" git repository. 102 | 103 | -------------------------------------------------------------------------------- /bench.c: -------------------------------------------------------------------------------- 1 | 2 | ///////////////////////////////////////////////////////// 3 | // INCLUDES 4 | ///////////////////////////////////////////////////////// 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "hash-list.h" 18 | 19 | ///////////////////////////////////////////////////////// 20 | // DEFINES 21 | ///////////////////////////////////////////////////////// 22 | 23 | #define RO 1 24 | #define RW 0 25 | 26 | #ifdef DEBUG 27 | # define IO_FLUSH fflush(NULL) 28 | /* Note: stdio is thread-safe */ 29 | #endif 30 | 31 | #define DEFAULT_BUCKETS 1 32 | #define DEFAULT_DURATION 10000 33 | #define DEFAULT_INITIAL 256 34 | #define DEFAULT_NB_THREADS 1 35 | #define DEFAULT_RANGE (DEFAULT_INITIAL * 2) 36 | #define DEFAULT_SEED 0 37 | #define DEFAULT_RLU_MAX_WS 1 38 | #define DEFAULT_UPDATE 200 39 | 40 | #define XSTR(s) STR(s) 41 | #define STR(s) #s 42 | 43 | ///////////////////////////////////////////////////////// 44 | // TYPES 45 | ///////////////////////////////////////////////////////// 46 | typedef struct thread_data { 47 | long uniq_id; 48 | hash_list_t *p_hash_list; 49 | struct barrier *barrier; 50 | unsigned long nb_add; 51 | unsigned long nb_remove; 52 | unsigned long nb_contains; 53 | unsigned long nb_found; 54 | unsigned short seed[3]; 55 | int initial; 56 | int diff; 57 | int range; 58 | int update; 59 | int alternate; 60 | rlu_thread_data_t *p_rlu_td; 61 | rlu_thread_data_t rlu_td; 62 | hp_thread_t *p_hp_td; 63 | hp_thread_t hp_td; 64 | char padding[64]; 65 | } thread_data_t; 66 | 67 | typedef struct barrier { 68 | pthread_cond_t complete; 69 | pthread_mutex_t mutex; 70 | int count; 71 | int crossing; 72 | } barrier_t; 73 | 74 | ///////////////////////////////////////////////////////// 75 | // GLOBALS 76 | ///////////////////////////////////////////////////////// 77 | static volatile long padding[50]; 78 | 79 | static volatile int stop; 80 | static unsigned short main_seed[3]; 81 | 82 | ///////////////////////////////////////////////////////// 83 | // HELPER FUNCTIONS 84 | ///////////////////////////////////////////////////////// 85 | static inline int MarsagliaXORV (int x) { 86 | if (x == 0) x = 1 ; 87 | x ^= x << 6; 88 | x ^= ((unsigned)x) >> 21; 89 | x ^= x << 7 ; 90 | return x ; // use either x or x & 0x7FFFFFFF 91 | } 92 | 93 | static inline int MarsagliaXOR (int * seed) { 94 | int x = MarsagliaXORV(*seed); 95 | *seed = x ; 96 | return x & 0x7FFFFFFF; 97 | } 98 | 99 | static inline void rand_init(unsigned short *seed) 100 | { 101 | seed[0] = (unsigned short)rand(); 102 | seed[1] = (unsigned short)rand(); 103 | seed[2] = (unsigned short)rand(); 104 | } 105 | 106 | static inline int rand_range(int n, unsigned short *seed) 107 | { 108 | /* Return a random number in range [0;n) */ 109 | 110 | /*int v = (int)(erand48(seed) * n); 111 | assert (v >= 0 && v < n);*/ 112 | 113 | int v = MarsagliaXOR((int *)seed) % n; 114 | return v; 115 | } 116 | 117 | static void barrier_init(barrier_t *b, int n) 118 | { 119 | pthread_cond_init(&b->complete, NULL); 120 | pthread_mutex_init(&b->mutex, NULL); 121 | b->count = n; 122 | b->crossing = 0; 123 | } 124 | 125 | static void barrier_cross(barrier_t *b) 126 | { 127 | pthread_mutex_lock(&b->mutex); 128 | /* One more thread through */ 129 | b->crossing++; 130 | /* If not all here, wait */ 131 | if (b->crossing < b->count) { 132 | pthread_cond_wait(&b->complete, &b->mutex); 133 | } else { 134 | pthread_cond_broadcast(&b->complete); 135 | /* Reset for next time */ 136 | b->crossing = 0; 137 | } 138 | pthread_mutex_unlock(&b->mutex); 139 | } 140 | 141 | ///////////////////////////////////////////////////////// 142 | // FUNCTIONS 143 | ///////////////////////////////////////////////////////// 144 | static void global_init(int n_threads, int rlu_max_ws) { 145 | RLU_INIT(RLU_TYPE_FINE_GRAINED, rlu_max_ws); 146 | RCU_INIT(n_threads); 147 | } 148 | 149 | static void thread_init(thread_data_t *d) { 150 | RLU_THREAD_INIT(d->p_rlu_td); 151 | RCU_THREAD_INIT(d->uniq_id); 152 | } 153 | 154 | static void thread_finish(thread_data_t *d) { 155 | RLU_THREAD_FINISH(d->p_rlu_td); 156 | RCU_THREAD_FINISH(); 157 | } 158 | 159 | static void print_stats() { 160 | RLU_PRINT_STATS(); 161 | RCU_PRINT_STATS(); 162 | } 163 | 164 | static void hash_list_init(hash_list_t **pp, int n_buckets) { 165 | #ifdef IS_RLU 166 | *pp = rlu_new_hash_list(n_buckets); 167 | #else 168 | #ifdef IS_RCU 169 | *pp = rcu_new_hash_list(n_buckets); 170 | #else 171 | #ifdef IS_HARRIS 172 | *pp = harris_new_hash_list(n_buckets); 173 | #else 174 | #ifdef IS_HAZARD_PTRS_HARRIS 175 | *pp = hp_harris_new_hash_list(n_buckets); 176 | #else 177 | printf("ERROR: benchmark not defined!\n"); 178 | abort(); 179 | #endif 180 | #endif 181 | #endif 182 | #endif 183 | } 184 | 185 | static int hash_list_contains(thread_data_t *d, int key) { 186 | #ifdef IS_RLU 187 | return rlu_hash_list_contains(d->p_rlu_td, d->p_hash_list, key); 188 | #else 189 | #ifdef IS_RCU 190 | return rcu_hash_list_contains(d->p_hash_list, key); 191 | #else 192 | #ifdef IS_HARRIS 193 | return harris_hash_list_contains(d->p_hash_list, key); 194 | #else 195 | #ifdef IS_HAZARD_PTRS_HARRIS 196 | return hp_harris_hash_list_contains(d->p_hp_td, d->p_hash_list, key); 197 | #else 198 | printf("ERROR: benchmark not defined!\n"); 199 | abort(); 200 | #endif 201 | #endif 202 | #endif 203 | #endif 204 | } 205 | 206 | static int hash_list_add(thread_data_t *d, int key) { 207 | #ifdef IS_RLU 208 | return rlu_hash_list_add(d->p_rlu_td, d->p_hash_list, key); 209 | #else 210 | #ifdef IS_RCU 211 | return rcu_hash_list_add(d->p_hash_list, key); 212 | #else 213 | #ifdef IS_HARRIS 214 | return harris_hash_list_add(d->p_hash_list, key); 215 | #else 216 | #ifdef IS_HAZARD_PTRS_HARRIS 217 | return hp_harris_hash_list_add(d->p_hp_td, d->p_hash_list, key); 218 | #else 219 | printf("ERROR: benchmark not defined!\n"); 220 | abort(); 221 | #endif 222 | #endif 223 | #endif 224 | #endif 225 | } 226 | 227 | static int hash_list_remove(thread_data_t *d, int key) { 228 | #ifdef IS_RLU 229 | return rlu_hash_list_remove(d->p_rlu_td, d->p_hash_list, key); 230 | #else 231 | #ifdef IS_RCU 232 | return rcu_hash_list_remove(d->p_hash_list, key); 233 | #else 234 | #ifdef IS_HARRIS 235 | return harris_hash_list_remove(d->p_hash_list, key); 236 | #else 237 | #ifdef IS_HAZARD_PTRS_HARRIS 238 | return hp_harris_hash_list_remove(d->p_hp_td, d->p_hash_list, key); 239 | #else 240 | printf("ERROR: benchmark not defined!\n"); 241 | abort(); 242 | #endif 243 | #endif 244 | #endif 245 | #endif 246 | } 247 | 248 | static void *test(void *data) 249 | { 250 | int op, key, last = -1; 251 | thread_data_t *d = (thread_data_t *)data; 252 | 253 | thread_init(d); 254 | 255 | if (d->uniq_id == 0) { 256 | /* Populate set */ 257 | printf("[%ld] Initializing\n", d->uniq_id); 258 | printf("[%ld] Adding %d entries to set\n", d->uniq_id, d->initial); 259 | d->p_rlu_td->is_no_quiescence = 1; 260 | int i = 0; 261 | while (i < d->initial) { 262 | key = rand_range(d->range, d->seed) + 1; 263 | 264 | if (hash_list_add(d, key)) { 265 | i++; 266 | } 267 | } 268 | printf("[%ld] Adding done\n", d->uniq_id); 269 | int size = hash_list_size(d->p_hash_list); 270 | printf("Hash-list size : %d\n", size); 271 | 272 | d->p_rlu_td->is_no_quiescence = 0; 273 | } 274 | 275 | /* Wait on barrier */ 276 | barrier_cross(d->barrier); 277 | 278 | while (stop == 0) { 279 | op = rand_range(1000, d->seed); 280 | if (op < d->update) { 281 | if (d->alternate) { 282 | /* Alternate insertions and removals */ 283 | if (last < 0) { 284 | /* Add random value */ 285 | key = rand_range(d->range, d->seed) + 1; 286 | if (hash_list_add(d, key)) { 287 | d->diff++; 288 | last = key; 289 | } 290 | d->nb_add++; 291 | } else { 292 | /* Remove last value */ 293 | if (hash_list_remove(d, last)) { 294 | d->diff--; 295 | } 296 | d->nb_remove++; 297 | last = -1; 298 | } 299 | } else { 300 | /* Randomly perform insertions and removals */ 301 | key = rand_range(d->range, d->seed) + 1; 302 | if ((op & 0x01) == 0) { 303 | /* Add random value */ 304 | if (hash_list_add(d, key)) { 305 | d->diff++; 306 | } 307 | d->nb_add++; 308 | } else { 309 | /* Remove random value */ 310 | if (hash_list_remove(d, key)) { 311 | d->diff--; 312 | } 313 | d->nb_remove++; 314 | } 315 | } 316 | } else { 317 | /* Look for random value */ 318 | key = rand_range(d->range, d->seed) + 1; 319 | if (hash_list_contains(d, key)) { 320 | d->nb_found++; 321 | } 322 | d->nb_contains++; 323 | } 324 | } 325 | 326 | thread_finish(d); 327 | 328 | return NULL; 329 | } 330 | 331 | int main(int argc, char **argv) 332 | { 333 | struct option long_options[] = { 334 | // These options don't set a flag 335 | {"help", no_argument, NULL, 'h'}, 336 | {"do-not-alternate", no_argument, NULL, 'a'}, 337 | {"buckets", required_argument, NULL, 'b'}, 338 | {"duration", required_argument, NULL, 'd'}, 339 | {"initial-size", required_argument, NULL, 'i'}, 340 | {"num-threads", required_argument, NULL, 'n'}, 341 | {"range", required_argument, NULL, 'r'}, 342 | {"seed", required_argument, NULL, 's'}, 343 | {"rlu-max-ws", required_argument, NULL, 'w'}, 344 | {"update-rate", required_argument, NULL, 'u'}, 345 | {NULL, 0, NULL, 0} 346 | }; 347 | 348 | hash_list_t *p_hash_list; 349 | int i, c, size; 350 | unsigned long reads, updates; 351 | thread_data_t *data; 352 | pthread_t *threads; 353 | pthread_attr_t attr; 354 | barrier_t barrier; 355 | struct timeval start, end; 356 | struct timespec timeout; 357 | int n_buckets = DEFAULT_BUCKETS; 358 | int duration = DEFAULT_DURATION; 359 | int initial = DEFAULT_INITIAL; 360 | int nb_threads = DEFAULT_NB_THREADS; 361 | int range = DEFAULT_RANGE; 362 | int seed = DEFAULT_SEED; 363 | int rlu_max_ws = DEFAULT_RLU_MAX_WS; 364 | int update = DEFAULT_UPDATE; 365 | int alternate = 1; 366 | sigset_t block_set; 367 | 368 | while(1) { 369 | i = 0; 370 | c = getopt_long(argc, argv, "hab:d:i:n:r:s:w:u:", long_options, &i); 371 | 372 | if(c == -1) 373 | break; 374 | 375 | if(c == 0 && long_options[i].flag == 0) 376 | c = long_options[i].val; 377 | 378 | switch(c) { 379 | case 0: 380 | /* Flag is automatically set */ 381 | break; 382 | case 'h': 383 | printf("intset " 384 | "(linked list)\n" 385 | "\n" 386 | "Usage:\n" 387 | " intset [options...]\n" 388 | "\n" 389 | "Options:\n" 390 | " -h, --help\n" 391 | " Print this message\n" 392 | " -a, --do-not-alternate\n" 393 | " Do not alternate insertions and removals\n" 394 | " -b, --buckets \n" 395 | " Number of buckets (default=" XSTR(DEFAULT_BUCKETS) ")\n" 396 | " -d, --duration \n" 397 | " Test duration in milliseconds (0=infinite, default=" XSTR(DEFAULT_DURATION) ")\n" 398 | " -i, --initial-size \n" 399 | " Number of elements to insert before test (default=" XSTR(DEFAULT_INITIAL) ")\n" 400 | " -n, --num-threads \n" 401 | " Number of threads (default=" XSTR(DEFAULT_NB_THREADS) ")\n" 402 | " -r, --range \n" 403 | " Range of integer values inserted in set (default=" XSTR(DEFAULT_RANGE) ")\n" 404 | " -s, --seed \n" 405 | " RNG seed (0=time-based, default=" XSTR(DEFAULT_SEED) ")\n" 406 | " -u, --update-rate \n" 407 | " Percentage of update transactions (1000 = 100 percent) (default=" XSTR(DEFAULT_UPDATE) ")\n" 408 | ); 409 | exit(0); 410 | case 'a': 411 | alternate = 0; 412 | break; 413 | case 'b': 414 | n_buckets = atoi(optarg); 415 | break; 416 | case 'd': 417 | duration = atoi(optarg); 418 | break; 419 | case 'i': 420 | initial = atoi(optarg); 421 | break; 422 | case 'n': 423 | nb_threads = atoi(optarg); 424 | break; 425 | case 'r': 426 | range = atoi(optarg); 427 | break; 428 | case 's': 429 | seed = atoi(optarg); 430 | break; 431 | case 'w': 432 | rlu_max_ws = atoi(optarg); 433 | break; 434 | case 'u': 435 | update = atoi(optarg); 436 | break; 437 | case '?': 438 | printf("Use -h or --help for help\n"); 439 | exit(0); 440 | default: 441 | exit(1); 442 | } 443 | } 444 | 445 | assert(n_buckets >= 1); 446 | assert(duration >= 0); 447 | assert(initial >= 0); 448 | assert(nb_threads > 0); 449 | assert(range > 0 && range >= initial); 450 | assert(rlu_max_ws >= 1 && rlu_max_ws <= 100 && update >= 0 && update <= 1000); 451 | 452 | printf("Set type : hash-list\n"); 453 | printf("Buckets : %d\n", n_buckets); 454 | printf("Duration : %d\n", duration); 455 | printf("Initial size : %d\n", initial); 456 | printf("Nb threads : %d\n", nb_threads); 457 | printf("Value range : %d\n", range); 458 | printf("Seed : %d\n", seed); 459 | printf("rlu-max-ws : %d\n", rlu_max_ws); 460 | printf("Update rate : %d\n", update); 461 | printf("Alternate : %d\n", alternate); 462 | printf("Node size : %lu\n", sizeof(node_t)); 463 | printf("Type sizes : int=%d/long=%d/ptr=%d/word=%d\n", 464 | (int)sizeof(int), 465 | (int)sizeof(long), 466 | (int)sizeof(void *), 467 | (int)sizeof(size_t)); 468 | 469 | timeout.tv_sec = duration / 1000; 470 | timeout.tv_nsec = (duration % 1000) * 1000000; 471 | 472 | if ((data = (thread_data_t *)malloc(nb_threads * sizeof(thread_data_t))) == NULL) { 473 | perror("malloc"); 474 | exit(1); 475 | } 476 | 477 | memset(data, 0, nb_threads * sizeof(thread_data_t)); 478 | 479 | if ((threads = (pthread_t *)malloc(nb_threads * sizeof(pthread_t))) == NULL) { 480 | perror("malloc"); 481 | exit(1); 482 | } 483 | 484 | if (seed == 0) 485 | srand((int)time(NULL)); 486 | else 487 | srand(seed); 488 | 489 | global_init(nb_threads, rlu_max_ws); 490 | 491 | hash_list_init(&p_hash_list, n_buckets); 492 | 493 | size = initial; 494 | 495 | stop = 0; 496 | 497 | /* Thread-local seed for main thread */ 498 | rand_init(main_seed); 499 | 500 | if (alternate == 0 && range != initial * 2) { 501 | printf("ERROR: range is not twice the initial set size\n"); 502 | exit(1); 503 | } 504 | 505 | /* Access set from all threads */ 506 | barrier_init(&barrier, nb_threads + 1); 507 | pthread_attr_init(&attr); 508 | pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); 509 | for (i = 0; i < nb_threads; i++) { 510 | printf("Creating thread %d\n", i); 511 | 512 | data[i].uniq_id = i; 513 | data[i].range = range; 514 | data[i].update = update; 515 | data[i].alternate = alternate; 516 | data[i].nb_add = 0; 517 | data[i].nb_remove = 0; 518 | data[i].nb_contains = 0; 519 | data[i].nb_found = 0; 520 | data[i].initial = initial; 521 | data[i].diff = 0; 522 | rand_init(data[i].seed); 523 | data[i].p_hash_list = p_hash_list; 524 | data[i].barrier = &barrier; 525 | data[i].p_rlu_td = &(data[i].rlu_td); 526 | data[i].p_hp_td = &(data[i].hp_td); 527 | if (pthread_create(&threads[i], &attr, test, (void *)(&data[i])) != 0) { 528 | fprintf(stderr, "Error creating thread\n"); 529 | exit(1); 530 | } 531 | } 532 | pthread_attr_destroy(&attr); 533 | 534 | /* Start threads */ 535 | barrier_cross(&barrier); 536 | 537 | printf("STARTING THREADS...\n"); 538 | gettimeofday(&start, NULL); 539 | if (duration > 0) { 540 | nanosleep(&timeout, NULL); 541 | } else { 542 | sigemptyset(&block_set); 543 | sigsuspend(&block_set); 544 | } 545 | stop = 1; 546 | gettimeofday(&end, NULL); 547 | printf("STOPPING THREADS...\n"); 548 | 549 | /* Wait for thread completion */ 550 | for (i = 0; i < nb_threads; i++) { 551 | if (pthread_join(threads[i], NULL) != 0) { 552 | fprintf(stderr, "Error waiting for thread completion\n"); 553 | exit(1); 554 | } 555 | } 556 | 557 | duration = (end.tv_sec * 1000 + end.tv_usec / 1000) - (start.tv_sec * 1000 + start.tv_usec / 1000); 558 | reads = 0; 559 | updates = 0; 560 | for (i = 0; i < nb_threads; i++) { 561 | printf("Thread %d\n", i); 562 | printf(" #add : %lu\n", data[i].nb_add); 563 | printf(" #remove : %lu\n", data[i].nb_remove); 564 | printf(" #contains : %lu\n", data[i].nb_contains); 565 | printf(" #found : %lu\n", data[i].nb_found); 566 | reads += data[i].nb_contains; 567 | updates += (data[i].nb_add + data[i].nb_remove); 568 | size += data[i].diff; 569 | } 570 | printf("Set size : %d (expected: %d)\n", hash_list_size(p_hash_list), size); 571 | printf("Duration : %d (ms)\n", duration); 572 | printf("#ops : %lu (%f / s)\n", reads + updates, (reads + updates) * 1000.0 / duration); 573 | printf("#read ops : %lu (%f / s)\n", reads, reads * 1000.0 / duration); 574 | printf("#update ops : %lu (%f / s)\n", updates, updates * 1000.0 / duration); 575 | 576 | print_stats(); 577 | 578 | /* Cleanup */ 579 | free(threads); 580 | free(data); 581 | 582 | return 0; 583 | } 584 | -------------------------------------------------------------------------------- /bench_run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # HLIST 1-1000 4 | 5 | #python run_tests.py 1 10000 harris 0 1000 2000 __harris_hlist_u0_1000.txt 1 2 4 6 8 10 12 14 16 6 | #python run_tests.py 1 10000 hp_harris 0 1000 2000 __hp_harris_hlist_u0_1000.txt 1 2 4 6 8 10 12 14 16 7 | #python run_tests.py 1 10000 rcu 0 1000 2000 __rcu_hlist_u0_1000.txt 1 2 4 6 8 10 12 14 16 8 | #python run_tests.py 1 10000 rlu 0 1000 2000 __rlu_hlist_u0_1000.txt 1 2 4 6 8 10 12 14 16 9 | 10 | #python run_tests.py 1 1 10000 harris 1 1000 2000 __harris_hlist_u1_1000.txt 1 2 4 6 8 10 12 14 16 11 | #python run_tests.py 1 1 10000 hp_harris 1 1000 2000 __hp_harris_hlist_u1_1000.txt 1 2 4 6 8 10 12 14 16 12 | ##python run_tests.py 1 1 10000 rcu 1 1000 2000 __rcu_hlist_u1_1000.txt 1 4 8 12 16 13 | ##python run_tests.py 1 1 10000 rlu 1 1000 2000 __rlu_hlist_u1_1000.txt 1 4 8 12 16 14 | 15 | #python run_tests.py 1 10000 harris 10 1000 2000 __harris_hlist_u10_1000.txt 1 2 4 6 8 10 12 14 16 16 | #python run_tests.py 1 10000 hp_harris 10 1000 2000 __hp_harris_hlist_u10_1000.txt 1 2 4 6 8 10 12 14 16 17 | #python run_tests.py 1 1 10000 rcu 10 1000 2000 __rcu_hlist_u10_1000.txt 1 2 4 6 8 10 12 14 16 18 | #python run_tests.py 1 1 10000 rlu 10 1000 2000 __rlu_hlist_u10_1000.txt 1 2 4 6 8 10 12 14 16 19 | 20 | #python run_tests.py 1 10000 harris 20 1000 2000 __harris_hlist_u20_1000.txt 1 2 4 6 8 10 12 14 16 21 | #python run_tests.py 1 10000 hp_harris 20 1000 2000 __hp_harris_hlist_u20_1000.txt 1 2 4 6 8 10 12 14 16 22 | ##python run_tests.py 1 1 10000 rcu 20 1000 2000 __rcu_hlist_u20_1000.txt 1 4 8 12 16 23 | ##python run_tests.py 1 1 10000 rlu 20 1000 2000 __rlu_hlist_u20_1000.txt 1 4 8 12 16 24 | 25 | #python run_tests.py 1 10000 harris 40 1000 2000 __harris_hlist_u40_1000.txt 1 2 4 6 8 10 12 14 16 26 | #python run_tests.py 1 10000 hp_harris 40 1000 2000 __hp_harris_hlist_u40_1000.txt 1 2 4 6 8 10 12 14 16 27 | ##python run_tests.py 1 1 10000 rcu 40 1000 2000 __rcu_hlist_u40_1000.txt 1 4 8 12 16 28 | ##python run_tests.py 1 1 10000 rlu 40 1000 2000 __rlu_hlist_u40_1000.txt 1 4 8 12 16 29 | 30 | # HLIST 100-1000 31 | 32 | #python run_tests.py 1 1000 10000 harris 0 100000 200000 __harris_hlist_1000_100_u0_1000.txt 1 2 4 6 8 10 12 14 16 33 | #python run_tests.py 1 1000 10000 hp_harris 0 100000 200000 __hp_harris_hlist_1000_100_u0_1000.txt 1 2 4 6 8 10 12 14 16 34 | #python run_tests.py 1 1000 10000 rcu 0 100000 200000 __rcu_hlist_1000_100_u0_1000.txt 1 2 4 6 8 10 12 14 16 35 | #python run_tests.py 1 1000 10000 rlu 0 100000 200000 __rlu_hlist_1000_100_u0_1000.txt 1 2 4 6 8 10 12 14 16 36 | 37 | #python run_tests.py 1 1000 10000 harris 1 100000 200000 __harris_hlist_1000_100_u1_1000.txt 1 2 4 6 8 10 12 14 16 38 | #python run_tests.py 1 1000 10000 hp_harris 1 100000 200000 __hp_harris_hlist_1000_100_u1_1000.txt 1 2 4 6 8 10 12 14 16 39 | 40 | #python run_tests.py 1 1000 10000 rcu 2 100000 200000 __rcu_hlist_1000_100_u1_1000.txt 1 2 4 8 12 16 41 | #python run_tests.py 1 1000 10000 rlu 2 100000 200000 __rlu_hlist_1000_100_u2_1000.txt 1 2 4 8 12 16 42 | 43 | #python run_tests.py 1 1000 10000 harris 10 100000 200000 __harris_hlist_1000_100_u10_1000.txt 1 2 4 6 8 10 12 14 16 44 | #python run_tests.py 1 1000 10000 hp_harris 10 100000 200000 __hp_harris_hlist_1000_100_u10_1000.txt 1 2 4 6 8 10 12 14 16 45 | #python run_tests.py 1 1000 10000 rcu 10 100000 200000 __rcu_hlist_1000_100_u10_1000.txt 1 2 4 6 8 10 12 14 16 46 | #python run_tests.py 1 1000 10000 rlu 10 100000 200000 __rlu_hlist_1000_100_u10_1000.txt 1 2 4 6 8 10 12 14 16 47 | 48 | #python run_tests.py 1 1000 10000 harris 20 100000 200000 __harris_hlist_1000_100_u20_1000.txt 1 2 4 6 8 10 12 14 16 49 | #python run_tests.py 1 1000 10000 hp_harris 20 100000 200000 __hp_harris_hlist_1000_100_u20_1000.txt 1 2 4 6 8 10 12 14 16 50 | #python run_tests.py 1 1000 10000 rcu 20 100000 200000 __rcu_hlist_1000_100_u20_1000.txt 1 2 4 6 8 10 12 14 16 51 | #python run_tests.py 1 1000 10000 rlu 20 100000 200000 __rlu_hlist_1000_100_u20_1000.txt 1 2 4 6 8 10 12 14 16 52 | 53 | #python run_tests.py 1 1000 10000 rcu 20 100000 200000 __rcu_hlist_1000_100_u20_1000.txt 1 2 4 8 12 16 54 | #python run_tests.py 1 1000 10000 rlu 20 100000 200000 __rlu_hlist_1000_100_u20_1000.txt 1 2 4 8 12 16 55 | 56 | 57 | #python run_tests.py 1 1000 10000 harris 40 100000 200000 __harris_hlist_1000_100_u40_1000.txt 1 2 4 6 8 10 12 14 16 58 | #python run_tests.py 1 1000 10000 hp_harris 40 100000 200000 __hp_harris_hlist_1000_100_u40_1000.txt 1 2 4 6 8 10 12 14 16 59 | #python run_tests.py 1 1000 10000 rcu 40 100000 200000 __rcu_hlist_1000_100_u40_1000.txt 1 2 4 6 8 10 12 14 16 60 | #python run_tests.py 1 1000 10000 rlu 40 100000 200000 __rlu_hlist_1000_100_u40_1000.txt 1 2 4 6 8 10 12 14 16 61 | 62 | #python run_tests.py 1 1000 10000 rcu 40 100000 200000 __rcu_hlist_1000_100_u40_1000.txt 1 2 4 8 12 16 63 | #python run_tests.py 1 1000 10000 rlu 40 100000 200000 __rlu_hlist_1000_100_u40_1000.txt 1 2 4 8 12 16 64 | 65 | ################## 66 | 67 | # LINKED LIST (1000) 68 | #python run_tests.py 1 1 1 10000 rcu 20 1000 2000 __rcu_list_1000_u20.txt 1 2 4 6 8 10 12 14 16 69 | #python run_tests.py 1 1 1 10000 harris 20 1000 2000 __harris_list_1000_u20.txt 1 2 4 6 8 10 12 14 16 70 | #python run_tests.py 1 1 1 10000 hp_harris 20 1000 2000 __hp_harris_list_1000_u20.txt 1 2 4 6 8 10 12 14 16 71 | 72 | #python run_tests.py 1 1 1 10000 rcu 200 1000 2000 __rcu_list_1000_u200.txt 1 2 4 6 8 10 12 14 16 73 | #python run_tests.py 1 1 1 10000 rlu 200 1000 2000 __rlu_list_1000_u200.txt 1 2 4 6 8 10 12 14 16 74 | #python run_tests.py 1 1 1 10000 harris 200 1000 2000 __harris_list_1000_u200.txt 1 2 4 6 8 10 12 14 16 75 | #python run_tests.py 1 1 1 10000 hp_harris 200 1000 2000 __hp_harris_list_1000_u200.txt 1 2 4 6 8 10 12 14 16 76 | 77 | #python run_tests.py 1 1 1 10000 rcu 400 1000 2000 __rcu_list_1000_u400.txt 1 2 4 6 8 10 12 14 16 78 | 79 | #python run_tests.py 1 1 1 10000 rlu 400 1000 2000 __rlu_list_1000_u400.txt 1 2 4 6 8 10 12 14 16 80 | #python run_tests.py 1 1 1 10000 rlu 400 1000 2000 __rlu_list_1000_u400.txt 10 81 | 82 | #python run_tests.py 1 1 1 10000 harris 400 1000 2000 __harris_list_1000_u400.txt 1 2 4 6 8 10 12 14 16 83 | #python run_tests.py 1 1 1 10000 hp_harris 400 1000 2000 __hp_harris_list_1000_u400.txt 1 2 4 6 8 10 12 14 16 84 | 85 | 86 | # HASH TABLE (1000-100) 87 | 88 | # python run_tests.py 1 1 1000 10000 rcu 0 100000 200000 __rcu_hlist_1000_100_u0.txt 1 2 4 6 8 10 12 14 16 89 | # python run_tests.py 1 1 1000 10000 rlu 0 100000 200000 __rlu_1_hlist_1000_100_u0.txt 1 2 4 6 8 10 12 14 16 90 | # 91 | # python run_tests.py 1 1 1000 10000 rcu 2 100000 200000 __rcu_hlist_1000_100_u2.txt 1 2 4 6 8 10 12 14 16 92 | # python run_tests.py 1 1 1000 10000 rlu 2 100000 200000 __rlu_1_hlist_1000_100_u2.txt 1 2 4 6 8 10 12 14 16 93 | # python run_tests.py 1 10 1000 10000 rlu 2 100000 200000 __rlu_10_hlist_1000_100_u2.txt 1 2 4 6 8 10 12 14 16 94 | # python run_tests.py 1 100 1000 10000 rlu 2 100000 200000 __rlu_100_hlist_1000_100_u2.txt 1 2 4 6 8 10 12 14 16 95 | # 96 | # python run_tests.py 1 1 1000 10000 rcu 20 100000 200000 __rcu_hlist_1000_100_u20.txt 1 2 4 6 8 10 12 14 16 97 | # python run_tests.py 1 1 1000 10000 rlu 20 100000 200000 __rlu_1_hlist_1000_100_u20.txt 1 2 4 6 8 10 12 14 16 98 | # python run_tests.py 1 10 1000 10000 rlu 20 100000 200000 __rlu_10_hlist_1000_100_u20.txt 1 2 4 6 8 10 12 14 16 99 | # python run_tests.py 1 100 1000 10000 rlu 20 100000 200000 __rlu_100_hlist_1000_100_u20.txt 1 2 4 6 8 10 12 14 16 100 | # 101 | # python run_tests.py 1 1 1000 10000 rcu 200 100000 200000 __rcu_hlist_1000_100_u200.txt 1 2 4 6 8 10 12 14 16 102 | # python run_tests.py 1 1 1000 10000 rlu 200 100000 200000 __rlu_1_hlist_1000_100_u200.txt 1 2 4 6 8 10 12 14 16 103 | # python run_tests.py 1 10 1000 10000 rlu 200 100000 200000 __rlu_10_hlist_1000_100_u200.txt 1 2 4 6 8 10 12 14 16 104 | # python run_tests.py 1 100 1000 10000 rlu 200 100000 200000 __rlu_100_hlist_1000_100_u200.txt 1 2 4 6 8 10 12 14 16 105 | # 106 | # python run_tests.py 1 1 1000 10000 rcu 400 100000 200000 __rcu_hlist_1000_100_u400.txt 1 2 4 6 8 10 12 14 16 107 | # python run_tests.py 1 1 1000 10000 rlu 400 100000 200000 __rlu_1_hlist_1000_100_u400.txt 1 2 4 6 8 10 12 14 16 108 | # python run_tests.py 1 10 1000 10000 rlu 400 100000 200000 __rlu_10_hlist_1000_100_u400.txt 1 2 4 6 8 10 12 14 16 109 | # python run_tests.py 1 100 1000 10000 rlu 400 100000 200000 __rlu_100_hlist_1000_100_u400.txt 1 2 4 6 8 10 12 14 16 110 | 111 | # HASH TABLE (10000-1) 112 | python run_tests.py 1 1 10000 10000 rcu 1000 10000 20000 __rcu_hlist_10000_1_u1000.txt 1 2 4 6 8 10 12 14 16 113 | python run_tests.py 1 1 10000 10000 rlu 1000 10000 20000 __rlu_1_hlist_10000_1_u1000.txt 1 2 4 6 8 10 12 14 16 114 | python run_tests.py 1 10 10000 10000 rlu 1000 10000 20000 __rlu_10_hlist_10000_1_u1000.txt 1 2 4 6 8 10 12 14 16 115 | python run_tests.py 1 100 10000 10000 rlu 1000 10000 20000 __rlu_100_hlist_10000_1_u1000.txt 1 2 4 6 8 10 12 14 16 116 | -------------------------------------------------------------------------------- /hash-list.c: -------------------------------------------------------------------------------- 1 | 2 | ///////////////////////////////////////////////////////// 3 | // INCLUDES 4 | ///////////////////////////////////////////////////////// 5 | #include 6 | #include 7 | 8 | #include "new-urcu.h" 9 | #include "rlu.h" 10 | #include "hash-list.h" 11 | 12 | ///////////////////////////////////////////////////////// 13 | // DEFINES 14 | ///////////////////////////////////////////////////////// 15 | #define HASH_VALUE(p_hash_list, val) (val % p_hash_list->n_buckets) 16 | 17 | #define MEMBARSTLD() __sync_synchronize() 18 | #define CAS(addr, expected_value, new_value) __sync_val_compare_and_swap((addr), (expected_value), (new_value)) 19 | #define HARRIS_CAS(p_addr, expected_value, new_value) (CAS((intptr_t *)p_addr, (intptr_t)expected_value, (intptr_t)new_value) == (intptr_t)expected_value) 20 | 21 | ///////////////////////////////////////////////////////// 22 | // TYPES 23 | ///////////////////////////////////////////////////////// 24 | 25 | ///////////////////////////////////////////////////////// 26 | // GLOBALS 27 | ///////////////////////////////////////////////////////// 28 | 29 | ///////////////////////////////////////////////////////// 30 | // FUNCTIONS 31 | ///////////////////////////////////////////////////////// 32 | 33 | ///////////////////////////////////////////////////////// 34 | // HARRIS LIST HELPER FUNCTIONS 35 | ///////////////////////////////////////////////////////// 36 | inline int is_marked_ref(long i) { 37 | return (int) (i & (LONG_MIN+1)); 38 | } 39 | 40 | inline long unset_mark(long i) { 41 | i &= LONG_MAX-1; 42 | return i; 43 | } 44 | 45 | inline long set_mark(long i) { 46 | i = unset_mark(i); 47 | i += 1; 48 | return i; 49 | } 50 | 51 | inline long get_unmarked_ref(long w) { 52 | return unset_mark(w); 53 | } 54 | 55 | inline long get_marked_ref(long w) { 56 | return set_mark(w); 57 | } 58 | 59 | ///////////////////////////////////////////////////////// 60 | // NEW NODE 61 | ///////////////////////////////////////////////////////// 62 | node_t *pure_new_node() { 63 | 64 | node_t *p_new_node = (node_t *)malloc(sizeof(node_t)); 65 | if (p_new_node == NULL){ 66 | printf("out of memory\n"); 67 | exit(1); 68 | } 69 | 70 | return p_new_node; 71 | } 72 | 73 | node_t *rlu_new_node() { 74 | 75 | node_t *p_new_node = (node_t *)RLU_ALLOC(sizeof(node_t)); 76 | if (p_new_node == NULL){ 77 | printf("out of memory\n"); 78 | exit(1); 79 | } 80 | 81 | return p_new_node; 82 | } 83 | 84 | node_t *harris_new_node() { 85 | return pure_new_node(); 86 | } 87 | 88 | node_t *hp_harris_new_node() { 89 | return pure_new_node(); 90 | } 91 | 92 | node_t *rcu_new_node() { 93 | return pure_new_node(); 94 | } 95 | 96 | ///////////////////////////////////////////////////////// 97 | // FREE NODE 98 | ///////////////////////////////////////////////////////// 99 | void pure_free_node(node_t *p_node) { 100 | if (p_node != NULL) { 101 | free(p_node); 102 | } 103 | } 104 | 105 | void harris_free_node(node_t *p_node) { 106 | /* leaks memory */ 107 | } 108 | 109 | void rcu_free_node(node_t *p_node) { 110 | RCU_FREE(p_node); 111 | } 112 | 113 | ///////////////////////////////////////////////////////// 114 | // NEW LIST 115 | ///////////////////////////////////////////////////////// 116 | list_t *pure_new_list() 117 | { 118 | list_t *p_list; 119 | node_t *p_min_node, *p_max_node; 120 | 121 | p_list = (list_t *)malloc(sizeof(list_t)); 122 | if (p_list == NULL) { 123 | perror("malloc"); 124 | exit(1); 125 | } 126 | 127 | p_max_node = pure_new_node(); 128 | p_max_node->val = LIST_VAL_MAX; 129 | p_max_node->p_next = NULL; 130 | 131 | p_min_node = pure_new_node(); 132 | p_min_node->val = LIST_VAL_MIN; 133 | p_min_node->p_next = p_max_node; 134 | 135 | p_list->p_head = p_min_node; 136 | 137 | return p_list; 138 | } 139 | 140 | list_t *rlu_new_list() 141 | { 142 | list_t *p_list; 143 | node_t *p_min_node, *p_max_node; 144 | 145 | p_list = (list_t *)malloc(sizeof(list_t)); 146 | if (p_list == NULL) { 147 | perror("malloc"); 148 | exit(1); 149 | } 150 | 151 | p_max_node = rlu_new_node(); 152 | p_max_node->val = LIST_VAL_MAX; 153 | p_max_node->p_next = NULL; 154 | 155 | p_min_node = rlu_new_node(); 156 | p_min_node->val = LIST_VAL_MIN; 157 | p_min_node->p_next = p_max_node; 158 | 159 | p_list->p_head = p_min_node; 160 | 161 | return p_list; 162 | } 163 | 164 | ///////////////////////////////////////////////////////// 165 | // NEW HASH LIST 166 | ///////////////////////////////////////////////////////// 167 | hash_list_t *pure_new_hash_list(int n_buckets) 168 | { 169 | int i; 170 | hash_list_t *p_hash_list; 171 | 172 | p_hash_list = (hash_list_t *)malloc(sizeof(hash_list_t)); 173 | 174 | if (p_hash_list == NULL) { 175 | perror("malloc"); 176 | exit(1); 177 | } 178 | 179 | p_hash_list->n_buckets = n_buckets; 180 | 181 | for (i = 0; i < p_hash_list->n_buckets; i++) { 182 | p_hash_list->buckets[i] = pure_new_list(); 183 | } 184 | 185 | return p_hash_list; 186 | } 187 | 188 | hash_list_t *harris_new_hash_list(int n_buckets) 189 | { 190 | return pure_new_hash_list(n_buckets); 191 | } 192 | 193 | hash_list_t *hp_harris_new_hash_list(int n_buckets) 194 | { 195 | return pure_new_hash_list(n_buckets); 196 | } 197 | 198 | hash_list_t *rcu_new_hash_list(int n_buckets) 199 | { 200 | return pure_new_hash_list(n_buckets); 201 | } 202 | 203 | hash_list_t *rlu_new_hash_list(int n_buckets) 204 | { 205 | int i; 206 | hash_list_t *p_hash_list; 207 | 208 | p_hash_list = (hash_list_t *)malloc(sizeof(hash_list_t)); 209 | 210 | if (p_hash_list == NULL) { 211 | perror("malloc"); 212 | exit(1); 213 | } 214 | 215 | p_hash_list->n_buckets = n_buckets; 216 | 217 | for (i = 0; i < p_hash_list->n_buckets; i++) { 218 | p_hash_list->buckets[i] = rlu_new_list(); 219 | } 220 | 221 | return p_hash_list; 222 | } 223 | 224 | ///////////////////////////////////////////////////////// 225 | // LIST SIZE 226 | ///////////////////////////////////////////////////////// 227 | int list_size(list_t *p_list) 228 | { 229 | int size = 0; 230 | node_t *p_node; 231 | 232 | /* We have at least 2 elements */ 233 | p_node = p_list->p_head->p_next; 234 | while (p_node->p_next != NULL) { 235 | size++; 236 | p_node = p_node->p_next; 237 | } 238 | 239 | return size; 240 | } 241 | 242 | ///////////////////////////////////////////////////////// 243 | // HASH LIST SIZE 244 | ///////////////////////////////////////////////////////// 245 | int hash_list_size(hash_list_t *p_hash_list) 246 | { 247 | int i; 248 | int size = 0; 249 | 250 | for (i = 0; i < p_hash_list->n_buckets; i++) { 251 | size += list_size(p_hash_list->buckets[i]); 252 | } 253 | 254 | return size; 255 | } 256 | 257 | ///////////////////////////////////////////////////////// 258 | // LIST CONTAINS 259 | ///////////////////////////////////////////////////////// 260 | int pure_list_contains(list_t *p_list, val_t val) { 261 | node_t *p_prev, *p_next; 262 | 263 | p_prev = p_list->p_head; 264 | p_next = p_prev->p_next; 265 | while (p_next->val < val) { 266 | p_prev = p_next; 267 | p_next = p_prev->p_next; 268 | } 269 | 270 | return p_next->val == val; 271 | } 272 | 273 | node_t *harris_list_search(list_t *p_list, val_t val, node_t **left_node) { 274 | node_t *left_node_next; 275 | node_t *right_node; 276 | 277 | search_again: 278 | 279 | do { 280 | left_node_next = p_list->p_head; 281 | 282 | node_t *t = p_list->p_head; 283 | node_t *t_next = p_list->p_head->p_next; 284 | 285 | /* Find left_node and right_node */ 286 | do { 287 | if (!is_marked_ref((long) t_next)) { 288 | (*left_node) = t; 289 | left_node_next = t_next; 290 | } 291 | 292 | t = (node_t *) get_unmarked_ref((long) t_next); 293 | if (!t->p_next) break; 294 | t_next = t->p_next; 295 | 296 | } while (is_marked_ref((long) t_next) || (t->val < val)); 297 | 298 | right_node = t; 299 | 300 | /* Check that nodes are adjacent */ 301 | if (left_node_next == right_node) { 302 | if (right_node->p_next && is_marked_ref((long) right_node->p_next)) { 303 | goto search_again; 304 | } else { 305 | return right_node; 306 | } 307 | } 308 | 309 | /* Remove one or more marked nodes */ 310 | if (HARRIS_CAS(&(*left_node)->p_next, left_node_next, right_node)) { 311 | if (right_node->p_next && is_marked_ref((long) right_node->p_next)) { 312 | goto search_again; 313 | } else { 314 | return right_node; 315 | } 316 | } 317 | 318 | } while (1); 319 | 320 | } 321 | 322 | node_t *hp_harris_list_search(hp_thread_t *p_hp_td, list_t *p_list, val_t val, node_t **left_node) { 323 | node_t *left_node_next; 324 | node_t *right_node; 325 | node_t *t; 326 | node_t *my_left_node; 327 | 328 | hp_record_t *p_hp_temp; 329 | hp_record_t *p_hp_left_node_next; 330 | hp_record_t *p_hp_right_node; 331 | hp_record_t *p_hp_t; 332 | hp_record_t *p_hp_t_next; 333 | hp_record_t *p_hp_my_left_node; 334 | 335 | my_left_node = *left_node; 336 | 337 | HP_save(p_hp_td); 338 | 339 | search_again: 340 | HP_restore(p_hp_td); 341 | 342 | p_hp_temp = HP_alloc(p_hp_td); 343 | p_hp_left_node_next = HP_alloc(p_hp_td); 344 | p_hp_right_node = HP_alloc(p_hp_td); 345 | p_hp_t = HP_alloc(p_hp_td); 346 | p_hp_t_next = HP_alloc(p_hp_td); 347 | p_hp_my_left_node = HP_alloc(p_hp_td); 348 | 349 | do { 350 | HP_INIT(p_hp_td, p_hp_left_node_next, &(p_list->p_head)); 351 | left_node_next = p_list->p_head; 352 | 353 | HP_INIT(p_hp_td, p_hp_t, &(p_list->p_head)); 354 | t = p_list->p_head; 355 | HP_INIT(p_hp_td, p_hp_t_next, &(p_list->p_head->p_next)); 356 | node_t *t_next = p_list->p_head->p_next; 357 | 358 | /* Find left_node and right_node */ 359 | do { 360 | if (!is_marked_ref((long) t_next)) { 361 | HP_INIT(p_hp_td, p_hp_my_left_node, &(t)); 362 | my_left_node = t; 363 | HP_INIT(p_hp_td, p_hp_left_node_next, &(t_next)); 364 | left_node_next = t_next; 365 | } 366 | 367 | t = (node_t *) get_unmarked_ref((long) t_next); 368 | if (!t->p_next) { 369 | break; 370 | } 371 | 372 | /* Swap hazard pointers */ 373 | p_hp_temp = p_hp_t; 374 | p_hp_t = p_hp_t_next; 375 | p_hp_t_next = p_hp_temp; 376 | 377 | HP_INIT(p_hp_td, p_hp_t_next, &(t->p_next)); 378 | t_next = t->p_next; 379 | 380 | } while (is_marked_ref((long) t_next) || (t->val < val)); 381 | 382 | HP_INIT(p_hp_td, p_hp_right_node, &(t)); 383 | right_node = t; 384 | 385 | /* Check that nodes are adjacent */ 386 | if (left_node_next == right_node) { 387 | if (right_node->p_next && is_marked_ref((long) right_node->p_next)) { 388 | goto search_again; 389 | } else { 390 | (*left_node) = my_left_node; 391 | return right_node; 392 | } 393 | } 394 | 395 | /* Remove one or more marked nodes */ 396 | if (HARRIS_CAS(&(*left_node)->p_next, left_node_next, right_node)) { 397 | if (right_node->p_next && is_marked_ref((long) right_node->p_next)) { 398 | goto search_again; 399 | } else { 400 | (*left_node) = my_left_node; 401 | return right_node; 402 | } 403 | } 404 | 405 | } while (1); 406 | 407 | } 408 | 409 | int harris_list_contains(list_t *p_list, val_t val) { 410 | node_t *right_node, *left_node; 411 | left_node = p_list->p_head; 412 | 413 | right_node = harris_list_search(p_list, val, &left_node); 414 | if ((!right_node->p_next) || right_node->val != val) { 415 | return 0; 416 | } else { 417 | return 1; 418 | } 419 | 420 | } 421 | 422 | int hp_harris_list_contains(hp_thread_t *p_hp_td, list_t *p_list, val_t val) { 423 | node_t *right_node; 424 | node_t *left_node; 425 | 426 | hp_record_t *p_hp_left_node; 427 | 428 | HP_RESET(p_hp_td); 429 | 430 | p_hp_left_node = HP_alloc(p_hp_td); 431 | 432 | HP_INIT(p_hp_td, p_hp_left_node, &(p_list->p_head)); 433 | left_node = p_list->p_head; 434 | 435 | right_node = hp_harris_list_search(p_hp_td, p_list, val, &left_node); 436 | if ((!right_node->p_next) || right_node->val != val) { 437 | HP_RESET(p_hp_td); 438 | return 0; 439 | } else { 440 | HP_RESET(p_hp_td); 441 | return 1; 442 | } 443 | 444 | } 445 | 446 | int rcu_list_contains(list_t *p_list, val_t val) { 447 | int result; 448 | val_t v; 449 | node_t *p_prev, *p_next; 450 | node_t *p_node; 451 | 452 | RCU_READER_LOCK(); 453 | 454 | p_prev = (node_t *)RCU_DEREF(p_list->p_head); 455 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 456 | while (1) { 457 | p_node = (node_t *)RCU_DEREF(p_next); 458 | v = p_node->val; 459 | 460 | if (v >= val) { 461 | break; 462 | } 463 | 464 | p_prev = p_next; 465 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 466 | } 467 | 468 | result = (v == val); 469 | 470 | RCU_READER_UNLOCK(); 471 | 472 | return result; 473 | } 474 | 475 | int rlu_list_contains(rlu_thread_data_t *self, list_t *p_list, val_t val) { 476 | int result; 477 | val_t v; 478 | node_t *p_prev, *p_next; 479 | 480 | RLU_READER_LOCK(self); 481 | 482 | p_prev = (node_t *)RLU_DEREF(self, (p_list->p_head)); 483 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 484 | while (1) { 485 | //p_node = (node_t *)RLU_DEREF(self, p_next); 486 | v = p_next->val; 487 | 488 | if (v >= val) { 489 | break; 490 | } 491 | 492 | p_prev = p_next; 493 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 494 | } 495 | 496 | result = (v == val); 497 | 498 | RLU_READER_UNLOCK(self); 499 | 500 | return result; 501 | } 502 | 503 | ///////////////////////////////////////////////////////// 504 | // HASH LIST CONTAINS 505 | ///////////////////////////////////////////////////////// 506 | int pure_hash_list_contains(hash_list_t *p_hash_list, val_t val) 507 | { 508 | int hash = HASH_VALUE(p_hash_list, val); 509 | 510 | return pure_list_contains(p_hash_list->buckets[hash], val); 511 | } 512 | 513 | int harris_hash_list_contains(hash_list_t *p_hash_list, val_t val) 514 | { 515 | int hash = HASH_VALUE(p_hash_list, val); 516 | 517 | return harris_list_contains(p_hash_list->buckets[hash], val); 518 | } 519 | 520 | int hp_harris_hash_list_contains(hp_thread_t *p_hp_td, hash_list_t *p_hash_list, val_t val) 521 | { 522 | int hash = HASH_VALUE(p_hash_list, val); 523 | 524 | return hp_harris_list_contains(p_hp_td, p_hash_list->buckets[hash], val); 525 | } 526 | 527 | int rcu_hash_list_contains(hash_list_t *p_hash_list, val_t val) 528 | { 529 | int hash = HASH_VALUE(p_hash_list, val); 530 | 531 | return rcu_list_contains(p_hash_list->buckets[hash], val); 532 | } 533 | 534 | int rlu_hash_list_contains(rlu_thread_data_t *self, hash_list_t *p_hash_list, val_t val) 535 | { 536 | int hash = HASH_VALUE(p_hash_list, val); 537 | 538 | return rlu_list_contains(self, p_hash_list->buckets[hash], val); 539 | } 540 | 541 | ///////////////////////////////////////////////////////// 542 | // LIST ADD 543 | ///////////////////////////////////////////////////////// 544 | int pure_list_add(list_t *p_list, val_t val) 545 | { 546 | int result; 547 | node_t *p_prev, *p_next, *p_new_node; 548 | 549 | p_prev = p_list->p_head; 550 | p_next = p_prev->p_next; 551 | while (p_next->val < val) { 552 | p_prev = p_next; 553 | p_next = p_prev->p_next; 554 | } 555 | 556 | result = (p_next->val != val); 557 | 558 | if (result) { 559 | p_new_node = pure_new_node(); 560 | p_new_node->val = val; 561 | p_new_node->p_next = p_next; 562 | 563 | p_prev->p_next = p_new_node; 564 | } 565 | 566 | return result; 567 | } 568 | 569 | int harris_list_add(list_t *p_list, val_t val) { 570 | node_t *new_node, *right_node, *left_node; 571 | left_node = p_list->p_head; 572 | 573 | do { 574 | right_node = harris_list_search(p_list, val, &left_node); 575 | if (right_node->val == val) { 576 | return 0; 577 | } 578 | new_node = harris_new_node(); 579 | new_node->val = val; 580 | new_node->p_next = right_node; 581 | 582 | /* mem-bar between node creation and insertion */ 583 | MEMBARSTLD(); 584 | if (HARRIS_CAS(&left_node->p_next, right_node, new_node)) { 585 | return 1; 586 | } 587 | 588 | } while(1); 589 | } 590 | 591 | int hp_harris_list_add(hp_thread_t *p_hp_td, list_t *p_list, val_t val) { 592 | node_t *new_node; 593 | node_t *right_node; 594 | node_t *left_node; 595 | 596 | hp_record_t *p_hp_left_node; 597 | 598 | do { 599 | HP_RESET(p_hp_td); 600 | 601 | p_hp_left_node = HP_alloc(p_hp_td); 602 | 603 | HP_INIT(p_hp_td, p_hp_left_node, &(p_list->p_head)); 604 | left_node = p_list->p_head; 605 | 606 | right_node = hp_harris_list_search(p_hp_td, p_list, val, &left_node); 607 | if (right_node->val == val) { 608 | HP_RESET(p_hp_td); 609 | return 0; 610 | } 611 | new_node = harris_new_node(); 612 | new_node->val = val; 613 | new_node->p_next = right_node; 614 | 615 | /* mem-bar between node creation and insertion */ 616 | MEMBARSTLD(); 617 | if (HARRIS_CAS(&left_node->p_next, right_node, new_node)) { 618 | HP_RESET(p_hp_td); 619 | return 1; 620 | } 621 | 622 | } while(1); 623 | } 624 | 625 | int rcu_list_add(list_t *p_list, val_t val, int bucket) { 626 | int result; 627 | node_t *p_prev, *p_next; 628 | node_t *p_node; 629 | val_t v; 630 | 631 | RCU_WRITER_LOCK(bucket); 632 | 633 | p_prev = (node_t *)RCU_DEREF(p_list->p_head); 634 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 635 | while (1) { 636 | p_node = (node_t *)RCU_DEREF(p_next); 637 | v = p_node->val; 638 | 639 | if (v >= val) { 640 | break; 641 | } 642 | 643 | p_prev = p_next; 644 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 645 | } 646 | 647 | result = (v != val); 648 | 649 | if (result) { 650 | node_t *p_new_node = rcu_new_node(); 651 | p_new_node->val = val; 652 | p_new_node->p_next = p_next; 653 | 654 | RCU_ASSIGN_PTR(&(p_prev->p_next), p_new_node); 655 | } 656 | 657 | RCU_WRITER_UNLOCK(bucket); 658 | 659 | return result; 660 | } 661 | 662 | int rlu_list_add(rlu_thread_data_t *self, list_t *p_list, val_t val) { 663 | int result; 664 | node_t *p_prev, *p_next; 665 | val_t v; 666 | 667 | restart: 668 | RLU_READER_LOCK(self); 669 | 670 | p_prev = (node_t *)RLU_DEREF(self, (p_list->p_head)); 671 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 672 | while (1) { 673 | //p_node = (node_t *)RLU_DEREF(self, p_next); 674 | v = p_next->val; 675 | 676 | if (v >= val) { 677 | break; 678 | } 679 | 680 | p_prev = p_next; 681 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 682 | } 683 | 684 | result = (v != val); 685 | 686 | if (result) { 687 | 688 | if (!RLU_TRY_LOCK(self, &p_prev)) { 689 | RLU_ABORT(self); 690 | goto restart; 691 | } 692 | 693 | if (!RLU_TRY_LOCK(self, &p_next)) { 694 | RLU_ABORT(self); 695 | goto restart; 696 | } 697 | 698 | node_t *p_new_node = rlu_new_node(); 699 | p_new_node->val = val; 700 | RLU_ASSIGN_PTR(self, &(p_new_node->p_next), p_next); 701 | 702 | RLU_ASSIGN_PTR(self, &(p_prev->p_next), p_new_node); 703 | } 704 | 705 | RLU_READER_UNLOCK(self); 706 | 707 | return result; 708 | } 709 | 710 | ///////////////////////////////////////////////////////// 711 | // HASH LIST ADD 712 | ///////////////////////////////////////////////////////// 713 | int pure_hash_list_add(hash_list_t *p_hash_list, val_t val) 714 | { 715 | int hash = HASH_VALUE(p_hash_list, val); 716 | 717 | return pure_list_add(p_hash_list->buckets[hash], val); 718 | } 719 | 720 | int harris_hash_list_add(hash_list_t *p_hash_list, val_t val) 721 | { 722 | int hash = HASH_VALUE(p_hash_list, val); 723 | 724 | return harris_list_add(p_hash_list->buckets[hash], val); 725 | } 726 | 727 | int hp_harris_hash_list_add(hp_thread_t *p_hp_td, hash_list_t *p_hash_list, val_t val) 728 | { 729 | int hash = HASH_VALUE(p_hash_list, val); 730 | 731 | return hp_harris_list_add(p_hp_td, p_hash_list->buckets[hash], val); 732 | } 733 | 734 | int rcu_hash_list_add(hash_list_t *p_hash_list, val_t val) 735 | { 736 | int hash = HASH_VALUE(p_hash_list, val); 737 | 738 | return rcu_list_add(p_hash_list->buckets[hash], val, hash); 739 | } 740 | 741 | int rlu_hash_list_add(rlu_thread_data_t *self, hash_list_t *p_hash_list, val_t val) 742 | { 743 | int hash = HASH_VALUE(p_hash_list, val); 744 | 745 | return rlu_list_add(self, p_hash_list->buckets[hash], val); 746 | } 747 | 748 | 749 | ///////////////////////////////////////////////////////// 750 | // LIST REMOVE 751 | ///////////////////////////////////////////////////////// 752 | int pure_list_remove(list_t *p_list, val_t val) { 753 | int result; 754 | node_t *p_prev, *p_next; 755 | 756 | p_prev = p_list->p_head; 757 | p_next = p_prev->p_next; 758 | while (p_next->val < val) { 759 | p_prev = p_next; 760 | p_next = p_prev->p_next; 761 | } 762 | 763 | result = (p_next->val == val); 764 | 765 | if (result) { 766 | p_prev->p_next = p_next->p_next; 767 | pure_free_node(p_next); 768 | } 769 | 770 | return result; 771 | } 772 | 773 | int harris_list_remove(list_t *p_list, val_t val) { 774 | node_t *right_node, *right_node_next, *left_node; 775 | left_node = p_list->p_head; 776 | 777 | do { 778 | right_node = harris_list_search(p_list, val, &left_node); 779 | if (right_node->val != val) { 780 | return 0; 781 | } 782 | 783 | right_node_next = right_node->p_next; 784 | if (!is_marked_ref((long) right_node_next)) { 785 | if (HARRIS_CAS(&right_node->p_next, right_node_next, get_marked_ref((long) right_node_next))) { 786 | break; 787 | } 788 | } 789 | 790 | } while(1); 791 | 792 | if (!HARRIS_CAS(&left_node->p_next, right_node, right_node_next)) { 793 | right_node = harris_list_search(p_list, right_node->val, &left_node); 794 | } 795 | 796 | return 1; 797 | } 798 | 799 | int hp_harris_list_remove(hp_thread_t *p_hp_td, list_t *p_list, val_t val) { 800 | node_t *right_node; 801 | node_t *right_node_next; 802 | node_t *left_node; 803 | 804 | hp_record_t *p_hp_right_node_next; 805 | hp_record_t *p_hp_left_node; 806 | 807 | do { 808 | HP_RESET(p_hp_td); 809 | 810 | p_hp_left_node = HP_alloc(p_hp_td); 811 | p_hp_right_node_next = HP_alloc(p_hp_td); 812 | 813 | HP_INIT(p_hp_td, p_hp_left_node, &(p_list->p_head)); 814 | left_node = p_list->p_head; 815 | 816 | right_node = hp_harris_list_search(p_hp_td, p_list, val, &left_node); 817 | if (right_node->val != val) { 818 | HP_RESET(p_hp_td); 819 | return 0; 820 | } 821 | 822 | HP_INIT(p_hp_td, p_hp_right_node_next, &(right_node->p_next)); 823 | right_node_next = right_node->p_next; 824 | 825 | if (!is_marked_ref((long) right_node_next)) { 826 | if (HARRIS_CAS(&right_node->p_next, right_node_next, get_marked_ref((long) right_node_next))) { 827 | break; 828 | } 829 | } 830 | 831 | } while(1); 832 | 833 | if (!HARRIS_CAS(&left_node->p_next, right_node, right_node_next)) { 834 | right_node = harris_list_search(p_list, right_node->val, &left_node); 835 | } 836 | 837 | HP_RESET(p_hp_td); 838 | return 1; 839 | } 840 | 841 | int rcu_list_remove(list_t *p_list, val_t val, int bucket) { 842 | int result; 843 | node_t *p_prev, *p_next; 844 | node_t *p_node; 845 | node_t *n; 846 | 847 | RCU_WRITER_LOCK(bucket); 848 | 849 | p_prev = (node_t *)RCU_DEREF(p_list->p_head); 850 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 851 | while (1) { 852 | p_node = (node_t *)RCU_DEREF(p_next); 853 | 854 | if (p_node->val >= val) { 855 | break; 856 | } 857 | 858 | p_prev = p_next; 859 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 860 | } 861 | 862 | result = (p_node->val == val); 863 | 864 | if (result) { 865 | n = (node_t *)RCU_DEREF(p_next->p_next); 866 | 867 | RCU_ASSIGN_PTR(&(p_prev->p_next), n); 868 | 869 | RCU_WRITER_UNLOCK(bucket); 870 | 871 | rcu_free_node(p_next); 872 | 873 | return result; 874 | } 875 | 876 | RCU_WRITER_UNLOCK(bucket); 877 | 878 | return result; 879 | } 880 | 881 | int rlu_list_remove(rlu_thread_data_t *self, list_t *p_list, val_t val) { 882 | int result; 883 | node_t *p_prev, *p_next; 884 | node_t *n; 885 | val_t v; 886 | 887 | restart: 888 | RLU_READER_LOCK(self); 889 | 890 | p_prev = (node_t *)RLU_DEREF(self, (p_list->p_head)); 891 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 892 | while (1) { 893 | //p_node = (node_t *)RLU_DEREF(self, p_next); 894 | 895 | v = p_next->val; 896 | 897 | if (v >= val) { 898 | break; 899 | } 900 | 901 | p_prev = p_next; 902 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 903 | } 904 | 905 | result = (v == val); 906 | 907 | if (result) { 908 | n = (node_t *)RLU_DEREF(self, (p_next->p_next)); 909 | 910 | if (!RLU_TRY_LOCK(self, &p_prev)) { 911 | RLU_ABORT(self); 912 | goto restart; 913 | } 914 | 915 | if (!RLU_TRY_LOCK(self, &p_next)) { 916 | RLU_ABORT(self); 917 | goto restart; 918 | } 919 | 920 | RLU_ASSIGN_PTR(self, &(p_prev->p_next), n); 921 | 922 | RLU_FREE(self, p_next); 923 | 924 | RLU_READER_UNLOCK(self); 925 | 926 | return result; 927 | } 928 | 929 | RLU_READER_UNLOCK(self); 930 | 931 | return result; 932 | } 933 | 934 | ///////////////////////////////////////////////////////// 935 | // HASH LIST REMOVE 936 | ///////////////////////////////////////////////////////// 937 | int pure_hash_list_remove(hash_list_t *p_hash_list, val_t val) 938 | { 939 | int hash = HASH_VALUE(p_hash_list, val); 940 | 941 | return pure_list_remove(p_hash_list->buckets[hash], val); 942 | } 943 | 944 | int harris_hash_list_remove(hash_list_t *p_hash_list, val_t val) 945 | { 946 | int hash = HASH_VALUE(p_hash_list, val); 947 | 948 | return harris_list_remove(p_hash_list->buckets[hash], val); 949 | } 950 | 951 | int hp_harris_hash_list_remove(hp_thread_t *p_hp_td, hash_list_t *p_hash_list, val_t val) 952 | { 953 | int hash = HASH_VALUE(p_hash_list, val); 954 | 955 | return hp_harris_list_remove(p_hp_td, p_hash_list->buckets[hash], val); 956 | } 957 | 958 | int rcu_hash_list_remove(hash_list_t *p_hash_list, val_t val) 959 | { 960 | int hash = HASH_VALUE(p_hash_list, val); 961 | 962 | return rcu_list_remove(p_hash_list->buckets[hash], val, hash); 963 | } 964 | 965 | int rlu_hash_list_remove(rlu_thread_data_t *self, hash_list_t *p_hash_list, val_t val) 966 | { 967 | int hash = HASH_VALUE(p_hash_list, val); 968 | 969 | return rlu_list_remove(self, p_hash_list->buckets[hash], val); 970 | } 971 | 972 | -------------------------------------------------------------------------------- /hash-list.h: -------------------------------------------------------------------------------- 1 | #ifndef _HASH_LIST_H_ 2 | #define _HASH_LIST_H_ 3 | 4 | ///////////////////////////////////////////////////////// 5 | // INCLUDES 6 | ///////////////////////////////////////////////////////// 7 | #include "hazard_ptrs.h" 8 | #include "new-urcu.h" 9 | #include "rlu.h" 10 | 11 | ///////////////////////////////////////////////////////// 12 | // DEFINES 13 | ///////////////////////////////////////////////////////// 14 | #define LIST_VAL_MIN (INT_MIN) 15 | #define LIST_VAL_MAX (INT_MAX) 16 | 17 | #define NODE_PADDING (16) 18 | 19 | #define MAX_BUCKETS (20000) 20 | 21 | ///////////////////////////////////////////////////////// 22 | // TYPES 23 | ///////////////////////////////////////////////////////// 24 | typedef intptr_t val_t; 25 | 26 | typedef struct node node_t; 27 | typedef struct node { 28 | val_t val; 29 | node_t *p_next; 30 | 31 | long padding[NODE_PADDING]; 32 | } node_t; 33 | 34 | typedef struct list { 35 | node_t *p_head; 36 | } list_t; 37 | 38 | typedef struct hash_list { 39 | int n_buckets; 40 | list_t *buckets[MAX_BUCKETS]; 41 | } hash_list_t; 42 | 43 | ///////////////////////////////////////////////////////// 44 | // INTERFACE 45 | ///////////////////////////////////////////////////////// 46 | hash_list_t *pure_new_hash_list(int n_buckets); 47 | hash_list_t *harris_new_hash_list(int n_buckets); 48 | hash_list_t *hp_harris_new_hash_list(int n_buckets); 49 | hash_list_t *rcu_new_hash_list(int n_buckets); 50 | hash_list_t *rlu_new_hash_list(int n_buckets); 51 | 52 | int hash_list_size(hash_list_t *p_hash_list); 53 | 54 | int pure_hash_list_contains(hash_list_t *p_hash_list, val_t val); 55 | int pure_hash_list_add(hash_list_t *p_hash_list, val_t val); 56 | int pure_hash_list_remove(hash_list_t *p_hash_list, val_t val); 57 | 58 | int harris_hash_list_contains(hash_list_t *p_hash_list, val_t val); 59 | int harris_hash_list_add(hash_list_t *p_hash_list, val_t val); 60 | int harris_hash_list_remove(hash_list_t *p_hash_list, val_t val); 61 | 62 | int hp_harris_hash_list_contains(hp_thread_t *p_hp_td, hash_list_t *p_hash_list, val_t val); 63 | int hp_harris_hash_list_add(hp_thread_t *p_hp_td, hash_list_t *p_hash_list, val_t val); 64 | int hp_harris_hash_list_remove(hp_thread_t *p_hp_td, hash_list_t *p_hash_list, val_t val); 65 | 66 | int rcu_hash_list_contains(hash_list_t *p_hash_list, val_t val); 67 | int rcu_hash_list_add(hash_list_t *p_hash_list, val_t val); 68 | int rcu_hash_list_remove(hash_list_t *p_hash_list, val_t val); 69 | 70 | int rlu_hash_list_contains(rlu_thread_data_t *self, hash_list_t *p_hash_list, val_t val); 71 | int rlu_hash_list_add(rlu_thread_data_t *self, hash_list_t *p_hash_list, val_t val); 72 | int rlu_hash_list_remove(rlu_thread_data_t *self, hash_list_t *p_hash_list, val_t val); 73 | 74 | #endif // _HASH_LIST_H_ -------------------------------------------------------------------------------- /hazard_ptrs.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include "hazard_ptrs.h" 5 | 6 | ///////////////////////////////////////////////////////// 7 | // DEFINES 8 | ///////////////////////////////////////////////////////// 9 | #define CPU_RELAX() asm volatile("pause\n": : :"memory"); 10 | #define MEMBARSTLD() __sync_synchronize() 11 | 12 | ///////////////////////////////////////////////////////// 13 | // EXTERNAL FUNCTIONS 14 | ///////////////////////////////////////////////////////// 15 | void HP_reset(hp_thread_t *self) { 16 | self->n_hp_records = 0; 17 | } 18 | 19 | void HP_save(hp_thread_t *self) { 20 | self->saved_n_hp_records = self->n_hp_records; 21 | } 22 | 23 | void HP_restore(hp_thread_t *self) { 24 | self->n_hp_records = self->saved_n_hp_records; 25 | } 26 | 27 | hp_record_t *HP_alloc(hp_thread_t *self) { 28 | hp_record_t *p_hp; 29 | 30 | p_hp = &(self->hp_records[self->n_hp_records]); 31 | self->n_hp_records++; 32 | 33 | if (self->n_hp_records >= HP_MAX_RECORDS) { 34 | abort(); 35 | } 36 | 37 | return p_hp; 38 | 39 | } 40 | 41 | void HP_init(hp_record_t *p_hp, volatile int64_t **ptr_ptr) { 42 | 43 | while (1) { 44 | p_hp->ptr = *ptr_ptr; 45 | MEMBARSTLD(); 46 | 47 | if (p_hp->ptr == *ptr_ptr) { 48 | return; 49 | } 50 | 51 | CPU_RELAX(); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /hazard_ptrs.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef HAZARD_PTRS_H 3 | #define HAZARD_PTRS 1 4 | 5 | ///////////////////////////////////////////////////////// 6 | // INCLUDES 7 | ///////////////////////////////////////////////////////// 8 | #ifdef __x86_64 9 | #include 10 | #endif 11 | 12 | ///////////////////////////////////////////////////////// 13 | // DEFINES 14 | ///////////////////////////////////////////////////////// 15 | #define IS_HP_ENABLED 16 | 17 | #define HP_MAX_RECORDS (100) 18 | 19 | ///////////////////////////////////////////////////////// 20 | // TYPES 21 | ///////////////////////////////////////////////////////// 22 | typedef struct _hp_record_t { 23 | volatile int64_t *ptr; 24 | } hp_record_t; 25 | 26 | typedef struct _hp_thread_t { 27 | long saved_n_hp_records; 28 | long n_hp_records; 29 | hp_record_t hp_records[HP_MAX_RECORDS]; 30 | 31 | } hp_thread_t ; 32 | 33 | 34 | ///////////////////////////////////////////////////////// 35 | // EXTERNAL FUNCTIONS 36 | ///////////////////////////////////////////////////////// 37 | void HP_reset(hp_thread_t *self); 38 | void HP_save(hp_thread_t *p_hp); 39 | void HP_restore(hp_thread_t *p_hp); 40 | 41 | hp_record_t *HP_alloc(hp_thread_t *self); 42 | void HP_init(hp_record_t *p_hp, volatile int64_t **ptr_ptr); 43 | 44 | #ifdef IS_HP_ENABLED 45 | 46 | #define HP_RESET(self) HP_reset(self) 47 | #define HP_SAVE(self) HP_save(self) 48 | #define HP_RESTORE(self) HP_restore(self) 49 | 50 | #define HP_ALLOC(self) HP_alloc(self) 51 | #define HP_INIT(self, p_hp, ptr_ptr) HP_init(p_hp, (volatile int64_t **)ptr_ptr) 52 | 53 | #else 54 | 55 | #define HP_RESET(self) 56 | #define HP_SAVE(self) 57 | #define HP_RESTORE(self) 58 | 59 | #define HP_ALLOC(self) 60 | #define HP_INIT(self, p_hp, ptr_ptr) 61 | 62 | #endif 63 | 64 | #endif // HAZARD_PTRS 65 | -------------------------------------------------------------------------------- /init_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #echo Set paths: 4 | #echo export LD_LIBRARY_PATH=/usr/local/lib 5 | #export LD_LIBRARY_PATH=/usr/local/lib 6 | 7 | echo Set malloc: 8 | 9 | echo export LD_PRELOAD=/home/amatveev/tools/gperftools-2.1/.libs/libtcmalloc_minimal.so 10 | export LD_PRELOAD=/home/amatveev/tools/gperftools-2.1/.libs/libtcmalloc_minimal.so 11 | #echo export GPERFTOOLS_LIB=/home/amatveev/tools/gperftools-2.1/.libs 12 | #export GPERFTOOLS_LIB=/home/amatveev/tools/gperftools-2.1/.libs 13 | 14 | #echo export LD_PRELOAD="/localdisk/amatveev/tools/gperftools-2.4/.libs/libtcmalloc_minimal.so" 15 | #export LD_PRELOAD="/localdisk/amatveev/tools/gperftools-2.4/.libs/libtcmalloc_minimal.so" 16 | -------------------------------------------------------------------------------- /kernel-bench/Makefile: -------------------------------------------------------------------------------- 1 | obj-m += sync.o 2 | sync-objs := sync_test.o barrier.o rlu.o rlu-hash-list.o rcu-hash-list.o rcuintset.o # rluintset.o 3 | 4 | CFLAGS_rluintset.o := -DRUN_RLU -DKERNEL 5 | CFLAGS_rcuintset.o := -DRUN_RCU -DKERNEL 6 | CFLAGS_rlu.o := -DKERNEL 7 | CFLAGS_rlu-hash-list.o := -DKERNEL 8 | CFLAGS_sync_test.o := -DKERNEL 9 | # To enable pr_debug/pr_devel in dmesg 10 | #CFLAGS_sync_test.o := -DDEBUG 11 | 12 | # V=1 for debug 13 | 14 | all: 15 | make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules 16 | 17 | clean: 18 | make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean 19 | 20 | info: 21 | modinfo sync.ko 22 | 23 | bench: 24 | dmesg -c > /dev/null 25 | insmod sync.ko benchmark="rculist" threads_nb=1 update=0 range=256 duration=500 26 | rmmod sync.ko 27 | -------------------------------------------------------------------------------- /kernel-bench/README.md: -------------------------------------------------------------------------------- 1 | Read-Log-Update: A Lightweight Synchronization Mechanism for Concurrent Programming 2 | =================================================================================== 3 | 4 | Linux kernel experiments 5 | ------------------------ 6 | This directory contains 2 micro benchmarks: The singly-linked list and hash list benchmarks. 7 | Those implementations are close to the one used in the user-space. 8 | These tests uses the same RLU implementation as the user-space implementation and the kernel RCU implementation. 9 | When the macro KERNEL is defined, it enables the Linux kernel specific part in RLU source code. 10 | 11 | Building the kernel module 12 | -------------------------- 13 | A simple 'make' creates the "sync.ko" kernel module that includes the 2 benchmarks and the RLU and RCU implementation. 14 | This 'make' uses your current kernel (headers and objects) so it has to be recompiled to be used for another kernel. 15 | 16 | Running tests with the kernel module 17 | ------------------------------------ 18 | To run one test, load the sync.ko module (this requires privileged mode, root or sudo). 19 | The module accepts some parameters to run: 20 | 21 | * benchmark : the benchmark to run, "rcuhashlist", "rluhashlist", "rcuintset", "rluintset". 22 | * threads\_nb : the number of threads to run. 23 | * update : the mutation rate (insert/delete) * 100 (10=0.1% update). The rest is read only. 24 | * range : the key range, the initial size is half the key range. 25 | * duration : the duration of the execution in ms. 26 | 27 | Example using root: 28 | 29 | ``` 30 | # insmod sync.ko benchmark="rculist" threads_nb=1 update=0 range=256 duration=500 31 | ``` 32 | 33 | -------------------------------------------------------------------------------- /kernel-bench/barrier.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include // schedule 3 | #include // cpu_relax 4 | 5 | #include "barrier.h" 6 | 7 | /* TODO make barrier that reinitialize itself */ 8 | 9 | void barrier_init(barrier_t *barrier, int n) 10 | { 11 | atomic_set(&barrier->count, n); 12 | atomic_set(&barrier->crossing, 0); 13 | } 14 | 15 | void barrier_cross(barrier_t *barrier) 16 | { 17 | int crossing; 18 | int count = atomic_read(&barrier->count); 19 | crossing = atomic_add_return(1, &barrier->crossing); 20 | while (crossing != count) { 21 | cpu_relax(); 22 | crossing = atomic_read(&barrier->crossing); 23 | /* Force all threads to be scheduled (note this may delay a bit the start but no choice) */ 24 | schedule(); 25 | } 26 | } 27 | 28 | -------------------------------------------------------------------------------- /kernel-bench/barrier.h: -------------------------------------------------------------------------------- 1 | #ifndef _BARRIER_H 2 | #define _BARRIER_H 3 | 4 | //#include // __cacheline_aligned 5 | #include 6 | 7 | typedef struct barrier { 8 | atomic_t count; 9 | atomic_t crossing; 10 | } barrier_t; 11 | 12 | void barrier_init(barrier_t *barrier, int n); 13 | void barrier_cross(barrier_t *barrier); 14 | 15 | #endif /* _BARRIER_H */ 16 | -------------------------------------------------------------------------------- /kernel-bench/hash-list.h: -------------------------------------------------------------------------------- 1 | #ifndef _HASH_LIST_H_ 2 | #define _HASH_LIST_H_ 3 | 4 | ///////////////////////////////////////////////////////// 5 | // INCLUDES 6 | ///////////////////////////////////////////////////////// 7 | #include 8 | 9 | ///////////////////////////////////////////////////////// 10 | // DEFINES 11 | ///////////////////////////////////////////////////////// 12 | #define LIST_VAL_MIN (INT_MIN) 13 | #define LIST_VAL_MAX (INT_MAX) 14 | 15 | #define NODE_PADDING (30) 16 | #define CACHELINE_SIZE (128) 17 | 18 | #define MAX_BUCKETS (1000) 19 | #define DEFAULT_BUCKETS 1 20 | 21 | ///////////////////////////////////////////////////////// 22 | // TYPES 23 | ///////////////////////////////////////////////////////// 24 | typedef int val_t; 25 | 26 | typedef union node node_t; 27 | typedef union node { 28 | struct { 29 | val_t val; 30 | node_t *p_next; 31 | struct rcu_head rcu; 32 | }; 33 | char * padding[CACHELINE_SIZE]; 34 | } node_t; 35 | 36 | typedef union list { 37 | struct { 38 | node_t *p_head; 39 | spinlock_t rcuspin; 40 | }; 41 | char *padding[CACHELINE_SIZE]; 42 | } list_t; 43 | 44 | typedef struct hash_list { 45 | int n_buckets; 46 | list_t *buckets[MAX_BUCKETS]; 47 | char *padding[CACHELINE_SIZE]; 48 | } hash_list_t; 49 | 50 | ///////////////////////////////////////////////////////// 51 | // INTERFACE 52 | ///////////////////////////////////////////////////////// 53 | hash_list_t *rcu_new_hash_list(int n_buckets); 54 | hash_list_t *rlu_new_hash_list(int n_buckets); 55 | 56 | int rcu_hash_list_init(void); 57 | int rcu_hash_list_contains(void *tl, val_t val); 58 | int rcu_hash_list_add(void *tl, val_t val); 59 | int rcu_hash_list_remove(void *tl, val_t val); 60 | 61 | int rlu_hash_list_init(void); 62 | int rlu_hash_list_contains(void *self, val_t val); 63 | int rlu_hash_list_add(void *self, val_t val); 64 | int rlu_hash_list_remove(void *self, val_t val); 65 | 66 | #endif // _HASH_LIST_H_ 67 | -------------------------------------------------------------------------------- /kernel-bench/intset.c: -------------------------------------------------------------------------------- 1 | #include // kmalloc 2 | #include 3 | #include 4 | #include "rlu.h" 5 | 6 | 7 | /////// 8 | /////// 9 | 10 | /////// 11 | /////// 12 | #ifdef DEBUG 13 | # define IO_FLUSH fflush(NULL) 14 | /* Note: stdio is thread-safe */ 15 | #endif 16 | 17 | /////////////////////////////// 18 | // CONFIGURATION 19 | /////////////////////////////// 20 | #define NODE_PADDING (20) 21 | 22 | #define MAX_VALUES (0) 23 | 24 | //#define IS_NODE_LOCAL_WORK 25 | 26 | /////////////////////////////// 27 | 28 | #define MAX_HASH_BUCKETS (10000) 29 | 30 | #define MAX_FREE_NODES (10000) 31 | 32 | #define HASH_VALUE(hash_set, val) (val % hash_set->hash_buckets) 33 | 34 | #define DEFAULT_HASH_BUCKETS (1) 35 | #define DEFAULT_BATCH_SIZE (1) 36 | 37 | #define XSTR(s) STR(s) 38 | #define STR(s) #s 39 | 40 | #ifdef RUN_RCU 41 | 42 | #define RCU_READ_LOCK(self) rcu_read_lock() 43 | #define RCU_READ_UNLOCK(self) rcu_read_unlock() 44 | #define RCU_SYNCHRONIZE synchronize_rcu() 45 | #define RCU_ASSIGN_POINTER(self, p_ptr, p_obj) \ 46 | rcu_assign_pointer(p_ptr, p_obj) 47 | 48 | #define RCU_DEREFERENCE(self, p_obj) (p_obj) 49 | 50 | #define RCU_LOCK(self) spin_lock(&rcuspin) 51 | #define RCU_UNLOCK(self) spin_unlock(&rcuspin) 52 | 53 | #define RCU_INIT 54 | #define RCU_EXIT 55 | #define RCU_INIT_THREAD(self) 56 | #define RCU_EXIT_THREAD(self) 57 | 58 | #else 59 | #ifdef RUN_RLU 60 | 61 | #define RCU_READ_LOCK(self) RLU_READER_LOCK(self) 62 | #define RCU_READ_UNLOCK(self) RLU_READER_UNLOCK(self) 63 | #define RCU_SYNCHRONIZE //synchronize_rcu() 64 | 65 | #define RLU_ALLOC(obj_size) rlu_alloc(obj_size) 66 | 67 | #define RLU_FORCE_REAL_REF(p_obj) ((node_t *)FORCE_REAL_OBJ((intptr_t *)p_obj)) 68 | #define RLU_FREE(p_obj) (rlu_free(NULL, (intptr_t *)p_obj)) 69 | #define TH_RLU_FREE(self, p_obj) (rlu_free(self, (intptr_t *)p_obj)) 70 | 71 | #define RCU_ASSIGN_POINTER(self, p_ptr, p_obj) \ 72 | rlu_assign_pointer(self, (intptr_t **)p_ptr, (intptr_t *)p_obj) 73 | 74 | #define RLU_LOCK_REF(self, p_obj) ( rlu_obj_lock(self, (intptr_t **)p_obj, sizeof(node_t)) ) 75 | #define RCU_DEREFERENCE(self, p_obj) ( (node_t *)RLU_DEREFERENCE(self, (intptr_t *)p_obj) ) 76 | 77 | #define RCU_LOCK(self) RLU_WRITER_LOCK(self) 78 | #define RCU_UNLOCK(self) RLU_WRITER_UNLOCK(self) 79 | 80 | #define RCU_INIT rlu_init() 81 | #define RCU_EXIT //pthread_mutex_destroy(&rculock) 82 | #define RCU_INIT_THREAD(self) rlu_thread_init(self, 64) 83 | #define RCU_EXIT_THREAD(self) rlu_thread_finish(self) 84 | 85 | 86 | 87 | #else 88 | #error define RUN_RCU or RUN_RLU 89 | #endif 90 | #endif 91 | 92 | /* ################################################################### * 93 | * GLOBALS 94 | * ################################################################### */ 95 | 96 | #ifdef RUN_RCU 97 | __cacheline_aligned static DEFINE_SPINLOCK(rcuspin); 98 | #endif 99 | 100 | /* ################################################################### * 101 | * LINKEDLIST 102 | * ################################################################### */ 103 | 104 | # define INIT_SET_PARAMETERS /* Nothing */ 105 | 106 | typedef intptr_t val_t; 107 | # define VAL_MIN INT_MIN 108 | # define VAL_MAX INT_MAX 109 | 110 | typedef struct node { 111 | val_t val; 112 | struct node *next; 113 | 114 | long values[MAX_VALUES]; 115 | #ifdef RUN_RCU 116 | struct rcu_head rcu; 117 | #endif 118 | 119 | #ifdef NODE_PADDING 120 | long padding[NODE_PADDING]; 121 | #endif 122 | } node_t; 123 | 124 | typedef struct intset { 125 | node_t *head; 126 | } intset_t; 127 | 128 | typedef struct hash_intset { 129 | int hash_buckets; 130 | intset_t *entry[MAX_HASH_BUCKETS]; 131 | } hash_intset_t; 132 | 133 | 134 | __cacheline_aligned static hash_intset_t * intset_set = NULL; 135 | 136 | 137 | static void node_init_values(node_t *p_node) { 138 | int i; 139 | 140 | for (i = 0; i < MAX_VALUES; i++) { 141 | p_node->values[i] = i + 1; 142 | } 143 | } 144 | 145 | static long node_compute(node_t *p_node) { 146 | #ifdef IS_NODE_LOCAL_WORK 147 | int i; 148 | long sum; 149 | 150 | for (i = 0; i < MAX_VALUES; i++) { 151 | sum += p_node->values[i]; 152 | } 153 | 154 | return sum /= MAX_VALUES; 155 | #else 156 | return 0; 157 | #endif 158 | } 159 | 160 | static void node_free(node_t *node_to_free, rlu_thread_data_t *td) { 161 | if (td == NULL) { 162 | #ifdef RUN_RLU 163 | RLU_FREE(node_to_free); 164 | #else 165 | kfree(node_to_free); 166 | #endif 167 | return; 168 | } 169 | 170 | #ifdef RUN_RLU 171 | TH_RLU_FREE(td, node_to_free); 172 | #else 173 | kfree_rcu(node_to_free, rcu); 174 | #endif 175 | 176 | return; 177 | } 178 | 179 | static node_t *new_node(val_t val, node_t *next) 180 | { 181 | node_t *node; 182 | 183 | #ifdef RUN_RLU 184 | node = (node_t *)RLU_ALLOC(sizeof(node_t)); 185 | #else 186 | node = (node_t *)kmalloc(sizeof(node_t), GFP_KERNEL); 187 | #endif 188 | if (node == NULL) { 189 | pr_err("%s: Cannot allocate memory", __FUNCTION__); 190 | return NULL; 191 | } 192 | 193 | node->val = val; 194 | node->next = next; 195 | 196 | node_init_values(node); 197 | 198 | return node; 199 | } 200 | 201 | #ifdef RUN_RLU 202 | static node_t *rlu_new_node(void) 203 | { 204 | return new_node(0, 0); 205 | } 206 | #endif 207 | 208 | static intset_t *set_new(void) 209 | { 210 | intset_t *set; 211 | node_t *min, *max; 212 | 213 | if ((set = (intset_t *)kmalloc(sizeof(intset_t), GFP_KERNEL)) == NULL) { 214 | pr_err("%s: Cannot allocate memory", __FUNCTION__); 215 | return NULL; 216 | } 217 | max = new_node(VAL_MAX, NULL); 218 | min = new_node(VAL_MIN, max); 219 | set->head = min; 220 | 221 | return set; 222 | } 223 | 224 | static hash_intset_t *hash_set_new(int hash_buckets) 225 | { 226 | int i; 227 | 228 | hash_intset_t *hash_set; 229 | 230 | if ((hash_set = (hash_intset_t *)kmalloc(sizeof(hash_intset_t), GFP_KERNEL)) == NULL) { 231 | pr_err("%s: Cannot allocate memory", __FUNCTION__); 232 | return NULL; 233 | } 234 | 235 | hash_set->hash_buckets = hash_buckets; 236 | 237 | for (i = 0; i < hash_set->hash_buckets; i++) { 238 | hash_set->entry[i] = set_new(); 239 | } 240 | 241 | return hash_set; 242 | } 243 | 244 | static void set_delete(intset_t *set) 245 | { 246 | node_t *node, *next; 247 | 248 | node = set->head; 249 | while (node != NULL) { 250 | next = node->next; 251 | 252 | node_free(node, NULL); 253 | 254 | node = next; 255 | } 256 | 257 | kfree(set); 258 | } 259 | 260 | static void hash_set_delete(hash_intset_t *hash_set) { 261 | int i; 262 | 263 | for (i = 0; i < hash_set->hash_buckets; i++) { 264 | set_delete(hash_set->entry[i]); 265 | hash_set->entry[i] = NULL; 266 | } 267 | 268 | kfree(hash_set); 269 | } 270 | 271 | static int set_size(intset_t *set) 272 | { 273 | int size = 0; 274 | node_t *node; 275 | 276 | /* We have at least 2 elements */ 277 | node = set->head->next; 278 | while (node->next != NULL) { 279 | size++; 280 | node = node->next; 281 | } 282 | 283 | return size; 284 | } 285 | 286 | static int hash_set_size(hash_intset_t *hash_set) 287 | { 288 | int i; 289 | int size = 0; 290 | 291 | for (i = 0; i < hash_set->hash_buckets; i++) { 292 | size += set_size(hash_set->entry[i]); 293 | } 294 | 295 | return size; 296 | } 297 | 298 | 299 | 300 | static int set_contains(intset_t *set, val_t val, rlu_thread_data_t *td) 301 | { 302 | int result; 303 | node_t *prev, *next; 304 | node_t *p_node; 305 | val_t v; 306 | 307 | # ifdef DEBUG 308 | printf("++> set_contains(%d)\n", val); 309 | IO_FLUSH; 310 | # endif 311 | 312 | if (td == NULL) { 313 | prev = set->head; 314 | next = prev->next; 315 | while (next->val < val) { 316 | prev = next; 317 | next = prev->next; 318 | } 319 | result = (next->val == val); 320 | } else { 321 | RCU_READ_LOCK(td); 322 | prev = (node_t *)RCU_DEREFERENCE(td, (set->head)); 323 | next = (node_t *)RCU_DEREFERENCE(td, (prev->next)); 324 | while (1) { 325 | p_node = RCU_DEREFERENCE(td, next); 326 | v = p_node->val; 327 | if (node_compute(next) >= 1000) { 328 | pr_err("%s: Invalid sum", __FUNCTION__); 329 | } 330 | if (v >= val) 331 | break; 332 | prev = next; 333 | next = (node_t *)RCU_DEREFERENCE(td, (prev->next)); 334 | } 335 | result = (v == val); 336 | } 337 | RCU_READ_UNLOCK(td); 338 | 339 | return result; 340 | } 341 | 342 | static int hash_set_contains(hash_intset_t *hash_set, val_t val, rlu_thread_data_t *td) 343 | { 344 | int hash = HASH_VALUE(hash_set, val); 345 | 346 | return set_contains(hash_set->entry[hash], val, td); 347 | } 348 | 349 | static int set_add(intset_t *set, val_t val, rlu_thread_data_t *td) 350 | { 351 | int result; 352 | node_t *prev, *next; 353 | node_t *p_node; 354 | val_t v; 355 | 356 | if (td == NULL) { 357 | prev = set->head; 358 | next = prev->next; 359 | while (next->val < val) { 360 | prev = next; 361 | next = prev->next; 362 | } 363 | result = (next->val != val); 364 | if (result) { 365 | prev->next = new_node(val, next); 366 | } 367 | } else { 368 | int count = 0; 369 | RCU_LOCK(td); 370 | prev = (node_t *)RCU_DEREFERENCE(td, (set->head)); 371 | next = (node_t *)RCU_DEREFERENCE(td, (prev->next)); 372 | while (1) { 373 | count++; 374 | p_node = RCU_DEREFERENCE(td, next); 375 | v = p_node->val; 376 | if (v >= val) 377 | break; 378 | prev = next; 379 | next = (node_t *)RCU_DEREFERENCE(td, (prev->next)); 380 | } 381 | result = (v != val); 382 | if (result) { 383 | #ifdef RUN_RLU 384 | node_t *p_new_node = rlu_new_node(); 385 | p_new_node->val = val; 386 | RLU_LOCK_REF(td, &prev); 387 | RCU_ASSIGN_POINTER(td, &(p_new_node->next), next); 388 | RCU_ASSIGN_POINTER(td, &(prev->next), p_new_node); 389 | #else 390 | RCU_ASSIGN_POINTER(td, (prev->next), (new_node(val, next))); 391 | #endif 392 | } 393 | RCU_UNLOCK(td); 394 | } 395 | 396 | return result; 397 | } 398 | 399 | static int hash_set_add(hash_intset_t *hash_set, val_t val, rlu_thread_data_t *td) 400 | { 401 | int hash = HASH_VALUE(hash_set, val); 402 | 403 | return set_add(hash_set->entry[hash], val, td); 404 | } 405 | 406 | static int set_remove(intset_t *set, val_t val, rlu_thread_data_t *td) 407 | { 408 | int result; 409 | node_t *prev, *next; 410 | node_t *p_node; 411 | val_t v; 412 | node_t *n; 413 | 414 | if (td == NULL) { 415 | prev = set->head; 416 | next = prev->next; 417 | while (next->val < val) { 418 | prev = next; 419 | next = prev->next; 420 | } 421 | result = (next->val == val); 422 | if (result) { 423 | prev->next = next->next; 424 | node_free(next, NULL); 425 | } 426 | } else { 427 | RCU_LOCK(td); 428 | prev = (node_t *)RCU_DEREFERENCE(td, (set->head)); 429 | next = (node_t *)RCU_DEREFERENCE(td, (prev->next)); 430 | while (1) { 431 | p_node = RCU_DEREFERENCE(td, next); 432 | v = p_node->val; 433 | if (v >= val) 434 | break; 435 | prev = next; 436 | next = (node_t *)RCU_DEREFERENCE(td, (prev->next)); 437 | } 438 | result = (v == val); 439 | if (result) { 440 | n = (node_t *)RCU_DEREFERENCE(td, (next->next)); 441 | #ifdef RUN_RLU 442 | RLU_LOCK_REF(td, &prev); 443 | RCU_ASSIGN_POINTER(td, &(prev->next), n); 444 | next = RLU_FORCE_REAL_REF(next); 445 | #else 446 | RCU_ASSIGN_POINTER(td, (prev->next), n); 447 | #endif 448 | RCU_UNLOCK(td); 449 | 450 | node_free(next, td); 451 | 452 | } else { 453 | RCU_UNLOCK(td); 454 | } 455 | } 456 | 457 | return result; 458 | } 459 | 460 | static int hash_set_remove(hash_intset_t *hash_set, val_t val, rlu_thread_data_t *td) 461 | { 462 | int hash = HASH_VALUE(hash_set, val); 463 | 464 | return set_remove(hash_set->entry[hash], val, td); 465 | } 466 | 467 | #ifdef RUN_RLU 468 | # define PREFIX(str) rlu##str 469 | #else 470 | # define PREFIX(str) rcu##str 471 | #endif 472 | 473 | 474 | /* Interface for driver */ 475 | int PREFIX(intset_init)(void) 476 | { 477 | int hash_buckets = DEFAULT_HASH_BUCKETS; 478 | intset_set = hash_set_new(hash_buckets); 479 | return 0; 480 | } 481 | 482 | int PREFIX(intset_lookup)(void *tl, int key) 483 | { 484 | rlu_thread_data_t *d = (rlu_thread_data_t *)tl; 485 | if (hash_set_contains(intset_set, key, d)) 486 | return 0; 487 | return -ENOENT; 488 | } 489 | 490 | int PREFIX(intset_delete)(void *tl, int key) 491 | { 492 | rlu_thread_data_t *d = (rlu_thread_data_t *)tl; 493 | if (hash_set_remove(intset_set, key, d)) 494 | return 0; 495 | return -ENOENT; 496 | } 497 | 498 | int PREFIX(intset_insert)(void *tl, int key) 499 | { 500 | rlu_thread_data_t *d = (rlu_thread_data_t *)tl; 501 | if (hash_set_add(intset_set, key, d)) 502 | return 0; 503 | return -EEXIST; 504 | } 505 | 506 | int PREFIX(intset_test)(void* tl) 507 | { 508 | rlu_thread_data_t *self = (rlu_thread_data_t *)tl; 509 | if (PREFIX(intset_insert)(self, 1010)) 510 | pr_warn("intset: Element 1010 cannot be inserted"); 511 | if (PREFIX(intset_lookup)(self, 1010)) 512 | pr_warn("intset: Element 1010 must be here"); 513 | if (!PREFIX(intset_lookup)(self, 1011)) 514 | pr_warn("intset: Element 1011 must not be here"); 515 | if (PREFIX(intset_insert)(self, 1011)) 516 | pr_warn("intset: Element 1011 cannot be inserted"); 517 | if (PREFIX(intset_insert)(self, 1012)) 518 | pr_warn("intset: Element 1012 cannot be inserted"); 519 | if (PREFIX(intset_lookup)(self, 1011)) 520 | pr_warn("intset: Element 1011 must be here"); 521 | if (PREFIX(intset_lookup)(self, 1012)) 522 | pr_warn("intset: Element 1012 must be here"); 523 | if (PREFIX(intset_lookup)(self, 1010)) 524 | pr_warn("intset: Element 1010 must be here"); 525 | if (PREFIX(intset_delete)(self, 1010)) 526 | pr_warn("intset: Removal of element 1010 failed"); 527 | if (!PREFIX(intset_lookup)(self, 1010)) 528 | pr_warn("intset: Element 1010 must not be here"); 529 | return 0; 530 | } 531 | 532 | -------------------------------------------------------------------------------- /kernel-bench/intset.h: -------------------------------------------------------------------------------- 1 | #ifndef _INTSET_H 2 | #define _INTSET_H 3 | 4 | int rluintset_init(void); 5 | int rluintset_lookup(void *tl, int key); 6 | int rluintset_delete(void *tl, int key); 7 | int rluintset_insert(void *tl, int key); 8 | int rluintset_test(void *arg); 9 | int rcuintset_init(void); 10 | int rcuintset_lookup(void *tl, int key); 11 | int rcuintset_delete(void *tl, int key); 12 | int rcuintset_insert(void *tl, int key); 13 | int rcuintset_test(void *arg); 14 | 15 | #endif /* _INTSET_H */ 16 | 17 | -------------------------------------------------------------------------------- /kernel-bench/rcu-hash-list.c: -------------------------------------------------------------------------------- 1 | #include // kmalloc 2 | #include 3 | #include 4 | 5 | 6 | #include "hash-list.h" 7 | 8 | 9 | ///////////////////////////////////////////////////////// 10 | // DEFINES 11 | ///////////////////////////////////////////////////////// 12 | #define HASH_VALUE(p_hash_list, val) (val % p_hash_list->n_buckets) 13 | 14 | #define RCU_READER_LOCK() rcu_read_lock() 15 | #define RCU_READER_UNLOCK() rcu_read_unlock() 16 | #define RCU_SYNCHRONIZE() synchronize_rcu() 17 | #define RCU_ASSIGN_PTR(p_ptr, p_obj) rcu_assign_pointer(p_ptr, p_obj) 18 | 19 | #define RCU_DEREF(p_obj) (p_obj) 20 | 21 | #define RCU_WRITER_LOCK(lock) spin_lock(&lock) 22 | #define RCU_WRITER_UNLOCK(lock) spin_unlock(&lock) 23 | #define RCU_FREE(ptr) kfree_rcu(ptr, rcu); 24 | 25 | ///////////////////////////////////////////////////////// 26 | // TYPES 27 | ///////////////////////////////////////////////////////// 28 | 29 | ///////////////////////////////////////////////////////// 30 | // GLOBALS 31 | ///////////////////////////////////////////////////////// 32 | // This global lock has been changed for a per-bucket lock 33 | //__cacheline_aligned static DEFINE_SPINLOCK(rcuspin); 34 | 35 | // FIXME Change function for no parameter and init func 36 | __cacheline_aligned static hash_list_t *g_hash_list; 37 | 38 | ///////////////////////////////////////////////////////// 39 | // FUNCTIONS 40 | ///////////////////////////////////////////////////////// 41 | 42 | ///////////////////////////////////////////////////////// 43 | // NEW NODE 44 | ///////////////////////////////////////////////////////// 45 | node_t *rcu_new_node(void) { 46 | 47 | node_t *p_new_node = (node_t *)kmalloc(sizeof(node_t), GFP_KERNEL); 48 | if (p_new_node == NULL){ 49 | pr_err("%s: out of memory\n", __FUNCTION__); 50 | } 51 | 52 | return p_new_node; 53 | } 54 | 55 | ///////////////////////////////////////////////////////// 56 | // FREE NODE 57 | ///////////////////////////////////////////////////////// 58 | 59 | void rcu_free_node(node_t *p_node) { 60 | RCU_FREE(p_node); 61 | } 62 | 63 | ///////////////////////////////////////////////////////// 64 | // NEW LIST 65 | ///////////////////////////////////////////////////////// 66 | list_t *rcu_new_list(void) 67 | { 68 | list_t *p_list; 69 | node_t *p_min_node, *p_max_node; 70 | 71 | p_list = (list_t *)kmalloc(sizeof(list_t), GFP_KERNEL); 72 | if (p_list == NULL) { 73 | pr_err("%s: out of memory\n", __FUNCTION__); 74 | } 75 | 76 | p_max_node = rcu_new_node(); 77 | p_max_node->val = LIST_VAL_MAX; 78 | p_max_node->p_next = NULL; 79 | 80 | p_min_node = rcu_new_node(); 81 | p_min_node->val = LIST_VAL_MIN; 82 | p_min_node->p_next = p_max_node; 83 | 84 | p_list->p_head = p_min_node; 85 | p_list->rcuspin = __SPIN_LOCK_UNLOCKED(p_list->rcuspin); 86 | 87 | return p_list; 88 | } 89 | 90 | ///////////////////////////////////////////////////////// 91 | // NEW HASH LIST 92 | ///////////////////////////////////////////////////////// 93 | hash_list_t *rcu_new_hash_list(int n_buckets) 94 | { 95 | int i; 96 | hash_list_t *p_hash_list; 97 | 98 | p_hash_list = (hash_list_t *)kmalloc(sizeof(hash_list_t), GFP_KERNEL); 99 | 100 | if (p_hash_list == NULL) { 101 | pr_err("%s: out of memory\n", __FUNCTION__); 102 | } 103 | 104 | p_hash_list->n_buckets = n_buckets; 105 | 106 | for (i = 0; i < p_hash_list->n_buckets; i++) { 107 | p_hash_list->buckets[i] = rcu_new_list(); 108 | } 109 | 110 | return p_hash_list; 111 | } 112 | 113 | int rcu_hash_list_init(void) 114 | { 115 | g_hash_list = rcu_new_hash_list(DEFAULT_BUCKETS); 116 | return 0; 117 | } 118 | 119 | ///////////////////////////////////////////////////////// 120 | // LIST SIZE 121 | ///////////////////////////////////////////////////////// 122 | static int list_size(list_t *p_list) 123 | { 124 | int size = 0; 125 | node_t *p_node; 126 | 127 | /* We have at least 2 elements */ 128 | p_node = p_list->p_head->p_next; 129 | while (p_node->p_next != NULL) { 130 | size++; 131 | p_node = p_node->p_next; 132 | } 133 | 134 | return size; 135 | } 136 | 137 | ///////////////////////////////////////////////////////// 138 | // HASH LIST SIZE 139 | ///////////////////////////////////////////////////////// 140 | static int hash_list_size(hash_list_t *p_hash_list) 141 | { 142 | int i; 143 | int size = 0; 144 | 145 | for (i = 0; i < p_hash_list->n_buckets; i++) { 146 | size += list_size(p_hash_list->buckets[i]); 147 | } 148 | 149 | return size; 150 | } 151 | 152 | ///////////////////////////////////////////////////////// 153 | // LIST CONTAINS 154 | ///////////////////////////////////////////////////////// 155 | int rcu_list_contains(list_t *p_list, val_t val) { 156 | int result; 157 | val_t v; 158 | node_t *p_prev, *p_next; 159 | node_t *p_node; 160 | 161 | RCU_READER_LOCK(); 162 | 163 | p_prev = (node_t *)RCU_DEREF(p_list->p_head); 164 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 165 | while (1) { 166 | p_node = (node_t *)RCU_DEREF(p_next); 167 | v = p_node->val; 168 | 169 | if (v >= val) { 170 | break; 171 | } 172 | 173 | p_prev = p_next; 174 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 175 | } 176 | 177 | result = (v == val); 178 | 179 | RCU_READER_UNLOCK(); 180 | 181 | return result; 182 | } 183 | 184 | ///////////////////////////////////////////////////////// 185 | // HASH LIST CONTAINS 186 | ///////////////////////////////////////////////////////// 187 | int rcu_hash_list_contains(void *tl, val_t val) 188 | { 189 | int hash = HASH_VALUE(g_hash_list, val); 190 | 191 | return rcu_list_contains(g_hash_list->buckets[hash], val) ? 0 : -ENOENT; 192 | } 193 | 194 | ///////////////////////////////////////////////////////// 195 | // LIST ADD 196 | ///////////////////////////////////////////////////////// 197 | int rcu_list_add(list_t *p_list, val_t val) { 198 | int result; 199 | node_t *p_prev, *p_next; 200 | node_t *p_node; 201 | val_t v; 202 | 203 | RCU_WRITER_LOCK(p_list->rcuspin); 204 | 205 | p_prev = (node_t *)RCU_DEREF(p_list->p_head); 206 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 207 | 208 | while (1) { 209 | p_node = (node_t *)RCU_DEREF(p_next); 210 | v = p_node->val; 211 | 212 | if (v >= val) { 213 | break; 214 | } 215 | 216 | p_prev = p_next; 217 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 218 | } 219 | 220 | result = (v != val); 221 | 222 | if (result) { 223 | node_t *p_new_node = rcu_new_node(); 224 | p_new_node->val = val; 225 | p_new_node->p_next = p_next; 226 | 227 | RCU_ASSIGN_PTR((p_prev->p_next), p_new_node); 228 | } 229 | 230 | RCU_WRITER_UNLOCK(p_list->rcuspin); 231 | 232 | return result; 233 | } 234 | 235 | 236 | ///////////////////////////////////////////////////////// 237 | // HASH LIST ADD 238 | ///////////////////////////////////////////////////////// 239 | int rcu_hash_list_add(void *tl, val_t val) 240 | { 241 | int hash = HASH_VALUE(g_hash_list, val); 242 | 243 | return rcu_list_add(g_hash_list->buckets[hash], val) ? 0 : -EEXIST; 244 | } 245 | 246 | ///////////////////////////////////////////////////////// 247 | // LIST REMOVE 248 | ///////////////////////////////////////////////////////// 249 | int rcu_list_remove(list_t *p_list, val_t val) { 250 | int result; 251 | node_t *p_prev, *p_next; 252 | node_t *p_node; 253 | node_t *n; 254 | 255 | RCU_WRITER_LOCK(p_list->rcuspin); 256 | 257 | p_prev = (node_t *)RCU_DEREF(p_list->p_head); 258 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 259 | while (1) { 260 | p_node = (node_t *)RCU_DEREF(p_next); 261 | 262 | if (p_node->val >= val) { 263 | break; 264 | } 265 | 266 | p_prev = p_next; 267 | p_next = (node_t *)RCU_DEREF(p_prev->p_next); 268 | } 269 | 270 | result = (p_node->val == val); 271 | 272 | if (result) { 273 | n = (node_t *)RCU_DEREF(p_next->p_next); 274 | 275 | RCU_ASSIGN_PTR((p_prev->p_next), n); 276 | 277 | RCU_WRITER_UNLOCK(p_list->rcuspin); 278 | 279 | rcu_free_node(p_next); 280 | 281 | return result; 282 | } 283 | 284 | RCU_WRITER_UNLOCK(p_list->rcuspin); 285 | 286 | return result; 287 | } 288 | 289 | ///////////////////////////////////////////////////////// 290 | // HASH LIST REMOVE 291 | ///////////////////////////////////////////////////////// 292 | int rcu_hash_list_remove(void *tl, val_t val) 293 | { 294 | int hash = HASH_VALUE(g_hash_list, val); 295 | 296 | return rcu_list_remove(g_hash_list->buckets[hash], val) ? 0 : -ENOENT; 297 | } 298 | -------------------------------------------------------------------------------- /kernel-bench/rcuintset.c: -------------------------------------------------------------------------------- 1 | intset.c -------------------------------------------------------------------------------- /kernel-bench/rlu-hash-list.c: -------------------------------------------------------------------------------- 1 | #include // kmalloc 2 | #include 3 | #include 4 | 5 | #include "rlu.h" 6 | #include "hash-list.h" 7 | 8 | ///////////////////////////////////////////////////////// 9 | // DEFINES 10 | ///////////////////////////////////////////////////////// 11 | #define HASH_VALUE(p_hash_list, val) (val % p_hash_list->n_buckets) 12 | 13 | ///////////////////////////////////////////////////////// 14 | // TYPES 15 | ///////////////////////////////////////////////////////// 16 | 17 | ///////////////////////////////////////////////////////// 18 | // GLOBALS 19 | ///////////////////////////////////////////////////////// 20 | __cacheline_aligned static hash_list_t *g_hash_list; 21 | 22 | ///////////////////////////////////////////////////////// 23 | // FUNCTIONS 24 | ///////////////////////////////////////////////////////// 25 | 26 | ///////////////////////////////////////////////////////// 27 | // NEW NODE 28 | ///////////////////////////////////////////////////////// 29 | node_t *rlu_new_node(void) { 30 | node_t *p_new_node = (node_t *)RLU_ALLOC(sizeof(node_t)); 31 | if (p_new_node == NULL){ 32 | pr_err("out of memory\n"); 33 | } 34 | 35 | return p_new_node; 36 | } 37 | 38 | ///////////////////////////////////////////////////////// 39 | // NEW LIST 40 | ///////////////////////////////////////////////////////// 41 | list_t *rlu_new_list(void) 42 | { 43 | list_t *p_list; 44 | node_t *p_min_node, *p_max_node; 45 | 46 | p_list = (list_t *)kmalloc(sizeof(list_t), GFP_KERNEL); 47 | if (p_list == NULL) { 48 | pr_err("malloc"); 49 | } 50 | 51 | p_max_node = rlu_new_node(); 52 | p_max_node->val = LIST_VAL_MAX; 53 | p_max_node->p_next = NULL; 54 | 55 | p_min_node = rlu_new_node(); 56 | p_min_node->val = LIST_VAL_MIN; 57 | p_min_node->p_next = p_max_node; 58 | 59 | p_list->p_head = p_min_node; 60 | 61 | return p_list; 62 | } 63 | 64 | ///////////////////////////////////////////////////////// 65 | // NEW HASH LIST 66 | ///////////////////////////////////////////////////////// 67 | hash_list_t *rlu_new_hash_list(int n_buckets) 68 | { 69 | int i; 70 | hash_list_t *p_hash_list; 71 | 72 | p_hash_list = (hash_list_t *)kmalloc(sizeof(hash_list_t), GFP_KERNEL); 73 | 74 | if (p_hash_list == NULL) { 75 | pr_err("malloc"); 76 | } 77 | 78 | p_hash_list->n_buckets = n_buckets; 79 | 80 | for (i = 0; i < p_hash_list->n_buckets; i++) { 81 | p_hash_list->buckets[i] = rlu_new_list(); 82 | } 83 | 84 | return p_hash_list; 85 | } 86 | 87 | int rlu_hash_list_init(void) 88 | { 89 | g_hash_list = rlu_new_hash_list(DEFAULT_BUCKETS); 90 | return 0; 91 | } 92 | 93 | ///////////////////////////////////////////////////////// 94 | // LIST SIZE 95 | ///////////////////////////////////////////////////////// 96 | static int list_size(list_t *p_list) 97 | { 98 | int size = 0; 99 | node_t *p_node; 100 | 101 | /* We have at least 2 elements */ 102 | p_node = p_list->p_head->p_next; 103 | while (p_node->p_next != NULL) { 104 | size++; 105 | p_node = p_node->p_next; 106 | } 107 | 108 | return size; 109 | } 110 | 111 | ///////////////////////////////////////////////////////// 112 | // HASH LIST SIZE 113 | ///////////////////////////////////////////////////////// 114 | static int hash_list_size(hash_list_t *p_hash_list) 115 | { 116 | int i; 117 | int size = 0; 118 | 119 | for (i = 0; i < p_hash_list->n_buckets; i++) { 120 | size += list_size(p_hash_list->buckets[i]); 121 | } 122 | 123 | return size; 124 | } 125 | 126 | ///////////////////////////////////////////////////////// 127 | // LIST CONTAINS 128 | ///////////////////////////////////////////////////////// 129 | int rlu_list_contains(rlu_thread_data_t *self, list_t *p_list, val_t val) { 130 | int result; 131 | val_t v; 132 | node_t *p_prev, *p_next; 133 | 134 | RLU_READER_LOCK(self); 135 | 136 | p_prev = (node_t *)RLU_DEREF(self, (p_list->p_head)); 137 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 138 | while (1) { 139 | //p_node = (node_t *)RLU_DEREF(self, p_next); 140 | v = p_next->val; 141 | 142 | if (v >= val) { 143 | break; 144 | } 145 | 146 | p_prev = p_next; 147 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 148 | } 149 | 150 | result = (v == val); 151 | 152 | RLU_READER_UNLOCK(self); 153 | 154 | return result; 155 | } 156 | 157 | ///////////////////////////////////////////////////////// 158 | // HASH LIST CONTAINS 159 | ///////////////////////////////////////////////////////// 160 | int rlu_hash_list_contains(void *tl, val_t val) 161 | { 162 | rlu_thread_data_t *self = (rlu_thread_data_t *)tl; 163 | int hash = HASH_VALUE(g_hash_list, val); 164 | 165 | int ret = rlu_list_contains(self, g_hash_list->buckets[hash], val); 166 | if (ret) 167 | return 0; 168 | return -ENOENT; 169 | } 170 | 171 | ///////////////////////////////////////////////////////// 172 | // LIST ADD 173 | ///////////////////////////////////////////////////////// 174 | int rlu_list_add(rlu_thread_data_t *self, list_t *p_list, val_t val) { 175 | int result; 176 | node_t *p_prev, *p_next; 177 | val_t v; 178 | 179 | restart: 180 | RLU_READER_LOCK(self); 181 | 182 | p_prev = (node_t *)RLU_DEREF(self, (p_list->p_head)); 183 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 184 | while (1) { 185 | //p_node = (node_t *)RLU_DEREF(self, p_next); 186 | v = p_next->val; 187 | 188 | if (v >= val) { 189 | break; 190 | } 191 | 192 | p_prev = p_next; 193 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 194 | } 195 | 196 | result = (v != val); 197 | 198 | if (result) { 199 | node_t *p_new_node; 200 | 201 | if (!RLU_TRY_LOCK(self, &p_prev)) { 202 | RLU_ABORT(self); 203 | goto restart; 204 | } 205 | 206 | if (!RLU_TRY_LOCK(self, &p_next)) { 207 | RLU_ABORT(self); 208 | goto restart; 209 | } 210 | 211 | p_new_node = rlu_new_node(); 212 | p_new_node->val = val; 213 | RLU_ASSIGN_PTR(self, &(p_new_node->p_next), p_next); 214 | 215 | RLU_ASSIGN_PTR(self, &(p_prev->p_next), p_new_node); 216 | } 217 | 218 | RLU_READER_UNLOCK(self); 219 | 220 | return result; 221 | } 222 | 223 | ///////////////////////////////////////////////////////// 224 | // HASH LIST ADD 225 | ///////////////////////////////////////////////////////// 226 | int rlu_hash_list_add(void *tl, val_t val) 227 | { 228 | rlu_thread_data_t *self = (rlu_thread_data_t *)tl; 229 | int hash = HASH_VALUE(g_hash_list, val); 230 | 231 | int ret = rlu_list_add(self, g_hash_list->buckets[hash], val); 232 | if (ret) 233 | return 0; 234 | return -EEXIST; 235 | } 236 | 237 | 238 | ///////////////////////////////////////////////////////// 239 | // LIST REMOVE 240 | ///////////////////////////////////////////////////////// 241 | int rlu_list_remove(rlu_thread_data_t *self, list_t *p_list, val_t val) { 242 | int result; 243 | node_t *p_prev, *p_next; 244 | node_t *n; 245 | val_t v; 246 | 247 | restart: 248 | RLU_READER_LOCK(self); 249 | 250 | p_prev = (node_t *)RLU_DEREF(self, (p_list->p_head)); 251 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 252 | while (1) { 253 | //p_node = (node_t *)RLU_DEREF(self, p_next); 254 | 255 | v = p_next->val; 256 | 257 | if (v >= val) { 258 | break; 259 | } 260 | 261 | p_prev = p_next; 262 | p_next = (node_t *)RLU_DEREF(self, (p_prev->p_next)); 263 | } 264 | 265 | result = (v == val); 266 | 267 | if (result) { 268 | n = (node_t *)RLU_DEREF(self, (p_next->p_next)); 269 | 270 | if (!RLU_TRY_LOCK(self, &p_prev)) { 271 | RLU_ABORT(self); 272 | goto restart; 273 | } 274 | 275 | if (!RLU_TRY_LOCK(self, &p_next)) { 276 | RLU_ABORT(self); 277 | goto restart; 278 | } 279 | 280 | RLU_ASSIGN_PTR(self, &(p_prev->p_next), n); 281 | 282 | RLU_FREE(self, p_next); 283 | 284 | RLU_READER_UNLOCK(self); 285 | 286 | return result; 287 | } 288 | 289 | RLU_READER_UNLOCK(self); 290 | 291 | return result; 292 | } 293 | 294 | ///////////////////////////////////////////////////////// 295 | // HASH LIST REMOVE 296 | ///////////////////////////////////////////////////////// 297 | int rlu_hash_list_remove(void *tl, val_t val) 298 | { 299 | rlu_thread_data_t *self = (rlu_thread_data_t *)tl; 300 | int hash = HASH_VALUE(g_hash_list, val); 301 | 302 | int ret = rlu_list_remove(self, g_hash_list->buckets[hash], val); 303 | if (ret) 304 | return 0; 305 | return -ENOENT; 306 | } 307 | 308 | -------------------------------------------------------------------------------- /kernel-bench/rlu.c: -------------------------------------------------------------------------------- 1 | ../rlu.c -------------------------------------------------------------------------------- /kernel-bench/rlu.h: -------------------------------------------------------------------------------- 1 | ../rlu.h -------------------------------------------------------------------------------- /kernel-bench/rluintset.c: -------------------------------------------------------------------------------- 1 | intset.c -------------------------------------------------------------------------------- /kernel-bench/sync_test.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include /* module_param */ 3 | #include /* pr_info/printk */ 4 | #include /* errors code */ 5 | #include /* current global variable */ 6 | //#include /* rcu */ 7 | #include /* __init __exit macros */ 8 | #include /* kthreads */ 9 | #include /* complete/wait_for_completion */ 10 | #include /* msleep */ 11 | #include /* kmalloc/kzalloc */ 12 | #include /* rnd_state/ */ 13 | #include /* get_cycles (same as rdtscll) */ 14 | 15 | #include "sync_test.h" 16 | #include "barrier.h" 17 | #include "rlu.h" 18 | #include "hash-list.h" 19 | #include "intset.h" 20 | 21 | #define MODULE_NAME "sync_test" 22 | #define MAX_BENCHMARKS (16) 23 | #ifndef RLU_DEFER_WS 24 | # define RLU_DEFER_WS (1) 25 | #endif 26 | #define FORCE_SCHED (1) /* Enable this define to allow long execution */ 27 | 28 | /* Bench configuration */ 29 | static char *benchmark = "rcuhashlist"; 30 | module_param(benchmark, charp, 0000); 31 | MODULE_PARM_DESC(benchmark, "Benchmark name"); 32 | static int threads_nb = 1; 33 | module_param(threads_nb, int, 0000); 34 | MODULE_PARM_DESC(threads_nb, "Number of threads"); 35 | static int duration = 100; 36 | module_param(duration, int, 0000); 37 | MODULE_PARM_DESC(duration, "Duration of the benchmark in ms"); 38 | static int update = 0; 39 | module_param(update, int, 0000); 40 | MODULE_PARM_DESC(update, "Probability for update operations. No floating-point in kernel so 10000 = 100%, 1 = 0.01%"); 41 | static int range = 1024; 42 | module_param(range, int, 0000); 43 | MODULE_PARM_DESC(range, "Key range. Initial set size is half the key range."); 44 | 45 | typedef struct benchmark { 46 | char name[32]; 47 | int (*init)(void); 48 | int (*lookup)(void *tl, int key); 49 | int (*insert)(void *tl, int key); 50 | int (*delete)(void *tl, int key); 51 | unsigned long nb_lookup; 52 | unsigned long nb_insert; 53 | unsigned long nb_delete; 54 | } benchmark_t; 55 | 56 | static benchmark_t benchmarks[MAX_BENCHMARKS] = { 57 | { 58 | .name = "rcuhashlist", 59 | .init = &rcu_hash_list_init, 60 | .lookup = &rcu_hash_list_contains, 61 | .insert = &rcu_hash_list_add, 62 | .delete = &rcu_hash_list_remove, 63 | }, 64 | { 65 | .name = "rluhashlist", 66 | .init = &rlu_hash_list_init, 67 | .lookup = &rlu_hash_list_contains, 68 | .insert = &rlu_hash_list_add, 69 | .delete = &rlu_hash_list_remove, 70 | }, 71 | /* 72 | { 73 | .name = "rluintset", 74 | .init = &rluintset_init, 75 | .lookup = &rluintset_lookup, 76 | .insert = &rluintset_insert, 77 | .delete = &rluintset_delete, 78 | },*/ 79 | { 80 | .name = "rcuintset", 81 | .init = &rcuintset_init, 82 | .lookup = &rcuintset_lookup, 83 | .insert = &rcuintset_insert, 84 | .delete = &rcuintset_delete, 85 | } 86 | }; 87 | 88 | typedef struct benchmark_thread { 89 | benchmark_t *benchmark; 90 | unsigned int id; 91 | rlu_thread_data_t *rlu; 92 | struct rnd_state rnd; 93 | struct { 94 | unsigned long nb_lookup; 95 | unsigned long nb_insert; 96 | unsigned long nb_delete; 97 | } ops; 98 | /* TODO to complete */ 99 | 100 | } benchmark_thread_t; 101 | 102 | benchmark_thread_t* benchmark_threads[RLU_MAX_THREADS]; 103 | 104 | struct completion sync_test_working; 105 | 106 | __cacheline_aligned static barrier_t sync_test_barrier; 107 | 108 | /* Generate a pseudo random in range [0:n[ */ 109 | static inline int rand_range(int n, struct rnd_state *seed) 110 | { 111 | return prandom_u32_state(seed) % n; 112 | } 113 | 114 | static int sync_test_thread(void* data) 115 | { 116 | benchmark_thread_t *bench = (benchmark_thread_t *)data; 117 | unsigned long duration_ms; 118 | struct timespec start, end; 119 | unsigned long long tsc_start, tsc_end; 120 | rlu_thread_data_t *self = bench->rlu; 121 | 122 | /* Wait on barrier */ 123 | barrier_cross(&sync_test_barrier); 124 | 125 | tsc_start = get_cycles(); 126 | start = current_kernel_time(); 127 | 128 | /* Thread main */ 129 | do { 130 | int op = rand_range(10000, &bench->rnd); 131 | int val = rand_range(range, &bench->rnd); 132 | if (op < update) { 133 | /* Update */ 134 | op = rand_range(2, &bench->rnd); 135 | if ((op & 1) == 0) { 136 | bench->benchmark->insert(self, val); 137 | bench->ops.nb_insert++; 138 | } else { 139 | bench->benchmark->delete(self, val); 140 | bench->ops.nb_delete++; 141 | } 142 | } else { 143 | /* Lookup */ 144 | bench->benchmark->lookup(self, val); 145 | bench->ops.nb_lookup++; 146 | } 147 | end = current_kernel_time(); 148 | duration_ms = (end.tv_sec * 1000 + end.tv_nsec / 1000000) - (start.tv_sec * 1000 + start.tv_nsec / 1000000); 149 | #ifdef FORCE_SCHED 150 | /* No need to force schedule(), time bound. */ 151 | cond_resched(); 152 | #endif /* FORCE_SCHED */ 153 | } while (duration_ms < duration); 154 | 155 | tsc_end = get_cycles(); 156 | pr_info(MODULE_NAME "(%i:%i) time: %lu ms (%llu cycles)\n", current->pid, bench->id, duration_ms, tsc_end - tsc_start); 157 | 158 | /* TODO probably better to do the ops/sec here. */ 159 | 160 | /* Thread finishing */ 161 | complete(&sync_test_working); 162 | rlu_thread_finish(bench->rlu); 163 | 164 | return 0; 165 | } 166 | 167 | static int __init sync_test_init(void) 168 | { 169 | benchmark_t *bench = NULL; 170 | int i; 171 | 172 | /* Benchmark to run */ 173 | for (i = 0; i < MAX_BENCHMARKS; i++) { 174 | if (benchmarks[i].name && !strcmp(benchmarks[i].name, benchmark)) { 175 | bench = &benchmarks[i]; 176 | break; 177 | } 178 | } 179 | if (!bench) { 180 | pr_err(MODULE_NAME ": Unknown benchmark %s\n", benchmark); 181 | return -EPERM; 182 | } 183 | if (bench->lookup == NULL || bench->insert == NULL || bench->delete == NULL) { 184 | pr_err(MODULE_NAME ": Benchmark %s has a NULL function defined\n", benchmark); 185 | return -EPERM; 186 | } 187 | /* TODO display all user parameters */ 188 | pr_notice(MODULE_NAME ": Running benchmark %s with %i threads\n", benchmark, threads_nb); 189 | 190 | if (threads_nb > num_online_cpus()) { 191 | pr_err(MODULE_NAME ": Invalid number of threads %d (MAX %d)\n", threads_nb, num_online_cpus()); 192 | return -EPERM; 193 | } 194 | 195 | /* Initialization */ 196 | init_completion(&sync_test_working); 197 | barrier_init(&sync_test_barrier, threads_nb); 198 | rlu_init(RLU_TYPE_FINE_GRAINED, RLU_DEFER_WS); 199 | bench->init(); 200 | for (i = 0; i < threads_nb; i++) { 201 | benchmark_threads[i] = kzalloc(sizeof(*benchmark_threads[i]), GFP_KERNEL); 202 | benchmark_threads[i]->id = i; 203 | prandom_seed_state(&benchmark_threads[i]->rnd, i + 1); 204 | benchmark_threads[i]->benchmark = bench; 205 | /* Other fields are set to 0 with kzalloc */ 206 | /* RLU Thread initialization */ 207 | benchmark_threads[i]->rlu = kmalloc(sizeof(rlu_thread_data_t), GFP_KERNEL); 208 | rlu_thread_init(benchmark_threads[i]->rlu); 209 | } 210 | 211 | /* Half fill the set */ 212 | for (i = 0; i < range / 2; i++) { 213 | /* Ensure the success of insertion */ 214 | while (bench->insert(benchmark_threads[0]->rlu, get_random_int() % range)); 215 | } 216 | 217 | /* Start N-1 threads */ 218 | for (i = 1; i < threads_nb; i++) { 219 | struct task_struct *t; 220 | /* kthread_run(...) can be also used to avoid wake_up_process */ 221 | t = kthread_create(sync_test_thread, benchmark_threads[i], "sync_test_thread"); 222 | if (t) { 223 | pr_notice(MODULE_NAME ": pid: %d (created from %d)\n", t->pid, current->pid); 224 | wake_up_process(t); 225 | /* kthread_bind(threads[cpu], cpu); */ 226 | } 227 | } 228 | 229 | /* Main thread is also doing work. */ 230 | sync_test_thread(benchmark_threads[0]); 231 | 232 | /* Wait for the threads to finish */ 233 | for (i = 0; i < threads_nb; i++) { 234 | pr_debug(MODULE_NAME ": Waiting still %d threads to finish\n", i); 235 | wait_for_completion(&sync_test_working); 236 | } 237 | 238 | /* Reinitialize one thread to cleanup */ 239 | rlu_thread_init(benchmark_threads[0]->rlu); 240 | 241 | /* Statistics output */ 242 | for (i = 0; i < threads_nb; i++) { 243 | bench->nb_lookup += benchmark_threads[i]->ops.nb_lookup; 244 | bench->nb_insert += benchmark_threads[i]->ops.nb_insert; 245 | bench->nb_delete += benchmark_threads[i]->ops.nb_delete; 246 | } 247 | pr_info(MODULE_NAME ": #lookup: %lu / s\n", bench->nb_lookup * 1000 / duration); 248 | pr_info(MODULE_NAME ": #insert: %lu / s\n", bench->nb_insert * 1000 / duration); 249 | pr_info(MODULE_NAME ": #delete: %lu / s\n", bench->nb_delete * 1000 / duration); 250 | 251 | /* Empty the set using all possible keys */ 252 | for (i = 0; i < range; i++) { 253 | bench->delete(benchmark_threads[0]->rlu, i); 254 | } 255 | 256 | /* Cleaning and free */ 257 | rlu_thread_finish(benchmark_threads[0]->rlu); 258 | for (i = 0; i < threads_nb; i++) { 259 | kfree(benchmark_threads[i]->rlu); 260 | } 261 | 262 | rlu_finish(); 263 | 264 | /* When the benchmark is done, the module is loaded. Maybe we can fail anyway to avoid empty unload. */ 265 | pr_info(MODULE_NAME ": Done\n"); 266 | return 0; 267 | } 268 | 269 | static void __exit sync_test_exit(void) 270 | { 271 | pr_info(MODULE_NAME ": Unloaded\n"); 272 | } 273 | 274 | module_init(sync_test_init); 275 | module_exit(sync_test_exit); 276 | 277 | MODULE_LICENSE("GPL"); 278 | -------------------------------------------------------------------------------- /kernel-bench/sync_test.h: -------------------------------------------------------------------------------- 1 | #ifndef _SYNC_TEST_H 2 | #define _SYNC_TEST_H 3 | 4 | #include /* complete/wait_for_completion */ 5 | 6 | extern struct completion sync_test_working; 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /new-urcu.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "new-urcu.h" 7 | 8 | /** 9 | * Copyright 2014 Maya Arbel (mayaarl [at] cs [dot] technion [dot] ac [dot] il). 10 | * 11 | * This file is part of Citrus. 12 | * 13 | * Citrus is free software: you can redistribute it and/or modify 14 | * it under the terms of the GNU General Public License as published by 15 | * the Free Software Foundation, either version 3 of the License, or 16 | * (at your option) any later version. 17 | * 18 | * This program is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 | * GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public License 24 | * along with this program. If not, see . 25 | * 26 | * Authors Maya Arbel and Adam Morrison 27 | */ 28 | 29 | int threads; 30 | rcu_node** urcu_table; 31 | 32 | #define MAX_SPIN_LOCKS (10000) 33 | /*typedef struct _my_lock { 34 | pthread_spinlock_t lock; 35 | //long padding[4]; 36 | } my_lock;*/ 37 | 38 | pthread_spinlock_t urcu_spin[MAX_SPIN_LOCKS]; 39 | 40 | void urcu_init(int num_threads){ 41 | rcu_node** result = (rcu_node**) malloc(sizeof(rcu_node)*num_threads); 42 | int i; 43 | rcu_node* new; 44 | threads = num_threads; 45 | for( i=0; itime = 1; 48 | new->f_size = 0; 49 | *(result + i) = new; 50 | } 51 | 52 | urcu_table = result; 53 | 54 | for (i = 0; i < MAX_SPIN_LOCKS; i++) { 55 | pthread_spin_init(&(urcu_spin[i]), PTHREAD_PROCESS_PRIVATE); 56 | } 57 | 58 | printf("initializing URCU finished, node_size: %zd\n", sizeof(rcu_node)); 59 | return; 60 | } 61 | 62 | __thread long* times = NULL; 63 | __thread int i; 64 | 65 | void urcu_register(int id){ 66 | times = (long*) malloc(sizeof(long)*threads); 67 | i = id; 68 | if (times == NULL ){ 69 | printf("malloc failed\n"); 70 | exit(1); 71 | } 72 | } 73 | void urcu_unregister(){ 74 | free(times); 75 | } 76 | 77 | void urcu_reader_lock(){ 78 | assert(urcu_table[i] != NULL); 79 | __sync_add_and_fetch(&urcu_table[i]->time, 1); 80 | } 81 | 82 | #if defined(__x86_64) 83 | static inline void set_bit(int nr, volatile unsigned long *addr){ 84 | asm("btsl %1,%0" : "+m" (*addr) : "Ir" (nr)); 85 | } 86 | #elif defined(__PPC64__) 87 | static __inline__ __attribute__((always_inline)) __attribute__((no_instrument_function)) 88 | void set_bits(unsigned long mask, volatile unsigned long *_p) 89 | { 90 | unsigned long old; 91 | unsigned long *p = (unsigned long *)_p; 92 | __asm__ __volatile__ ( "" "1:" ".long 0x7c0000a8 | (((%0) & 0x1f) << 21) | (((0) & 0x1f) << 16) | (((%3) & 0x1f) << 11) | (((0) & 0x1) << 0)" " " "\n" "or" " " "%0,%0,%2\n" "stdcx." " " "%0,0,%3\n" "bne- 1b\n" : "=&r" (old), "+m" (*p) : "r" (mask), "r" (p) : "cc", "memory"); 93 | } 94 | 95 | static __inline__ __attribute__((always_inline)) __attribute__((no_instrument_function)) 96 | void set_bit(int nr, volatile unsigned long *addr) 97 | { 98 | set_bits((1UL << ((nr) % 64)), addr + ((nr) / 64)); 99 | } 100 | #endif 101 | 102 | void urcu_reader_unlock(){ 103 | assert(urcu_table[i]!= NULL); 104 | set_bit(0, (unsigned long *)&urcu_table[i]->time); 105 | } 106 | 107 | void urcu_writer_lock(int lock_id){ 108 | pthread_spin_lock(&(urcu_spin[lock_id])); 109 | } 110 | 111 | void urcu_writer_unlock(int lock_id){ 112 | pthread_spin_unlock(&(urcu_spin[lock_id])); 113 | } 114 | 115 | void urcu_synchronize(){ 116 | int i; 117 | //read old counters 118 | for( i=0; itime; 120 | } 121 | for( i=0; itime; 125 | if (t & 1 || t > times[i]){ 126 | break; 127 | } 128 | } 129 | } 130 | } 131 | 132 | void urcu_free(void *ptr) { 133 | int k; 134 | 135 | urcu_table[i]->free_ptrs[urcu_table[i]->f_size] = ptr; 136 | urcu_table[i]->f_size++; 137 | 138 | if (urcu_table[i]->f_size == URCU_MAX_FREE_PTRS) { 139 | 140 | urcu_synchronize(); 141 | 142 | for (k = 0; k < urcu_table[i]->f_size; k++) { 143 | free(urcu_table[i]->free_ptrs[k]); 144 | } 145 | 146 | urcu_table[i]->f_size = 0; 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /new-urcu.h: -------------------------------------------------------------------------------- 1 | #ifndef _NEW_URCU_H_ 2 | #define _NEW_URCU_H_ 3 | 4 | /** 5 | * Copyright 2014 Maya Arbel (mayaarl [at] cs [dot] technion [dot] ac [dot] il). 6 | * 7 | * This file is part of Citrus. 8 | * 9 | * Citrus is free software: you can redistribute it and/or modify 10 | * it under the terms of the GNU General Public License as published by 11 | * the Free Software Foundation, either version 3 of the License, or 12 | * (at your option) any later version. 13 | * 14 | * This program is distributed in the hope that it will be useful, 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | * GNU General Public License for more details. 18 | * 19 | * You should have received a copy of the GNU General Public License 20 | * along with this program. If not, see . 21 | * 22 | * Authors Maya Arbel and Adam Morrison 23 | */ 24 | 25 | #define URCU_MAX_FREE_PTRS (1000) 26 | 27 | #if !defined(EXTERNAL_RCU) 28 | 29 | typedef struct rcu_node_t { 30 | volatile long time; 31 | int f_size; 32 | void *free_ptrs[URCU_MAX_FREE_PTRS]; 33 | char p[184]; 34 | } rcu_node; 35 | 36 | void urcu_init(int num_threads); 37 | void urcu_reader_lock(); 38 | void urcu_reader_unlock(); 39 | void urcu_writer_lock(int lock_id); 40 | void urcu_writer_unlock(int lock_id); 41 | void urcu_synchronize(); 42 | void urcu_register(int id); 43 | void urcu_unregister(); 44 | void urcu_free(void *ptr); 45 | 46 | #else 47 | 48 | #include 49 | 50 | static inline void initURCU(int num_threads) 51 | { 52 | rcu_init(); 53 | } 54 | 55 | static inline void urcu_register(int id) 56 | { 57 | rcu_register_thread(); 58 | } 59 | 60 | static inline void urcu_unregister() 61 | { 62 | rcu_unregister_thread(); 63 | } 64 | 65 | static inline void urcu_read_lock() 66 | { 67 | rcu_read_lock(); 68 | } 69 | 70 | static inline void urcu_read_unlock() 71 | { 72 | rcu_read_unlock(); 73 | } 74 | 75 | static inline void urcu_synchronize() 76 | { 77 | synchronize_rcu(); 78 | } 79 | 80 | #endif /* EXTERNAL RCU */ 81 | 82 | ////////// 83 | 84 | #define RCU_PRINT_STATS() 85 | #define RCU_INIT(n_threads) urcu_init(n_threads) 86 | #define RCU_THREAD_INIT(th_id) urcu_register(th_id) 87 | #define RCU_THREAD_FINISH() /* */ 88 | #define RCU_READER_LOCK() urcu_reader_lock() 89 | #define RCU_READER_UNLOCK() urcu_reader_unlock() 90 | #define RCU_WRITER_LOCK(lock_id) urcu_writer_lock(lock_id) 91 | #define RCU_WRITER_UNLOCK(lock_id) urcu_writer_unlock(lock_id) 92 | #define RCU_SYNCHRONIZE() urcu_synchronize() 93 | #define RCU_FREE(p_obj) urcu_free(p_obj) 94 | 95 | #define RCU_ASSIGN_PTR(p_ptr, p_obj) (*p_ptr) = p_obj 96 | #define RCU_DEREF(p_obj) (p_obj) 97 | 98 | 99 | ////////// 100 | 101 | #endif /* _NEW_URCU_H_ */ -------------------------------------------------------------------------------- /rlu.c: -------------------------------------------------------------------------------- 1 | ///////////////////////////////////////////////////////////////////////////////////////// 2 | // 3 | // 4 | // 5 | ///////////////////////////////////////////////////////////////////////////////////////// 6 | 7 | ///////////////////////////////////////////////////////////////////////////////////////// 8 | // INCLUDES 9 | ///////////////////////////////////////////////////////////////////////////////////////// 10 | 11 | #ifndef KERNEL 12 | # include 13 | # include 14 | # include 15 | # define likely(x) __builtin_expect ((x), 1) 16 | # define unlikely(x) __builtin_expect ((x), 0) 17 | #else /* KERNEL */ 18 | # include 19 | # include 20 | # include 21 | # include 22 | # define printf(...) pr_info(__VA_ARGS__) 23 | /* TODO */ 24 | # define printf_err(...) pr_err(__VA_ARGS__) 25 | # define fprintf(arg, ...) pr_err(__VA_ARGS__) 26 | # define free(ptr) kfree(ptr) 27 | # define malloc(size) kmalloc(size, GFP_KERNEL) 28 | #endif /* KERNEL */ 29 | 30 | #include "rlu.h" 31 | 32 | ///////////////////////////////////////////////////////////////////////////////////////// 33 | // DEFINES - GENERAL 34 | ///////////////////////////////////////////////////////////////////////////////////////// 35 | 36 | #define MAX_VERSION (LONG_MAX-1) 37 | 38 | #define LOCK_ID(th_id) (th_id + 1) 39 | 40 | #define WS_INDEX(ws_counter) ((ws_counter) % RLU_MAX_WRITE_SETS) 41 | 42 | #define ALIGN_NUMBER (8) 43 | #define ALIGN_MASK (ALIGN_NUMBER-1) 44 | #define PERFORM_ALIGNMENT(obj_size) (obj_size + (ALIGN_NUMBER - (obj_size & ALIGN_MASK))) 45 | #define ALIGN_OBJ_SIZE(obj_size) ((obj_size & ALIGN_MASK) ? PERFORM_ALIGNMENT(obj_size) : obj_size) 46 | 47 | #define MOVE_PTR_FORWARD(p_obj, offset) ((intptr_t *)(((volatile unsigned char *)p_obj) + offset)) 48 | #define MOVE_PTR_BACK(p_obj, offset) ((intptr_t *)(((volatile unsigned char *)p_obj) - offset)) 49 | 50 | #define OBJ_HEADER_SIZE (sizeof(rlu_obj_header_t)) 51 | #define WS_OBJ_HEADER_SIZE (sizeof(rlu_ws_obj_header_t)) 52 | 53 | #define OBJ_TO_H(p_obj) ((volatile rlu_obj_header_t *)MOVE_PTR_BACK(p_obj, OBJ_HEADER_SIZE)) 54 | #define H_TO_OBJ(p_h_obj) ((volatile intptr_t *)MOVE_PTR_FORWARD(p_h_obj, OBJ_HEADER_SIZE)) 55 | #define OBJ_COPY_TO_WS_H(p_obj_copy) ((volatile rlu_ws_obj_header_t *)MOVE_PTR_BACK(p_obj_copy, (OBJ_HEADER_SIZE+WS_OBJ_HEADER_SIZE))) 56 | 57 | #define PTR_ID_OBJ_COPY ((intptr_t *)0x12341234) 58 | 59 | #define GET_COPY(p_obj) (OBJ_TO_H(p_obj)->p_obj_copy) 60 | 61 | #define PTR_IS_LOCKED(p_obj_copy) (p_obj_copy != NULL) 62 | #define PTR_IS_COPY(p_obj_copy) (p_obj_copy == PTR_ID_OBJ_COPY) 63 | 64 | #define PTR_GET_WS_HEADER(p_obj_copy) (OBJ_COPY_TO_WS_H(p_obj_copy)) 65 | 66 | #define WS_GET_THREAD_ID(p_ws_obj_header) (p_ws_obj_header->thread_id) 67 | #define WS_GET_RUN_COUNTER(p_ws_obj_header) (p_ws_obj_header->run_counter) 68 | 69 | #define IS_UNLOCKED(p_obj) (!PTR_IS_LOCKED(GET_COPY(p_obj))) 70 | #define IS_COPY(p_obj) (PTR_IS_COPY(GET_COPY(p_obj))) 71 | #define GET_THREAD_ID(p_obj) (WS_GET_THREAD_ID(PTR_GET_WS_HEADER(GET_COPY(p_obj)))) 72 | 73 | #define GET_ACTUAL(p_obj_copy) (PTR_GET_WS_HEADER(p_obj_copy)->p_obj_actual) 74 | #define FORCE_ACTUAL(p_obj) (IS_COPY(p_obj) ? GET_ACTUAL(p_obj) : p_obj) 75 | 76 | #define TRY_CAS_PTR_OBJ_COPY(p_obj, new_ptr_obj_copy) (CAS((volatile int64_t *)&(OBJ_TO_H(p_obj)->p_obj_copy), 0, (int64_t)new_ptr_obj_copy) == 0) 77 | #define UNLOCK(p_obj) OBJ_TO_H(p_obj)->p_obj_copy = NULL 78 | 79 | #ifdef __x86_64 80 | # define RLU_CACHE_LINE_SIZE (64) 81 | #elif defined(__PPC64__) 82 | # define RLU_CACHE_LINE_SIZE (128) 83 | #endif 84 | 85 | ///////////////////////////////////////////////////////////////////////////////////////// 86 | // DEFINES - ATOMICS 87 | ///////////////////////////////////////////////////////////////////////////////////////// 88 | /* TODO for PPC */ 89 | #define CPU_RELAX() asm volatile("pause\n": : :"memory"); 90 | #define MEMBARSTLD() __sync_synchronize() 91 | #define FETCH_AND_ADD(addr, v) __sync_fetch_and_add((addr), (v)) 92 | #define CAS(addr, expected_value, new_value) __sync_val_compare_and_swap((addr), (expected_value), (new_value)) 93 | 94 | 95 | ///////////////////////////////////////////////////////////////////////////////////////// 96 | // DEFINES - ASSERT AND DEBUG 97 | ///////////////////////////////////////////////////////////////////////////////////////// 98 | #define RLU_TRACE_GLOBAL(fmt, ...) \ 99 | fprintf(stderr, "%s:%d:%s(): " fmt, \ 100 | __FILE__, __LINE__, __func__, \ 101 | __VA_ARGS__); 102 | 103 | #define RLU_TRACE(self, fmt, ...) \ 104 | fprintf(stderr, "[%ld][%ld][%ld]:%s:%d:%s(): " fmt, \ 105 | self->uniq_id, self->run_counter, self->local_version, \ 106 | __FILE__, __LINE__, __func__, \ 107 | __VA_ARGS__); 108 | 109 | #define RLU_ASSERT(cond) \ 110 | if (unlikely(!(cond))) { \ 111 | printf ("\n-----------------------------------------------\n"); \ 112 | printf ("\nAssertion failure: %s:%d '%s'\n", __FILE__, __LINE__, #cond); \ 113 | abort(); \ 114 | } 115 | 116 | #define RLU_ASSERT_MSG(cond, self, fmt, ...) \ 117 | if (unlikely(!(cond))) { \ 118 | printf ("\n-----------------------------------------------\n"); \ 119 | printf ("\nAssertion failure: %s:%d '%s'\n", __FILE__, __LINE__, #cond); \ 120 | RLU_TRACE(self, fmt, __VA_ARGS__); \ 121 | abort(); \ 122 | } 123 | 124 | #ifdef RLU_ENABLE_TRACE_1 125 | #define TRACE_1(self, fmt, ...) RLU_TRACE(self, fmt, __VA_ARGS__) 126 | #else 127 | #define TRACE_1(self, fmt, ...) 128 | #endif 129 | #ifdef RLU_ENABLE_TRACE_2 130 | #define TRACE_2(self, fmt, ...) RLU_TRACE(self, fmt, __VA_ARGS__) 131 | #else 132 | #define TRACE_2(self, fmt, ...) 133 | #endif 134 | #ifdef RLU_ENABLE_TRACE_3 135 | #define TRACE_3(self, fmt, ...) RLU_TRACE(self, fmt, __VA_ARGS__) 136 | #define TRACE_3_GLOBAL(fmt, ...) RLU_TRACE_GLOBAL(fmt, __VA_ARGS__) 137 | #else 138 | #define TRACE_3(self, fmt, ...) 139 | #define TRACE_3_GLOBAL(fmt, ...) 140 | #endif 141 | 142 | #define Q_ITERS_LIMIT (100000000) 143 | 144 | ///////////////////////////////////////////////////////////////////////////////////////// 145 | // TYPES 146 | ///////////////////////////////////////////////////////////////////////////////////////// 147 | typedef struct rlu_data { 148 | volatile long n_starts; 149 | volatile long n_finish; 150 | volatile long n_writers; 151 | 152 | volatile long n_writer_writeback; 153 | volatile long n_writeback_q_iters; 154 | volatile long n_pure_readers; 155 | volatile long n_steals; 156 | volatile long n_aborts; 157 | volatile long n_sync_requests; 158 | volatile long n_sync_and_writeback; 159 | 160 | } rlu_data_t; 161 | 162 | ///////////////////////////////////////////////////////////////////////////////////////// 163 | // GLOBALS 164 | ///////////////////////////////////////////////////////////////////////////////////////// 165 | static volatile rlu_data_t g_rlu_data = {0,}; 166 | 167 | static volatile int g_rlu_type = 0; 168 | static volatile int g_rlu_max_write_sets = 0; 169 | 170 | static volatile long g_rlu_cur_threads = 0; 171 | static volatile rlu_thread_data_t *g_rlu_threads[RLU_MAX_THREADS] = {0,}; 172 | 173 | static volatile long g_rlu_writer_locks[RLU_MAX_WRITER_LOCKS] = {0,}; 174 | 175 | static volatile long g_rlu_array[RLU_CACHE_LINE_SIZE * 64] = {0,}; 176 | 177 | #define g_rlu_writer_version g_rlu_array[RLU_CACHE_LINE_SIZE * 2] 178 | #define g_rlu_commit_version g_rlu_array[RLU_CACHE_LINE_SIZE * 4] 179 | 180 | ///////////////////////////////////////////////////////// 181 | // HELPER FUNCTIONS 182 | ///////////////////////////////////////////////////////// 183 | #ifdef KERNEL 184 | static void abort(void) 185 | { 186 | BUG(); 187 | /* if that doesn't kill us, halt */ 188 | panic("Oops failed to kill thread"); 189 | } 190 | #endif /* KERNEL */ 191 | 192 | ///////////////////////////////////////////////////////////////////////////////////////// 193 | // INTERNAL FUNCTIONS 194 | ///////////////////////////////////////////////////////////////////////////////////////// 195 | 196 | ///////////////////////////////////////////////////////////////////////////////////////// 197 | // Writer locks 198 | ///////////////////////////////////////////////////////////////////////////////////////// 199 | static void rlu_reset_writer_locks(rlu_thread_data_t *self, long ws_id) { 200 | self->obj_write_set[ws_id].writer_locks.size = 0; 201 | } 202 | 203 | static void rlu_add_writer_lock(rlu_thread_data_t *self, long writer_lock_id) { 204 | int i; 205 | long n_locks; 206 | 207 | n_locks = self->obj_write_set[self->ws_cur_id].writer_locks.size; 208 | for (i = 0; i < n_locks; i++) { 209 | RLU_ASSERT(self->obj_write_set[self->ws_cur_id].writer_locks.ids[i] != writer_lock_id); 210 | } 211 | 212 | self->obj_write_set[self->ws_cur_id].writer_locks.ids[n_locks] = writer_lock_id; 213 | self->obj_write_set[self->ws_cur_id].writer_locks.size++; 214 | 215 | RLU_ASSERT(self->obj_write_set[self->ws_cur_id].writer_locks.size < RLU_MAX_NESTED_WRITER_LOCKS); 216 | } 217 | 218 | static int rlu_try_acquire_writer_lock(rlu_thread_data_t *self, long writer_lock_id) { 219 | volatile long cur_lock; 220 | 221 | cur_lock = g_rlu_writer_locks[writer_lock_id]; 222 | if (cur_lock == 0) { 223 | if (CAS(&g_rlu_writer_locks[writer_lock_id], 0, LOCK_ID(self->uniq_id)) == 0) { 224 | return 1; 225 | } 226 | } 227 | 228 | return 0; 229 | } 230 | 231 | static void rlu_release_writer_lock(rlu_thread_data_t *self, long writer_lock_id) { 232 | RLU_ASSERT(g_rlu_writer_locks[writer_lock_id] == LOCK_ID(self->uniq_id)); 233 | 234 | g_rlu_writer_locks[writer_lock_id] = 0; 235 | } 236 | 237 | static void rlu_release_writer_locks(rlu_thread_data_t *self, int ws_id) { 238 | int i; 239 | 240 | for (i = 0; i < self->obj_write_set[ws_id].writer_locks.size; i++) { 241 | rlu_release_writer_lock(self, self->obj_write_set[ws_id].writer_locks.ids[i]); 242 | } 243 | } 244 | 245 | ///////////////////////////////////////////////////////////////////////////////////////// 246 | // Write-set processing 247 | ///////////////////////////////////////////////////////////////////////////////////////// 248 | static void rlu_reset(rlu_thread_data_t *self) { 249 | self->is_write_detected = 0; 250 | self->is_steal = 1; 251 | self->is_check_locks = 1; 252 | 253 | } 254 | 255 | static void rlu_reset_write_set(rlu_thread_data_t *self, long ws_counter) { 256 | long ws_id = WS_INDEX(ws_counter); 257 | 258 | self->obj_write_set[ws_id].num_of_objs = 0; 259 | self->obj_write_set[ws_id].p_cur = (intptr_t *)&(self->obj_write_set[ws_id].buffer[0]); 260 | 261 | rlu_reset_writer_locks(self, ws_id); 262 | } 263 | 264 | static intptr_t *rlu_add_ws_obj_header_to_write_set(rlu_thread_data_t *self, intptr_t *p_obj, obj_size_t obj_size) { 265 | intptr_t *p_cur; 266 | rlu_ws_obj_header_t *p_ws_obj_h; 267 | rlu_obj_header_t *p_obj_h; 268 | 269 | p_cur = (intptr_t *)self->obj_write_set[self->ws_cur_id].p_cur; 270 | 271 | p_ws_obj_h = (rlu_ws_obj_header_t *)p_cur; 272 | 273 | p_ws_obj_h->p_obj_actual = p_obj; 274 | p_ws_obj_h->obj_size = obj_size; 275 | p_ws_obj_h->run_counter = self->run_counter; 276 | p_ws_obj_h->thread_id = self->uniq_id; 277 | 278 | p_cur = MOVE_PTR_FORWARD(p_cur, WS_OBJ_HEADER_SIZE); 279 | 280 | p_obj_h = (rlu_obj_header_t *)p_cur; 281 | p_obj_h->p_obj_copy = PTR_ID_OBJ_COPY; 282 | 283 | p_cur = MOVE_PTR_FORWARD(p_cur, OBJ_HEADER_SIZE); 284 | 285 | self->obj_write_set[self->ws_cur_id].p_cur = p_cur; 286 | 287 | return p_cur; 288 | 289 | } 290 | 291 | static void rlu_add_obj_copy_to_write_set(rlu_thread_data_t *self, intptr_t *p_obj, obj_size_t obj_size) { 292 | intptr_t *p_cur; 293 | long cur_ws_size; 294 | 295 | p_cur = (intptr_t *)self->obj_write_set[self->ws_cur_id].p_cur; 296 | 297 | memcpy((unsigned char *)p_cur, (unsigned char *)p_obj, obj_size); 298 | 299 | p_cur = MOVE_PTR_FORWARD(p_cur, ALIGN_OBJ_SIZE(obj_size)); 300 | 301 | self->obj_write_set[self->ws_cur_id].p_cur = p_cur; 302 | self->obj_write_set[self->ws_cur_id].num_of_objs++; 303 | 304 | cur_ws_size = (long)p_cur - (long)self->obj_write_set[self->ws_cur_id].buffer; 305 | RLU_ASSERT(cur_ws_size < RLU_MAX_WRITE_SET_BUFFER_SIZE); 306 | } 307 | 308 | static void rlu_writeback_write_set(rlu_thread_data_t *self, long ws_counter) { 309 | unsigned int i; 310 | long ws_id; 311 | obj_size_t obj_size; 312 | intptr_t *p_cur; 313 | intptr_t *p_obj_copy; 314 | intptr_t *p_obj_actual; 315 | rlu_ws_obj_header_t *p_ws_obj_h; 316 | rlu_obj_header_t *p_obj_h; 317 | 318 | ws_id = WS_INDEX(ws_counter); 319 | 320 | p_cur = (intptr_t *)&(self->obj_write_set[ws_id].buffer[0]); 321 | 322 | for (i = 0; i < self->obj_write_set[ws_id].num_of_objs; i++) { 323 | 324 | p_ws_obj_h = (rlu_ws_obj_header_t *)p_cur; 325 | 326 | p_obj_actual = (intptr_t *)p_ws_obj_h->p_obj_actual; 327 | obj_size = (obj_size_t)p_ws_obj_h->obj_size; 328 | 329 | p_cur = MOVE_PTR_FORWARD(p_cur, WS_OBJ_HEADER_SIZE); 330 | p_obj_h = (rlu_obj_header_t *)p_cur; 331 | 332 | RLU_ASSERT(p_obj_h->p_obj_copy == PTR_ID_OBJ_COPY); 333 | 334 | p_cur = MOVE_PTR_FORWARD(p_cur, OBJ_HEADER_SIZE); 335 | 336 | p_obj_copy = (intptr_t *)p_cur; 337 | 338 | TRACE_2(self, "[%ld] rlu_writeback_and_unlock: copy [%p] <- [%p] [%zu]\n", 339 | self->writer_version, p_obj_actual, p_obj_copy, obj_size); 340 | 341 | memcpy((unsigned char *)p_obj_actual, (unsigned char *)p_obj_copy, obj_size); 342 | 343 | p_cur = MOVE_PTR_FORWARD(p_cur, ALIGN_OBJ_SIZE(obj_size)); 344 | 345 | RLU_ASSERT_MSG(GET_THREAD_ID(p_obj_actual) == self->uniq_id, 346 | self, "th_id = %ld my_id = %ld\n p_obj_actual = %p num_of_objs = %u\n", 347 | GET_THREAD_ID(p_obj_actual), self->uniq_id, p_obj_actual, self->obj_write_set[ws_id].num_of_objs); 348 | 349 | UNLOCK(p_obj_actual); 350 | 351 | } 352 | 353 | RLU_ASSERT(p_cur == self->obj_write_set[ws_id].p_cur); 354 | } 355 | 356 | static int rlu_writeback_write_sets_and_unlock(rlu_thread_data_t *self) { 357 | long ws_wb_num; 358 | long ws_counter; 359 | 360 | for (ws_counter = self->ws_head_counter; ws_counter < self->ws_wb_counter; ws_counter++) { 361 | rlu_reset_write_set(self, ws_counter); 362 | } 363 | 364 | self->ws_head_counter = self->ws_wb_counter; 365 | 366 | ws_wb_num = 0; 367 | for (ws_counter = self->ws_wb_counter; ws_counter < self->ws_tail_counter; ws_counter++) { 368 | rlu_writeback_write_set(self, ws_counter); 369 | ws_wb_num++; 370 | } 371 | 372 | self->ws_wb_counter = self->ws_tail_counter; 373 | 374 | return ws_wb_num; 375 | 376 | } 377 | 378 | static void rlu_unlock_objs(rlu_thread_data_t *self, int ws_counter) { 379 | unsigned int i; 380 | long ws_id; 381 | obj_size_t obj_size; 382 | intptr_t *p_cur; 383 | intptr_t *p_obj_actual; 384 | rlu_ws_obj_header_t *p_ws_obj_h; 385 | rlu_obj_header_t *p_obj_h; 386 | 387 | ws_id = WS_INDEX(ws_counter); 388 | 389 | p_cur = (intptr_t *)&(self->obj_write_set[ws_id].buffer[0]); 390 | 391 | for (i = 0; i < self->obj_write_set[ws_id].num_of_objs; i++) { 392 | 393 | p_ws_obj_h = (rlu_ws_obj_header_t *)p_cur; 394 | 395 | p_obj_actual = (intptr_t *)p_ws_obj_h->p_obj_actual; 396 | obj_size = p_ws_obj_h->obj_size; 397 | 398 | p_cur = MOVE_PTR_FORWARD(p_cur, WS_OBJ_HEADER_SIZE); 399 | p_obj_h = (rlu_obj_header_t *)p_cur; 400 | 401 | RLU_ASSERT(p_obj_h->p_obj_copy == PTR_ID_OBJ_COPY); 402 | 403 | p_cur = MOVE_PTR_FORWARD(p_cur, OBJ_HEADER_SIZE); 404 | 405 | RLU_ASSERT(GET_COPY(p_obj_actual) == p_cur); 406 | 407 | p_cur = MOVE_PTR_FORWARD(p_cur, ALIGN_OBJ_SIZE(obj_size)); 408 | 409 | UNLOCK(p_obj_actual); 410 | } 411 | } 412 | 413 | ///////////////////////////////////////////////////////////////////////////////////////// 414 | // Thread asserts 415 | ///////////////////////////////////////////////////////////////////////////////////////// 416 | static void rlu_assert_in_section(rlu_thread_data_t *self) { 417 | RLU_ASSERT(self->run_counter & 0x1); 418 | } 419 | 420 | ///////////////////////////////////////////////////////////////////////////////////////// 421 | // Thread register and unregister 422 | ///////////////////////////////////////////////////////////////////////////////////////// 423 | static void rlu_register_thread(rlu_thread_data_t *self) { 424 | RLU_ASSERT((self->run_counter & 0x1) == 0); 425 | 426 | FETCH_AND_ADD(&self->run_counter, 1); 427 | 428 | self->local_version = g_rlu_writer_version; 429 | self->local_commit_version = g_rlu_commit_version; 430 | } 431 | 432 | static void rlu_unregister_thread(rlu_thread_data_t *self) { 433 | RLU_ASSERT((self->run_counter & 0x1) != 0); 434 | 435 | self->run_counter++; 436 | } 437 | 438 | ///////////////////////////////////////////////////////////////////////////////////////// 439 | // Free buffer processing 440 | ///////////////////////////////////////////////////////////////////////////////////////// 441 | static void rlu_process_free(rlu_thread_data_t *self) { 442 | int i; 443 | intptr_t *p_obj; 444 | 445 | TRACE_3(self, "start free process free_nodes_size = %ld.\n", self->free_nodes_size); 446 | 447 | for (i = 0; i < self->free_nodes_size; i++) { 448 | p_obj = self->free_nodes[i]; 449 | 450 | RLU_ASSERT_MSG(IS_UNLOCKED(p_obj), 451 | self, "object is locked. p_obj = %p th_id = %ld\n", 452 | p_obj, GET_THREAD_ID(p_obj)); 453 | 454 | TRACE_3(self, "freeing: p_obj = %p, p_actual = %p\n", 455 | p_obj, (intptr_t *)OBJ_TO_H(p_obj)); 456 | 457 | free((intptr_t *)OBJ_TO_H(p_obj)); 458 | } 459 | 460 | self->free_nodes_size = 0; 461 | 462 | } 463 | 464 | ///////////////////////////////////////////////////////////////////////////////////////// 465 | // Sync 466 | ///////////////////////////////////////////////////////////////////////////////////////// 467 | static void rlu_init_quiescence(rlu_thread_data_t *self) { 468 | int th_id; 469 | 470 | MEMBARSTLD(); 471 | 472 | for (th_id = 0; th_id < g_rlu_cur_threads; th_id++) { 473 | 474 | self->q_threads[th_id].is_wait = 0; 475 | 476 | if (th_id == self->uniq_id) { 477 | // No need to wait for myself 478 | continue; 479 | } 480 | 481 | if (g_rlu_threads[th_id] == NULL) { 482 | // No need to wait for uninitialized threads 483 | continue; 484 | } 485 | 486 | self->q_threads[th_id].run_counter = g_rlu_threads[th_id]->run_counter; 487 | 488 | if (self->q_threads[th_id].run_counter & 0x1) { 489 | // The other thread is running -> wait for the thread 490 | self->q_threads[th_id].is_wait = 1; 491 | } 492 | } 493 | } 494 | 495 | static long rlu_wait_for_quiescence(rlu_thread_data_t *self, long version_limit) { 496 | int th_id; 497 | long iters; 498 | long cur_threads; 499 | 500 | iters = 0; 501 | cur_threads = g_rlu_cur_threads; 502 | for (th_id = 0; th_id < cur_threads; th_id++) { 503 | 504 | while (self->q_threads[th_id].is_wait) { 505 | iters++; 506 | 507 | if (self->q_threads[th_id].run_counter != g_rlu_threads[th_id]->run_counter) { 508 | self->q_threads[th_id].is_wait = 0; 509 | break; 510 | } 511 | 512 | if (version_limit) { 513 | if (g_rlu_threads[th_id]->local_version >= version_limit) { 514 | self->q_threads[th_id].is_wait = 0; 515 | break; 516 | } 517 | } 518 | 519 | if (iters > Q_ITERS_LIMIT) { 520 | iters = 0; 521 | printf("[%ld] waiting for [%d] with: local_version = %ld , run_cnt = %ld\n", self->uniq_id, th_id, 522 | g_rlu_threads[th_id]->local_version, g_rlu_threads[th_id]->run_counter); 523 | } 524 | 525 | CPU_RELAX(); 526 | 527 | } 528 | } 529 | 530 | 531 | return iters; 532 | } 533 | 534 | static void rlu_synchronize(rlu_thread_data_t *self) { 535 | long q_iters; 536 | 537 | if (self->is_no_quiescence) { 538 | return; 539 | } 540 | 541 | rlu_init_quiescence(self); 542 | 543 | q_iters = rlu_wait_for_quiescence(self, self->writer_version); 544 | 545 | self->n_writeback_q_iters += q_iters; 546 | 547 | } 548 | 549 | static void rlu_sync_and_writeback(rlu_thread_data_t *self) { 550 | long ws_num; 551 | long ws_wb_num; 552 | 553 | RLU_ASSERT((self->run_counter & 0x1) == 0); 554 | 555 | if (self->ws_tail_counter == self->ws_head_counter) { 556 | return; 557 | } 558 | 559 | self->n_sync_and_writeback++; 560 | 561 | ws_num = self->ws_tail_counter - self->ws_wb_counter; 562 | 563 | self->writer_version = g_rlu_writer_version + 1; 564 | FETCH_AND_ADD(&g_rlu_writer_version, 1); 565 | 566 | TRACE_1(self, "start, ws_num = %ld writer_version = %ld\n", 567 | ws_num, self->writer_version); 568 | 569 | rlu_synchronize(self); 570 | 571 | ws_wb_num = rlu_writeback_write_sets_and_unlock(self); 572 | 573 | RLU_ASSERT_MSG(ws_num == ws_wb_num, self, "failed: %ld != %ld\n", ws_num, ws_wb_num); 574 | 575 | self->writer_version = MAX_VERSION; 576 | 577 | FETCH_AND_ADD(&g_rlu_commit_version, 1); 578 | 579 | if (self->is_sync) { 580 | self->is_sync = 0; 581 | } 582 | 583 | rlu_process_free(self); 584 | 585 | } 586 | 587 | static void rlu_send_sync_request(int other_th_id) { 588 | g_rlu_threads[other_th_id]->is_sync++; 589 | MEMBARSTLD(); 590 | } 591 | 592 | static void rlu_commit_write_set(rlu_thread_data_t *self) { 593 | self->n_writers++; 594 | self->n_writer_writeback++; 595 | 596 | // Move to the next write-set 597 | self->ws_tail_counter++; 598 | self->ws_cur_id = WS_INDEX(self->ws_tail_counter); 599 | 600 | // Sync and writeback when: 601 | // (1) All write-sets are full 602 | // (2) Aggregared MAX_ACTUAL_WRITE_SETS 603 | if ((WS_INDEX(self->ws_tail_counter) == WS_INDEX(self->ws_head_counter)) || 604 | ((self->ws_tail_counter - self->ws_wb_counter) >= self->max_write_sets)) { 605 | rlu_sync_and_writeback(self); 606 | } 607 | 608 | RLU_ASSERT(self->ws_tail_counter > self->ws_head_counter); 609 | RLU_ASSERT(WS_INDEX(self->ws_tail_counter) != WS_INDEX(self->ws_head_counter)); 610 | } 611 | 612 | ///////////////////////////////////////////////////////////////////////////////////////// 613 | // EXTERNAL FUNCTIONS 614 | ///////////////////////////////////////////////////////////////////////////////////////// 615 | void rlu_init(int type, int max_write_sets) { 616 | g_rlu_writer_version = 0; 617 | g_rlu_commit_version = 0; 618 | 619 | if (type == RLU_TYPE_COARSE_GRAINED) { 620 | g_rlu_type = RLU_TYPE_COARSE_GRAINED; 621 | g_rlu_max_write_sets = 1; 622 | printf("RLU - COARSE_GRAINED initialized\n"); 623 | 624 | } else if (type == RLU_TYPE_FINE_GRAINED) { 625 | g_rlu_type = RLU_TYPE_FINE_GRAINED; 626 | g_rlu_max_write_sets = max_write_sets; 627 | printf("RLU - FINE_GRAINED initialized [max_write_sets = %d]\n", g_rlu_max_write_sets); 628 | 629 | } else { 630 | RLU_TRACE_GLOBAL("unknown type [%d]", type); 631 | abort(); 632 | } 633 | 634 | RLU_ASSERT(RLU_MAX_WRITE_SETS >= 2); 635 | RLU_ASSERT(max_write_sets >= 1); 636 | RLU_ASSERT(max_write_sets * 2 <= RLU_MAX_WRITE_SETS); 637 | } 638 | 639 | void rlu_finish(void) { } 640 | 641 | void rlu_print_stats(void) { 642 | printf("=================================================\n"); 643 | printf("RLU statistics:\n"); 644 | printf("-------------------------------------------------\n"); 645 | printf(" t_starts = %lu\n", g_rlu_data.n_starts); 646 | printf(" t_finish = %lu\n", g_rlu_data.n_finish); 647 | printf(" t_writers = %lu\n", g_rlu_data.n_writers); 648 | printf("-------------------------------------------------\n"); 649 | printf(" t_writer_writebacks = %lu\n", g_rlu_data.n_writer_writeback); 650 | printf(" t_writeback_q_iters = %lu\n", g_rlu_data.n_writeback_q_iters); 651 | if (g_rlu_data.n_writer_writeback > 0) { 652 | printf(" a_writeback_q_iters = %lu\n", g_rlu_data.n_writeback_q_iters / g_rlu_data.n_writer_writeback); 653 | } else { 654 | printf(" a_writeback_q_iters = 0\n"); 655 | } 656 | printf(" t_pure_readers = %lu\n", g_rlu_data.n_pure_readers); 657 | printf(" t_steals = %lu\n", g_rlu_data.n_steals); 658 | printf(" t_aborts = %lu\n", g_rlu_data.n_aborts); 659 | printf(" t_sync_requests = %lu\n", g_rlu_data.n_sync_requests); 660 | printf(" t_sync_and_writeback = %lu\n", g_rlu_data.n_sync_and_writeback); 661 | 662 | printf("=================================================\n"); 663 | } 664 | 665 | void rlu_thread_init(rlu_thread_data_t *self) { 666 | int ws_counter; 667 | 668 | memset(self, 0, sizeof(rlu_thread_data_t)); 669 | 670 | self->type = g_rlu_type; 671 | self->max_write_sets = g_rlu_max_write_sets; 672 | 673 | self->uniq_id = FETCH_AND_ADD(&g_rlu_cur_threads, 1); 674 | 675 | self->local_version = 0; 676 | self->writer_version = MAX_VERSION; 677 | 678 | for (ws_counter = 0; ws_counter < RLU_MAX_WRITE_SETS; ws_counter++) { 679 | rlu_reset_write_set(self, ws_counter); 680 | } 681 | 682 | g_rlu_threads[self->uniq_id] = self; 683 | MEMBARSTLD(); 684 | 685 | } 686 | 687 | void rlu_thread_finish(rlu_thread_data_t *self) { 688 | rlu_sync_and_writeback(self); 689 | rlu_sync_and_writeback(self); 690 | 691 | FETCH_AND_ADD(&g_rlu_data.n_starts, self->n_starts); 692 | FETCH_AND_ADD(&g_rlu_data.n_finish, self->n_finish); 693 | FETCH_AND_ADD(&g_rlu_data.n_writers, self->n_writers); 694 | FETCH_AND_ADD(&g_rlu_data.n_writer_writeback, self->n_writer_writeback); 695 | FETCH_AND_ADD(&g_rlu_data.n_writeback_q_iters, self->n_writeback_q_iters); 696 | FETCH_AND_ADD(&g_rlu_data.n_pure_readers, self->n_pure_readers); 697 | FETCH_AND_ADD(&g_rlu_data.n_steals, self->n_steals); 698 | FETCH_AND_ADD(&g_rlu_data.n_aborts, self->n_aborts); 699 | FETCH_AND_ADD(&g_rlu_data.n_sync_requests, self->n_sync_requests); 700 | FETCH_AND_ADD(&g_rlu_data.n_sync_and_writeback, self->n_sync_and_writeback); 701 | } 702 | 703 | intptr_t *rlu_alloc(obj_size_t obj_size) { 704 | intptr_t *ptr; 705 | rlu_obj_header_t *p_obj_h; 706 | 707 | ptr = (intptr_t *)malloc(OBJ_HEADER_SIZE + obj_size); 708 | if (ptr == NULL) { 709 | return NULL; 710 | } 711 | p_obj_h = (rlu_obj_header_t *)ptr; 712 | p_obj_h->p_obj_copy = NULL; 713 | 714 | TRACE_3_GLOBAL("ptr=%p full_size=%zu ptr_to_obj=%p\n", 715 | ptr, (OBJ_HEADER_SIZE + obj_size), (intptr_t *)H_TO_OBJ(p_obj_h)); 716 | 717 | return (intptr_t *)H_TO_OBJ(p_obj_h); 718 | } 719 | 720 | void rlu_free(rlu_thread_data_t *self, intptr_t *p_obj) { 721 | if (p_obj == NULL) { 722 | return; 723 | } 724 | 725 | if (self == NULL) { 726 | free((intptr_t *)OBJ_TO_H(p_obj)); 727 | return; 728 | } 729 | 730 | rlu_assert_in_section(self); 731 | 732 | p_obj = (intptr_t *)FORCE_ACTUAL(p_obj); 733 | 734 | self->free_nodes[self->free_nodes_size] = p_obj; 735 | self->free_nodes_size++; 736 | 737 | RLU_ASSERT(self->free_nodes_size < RLU_MAX_FREE_NODES); 738 | 739 | } 740 | 741 | void rlu_sync_checkpoint(rlu_thread_data_t *self) { 742 | 743 | if (likely(!self->is_sync)) { 744 | return; 745 | } 746 | 747 | self->n_sync_requests++; 748 | rlu_sync_and_writeback(self); 749 | 750 | } 751 | 752 | void rlu_reader_lock(rlu_thread_data_t *self) { 753 | self->n_starts++; 754 | 755 | rlu_sync_checkpoint(self); 756 | 757 | rlu_reset(self); 758 | 759 | rlu_register_thread(self); 760 | 761 | self->is_steal = 1; 762 | if ((self->local_version - self->local_commit_version) == 0) { 763 | self->is_steal = 0; 764 | } 765 | 766 | self->is_check_locks = 1; 767 | if (((self->local_version - self->local_commit_version) == 0) && ((self->ws_tail_counter - self->ws_wb_counter) == 0)) { 768 | self->is_check_locks = 0; 769 | self->n_pure_readers++; 770 | } 771 | } 772 | 773 | int rlu_try_writer_lock(rlu_thread_data_t *self, int writer_lock_id) { 774 | RLU_ASSERT(self->type == RLU_TYPE_COARSE_GRAINED); 775 | 776 | if (!rlu_try_acquire_writer_lock(self, writer_lock_id)) { 777 | return 0; 778 | } 779 | 780 | rlu_add_writer_lock(self, writer_lock_id); 781 | 782 | return 1; 783 | } 784 | 785 | void rlu_reader_unlock(rlu_thread_data_t *self) { 786 | self->n_finish++; 787 | 788 | rlu_unregister_thread(self); 789 | 790 | if (self->is_write_detected) { 791 | self->is_write_detected = 0; 792 | rlu_commit_write_set(self); 793 | rlu_release_writer_locks(self, WS_INDEX(self->ws_tail_counter - 1)); 794 | } else { 795 | rlu_release_writer_locks(self, self->ws_cur_id); 796 | rlu_reset_writer_locks(self, self->ws_cur_id); 797 | } 798 | 799 | rlu_sync_checkpoint(self); 800 | 801 | } 802 | 803 | intptr_t *rlu_deref_slow_path(rlu_thread_data_t *self, intptr_t *p_obj) { 804 | long th_id; 805 | intptr_t *p_obj_copy; 806 | volatile rlu_ws_obj_header_t *p_ws_obj_h; 807 | 808 | if (p_obj == NULL) { 809 | return p_obj; 810 | } 811 | 812 | p_obj_copy = (intptr_t *)GET_COPY(p_obj); 813 | 814 | if (!PTR_IS_LOCKED(p_obj_copy)) { 815 | return p_obj; 816 | } 817 | 818 | if (PTR_IS_COPY(p_obj_copy)) { 819 | // p_obj points to a copy -> it has been already dereferenced. 820 | TRACE_1(self, "got pointer to a copy. p_obj = %p p_obj_copy = %p.\n", p_obj, p_obj_copy); 821 | return p_obj; 822 | } 823 | 824 | p_ws_obj_h = PTR_GET_WS_HEADER(p_obj_copy); 825 | 826 | th_id = WS_GET_THREAD_ID(p_ws_obj_h); 827 | 828 | if (th_id == self->uniq_id) { 829 | // p_obj is locked by this thread -> return the copy 830 | TRACE_1(self, "got pointer to a copy. p_obj = %p th_id = %ld.\n", p_obj, th_id); 831 | return p_obj_copy; 832 | } 833 | 834 | // p_obj is locked by another thread 835 | if ((self->is_steal) && 836 | (g_rlu_threads[th_id]->writer_version <= self->local_version)) { 837 | // This thread started after the other thread updated g_writer_version. 838 | // and this thread observed a valid p_obj_copy (!= NULL) 839 | // => The other thread is going to wait for this thread to finish before reusing the write-set log 840 | // (to which p_obj_copy points) 841 | TRACE_1(self, "take copy from other writer th_id = %ld p_obj = %p p_obj_copy = %p\n", 842 | th_id, p_obj, p_obj_copy); 843 | self->n_steals++; 844 | return p_obj_copy; 845 | } 846 | 847 | return p_obj; 848 | } 849 | 850 | int rlu_try_lock(rlu_thread_data_t *self, intptr_t **p_p_obj, size_t obj_size) { 851 | intptr_t *p_obj; 852 | intptr_t *p_obj_copy; 853 | volatile long th_id; 854 | volatile rlu_ws_obj_header_t *p_ws_obj_h; 855 | 856 | p_obj = *p_p_obj; 857 | 858 | RLU_ASSERT_MSG(p_obj != NULL, self, "[%ld] rlu_try_lock: tried to lock a NULL pointer\n", self->writer_version); 859 | 860 | p_obj_copy = (intptr_t *)GET_COPY(p_obj); 861 | 862 | if (PTR_IS_COPY(p_obj_copy)) { 863 | TRACE_1(self, "tried to lock a copy of an object. p_obj = %p\n", p_obj); 864 | TRACE_1(self, " => converting \n => copy: %p\n", p_obj); 865 | 866 | p_obj = (intptr_t *)GET_ACTUAL(p_obj); 867 | p_obj_copy = (intptr_t *)GET_COPY(p_obj); 868 | 869 | TRACE_1(self, " => real: %p , p_obj_copy = %p\n", p_obj, p_obj_copy); 870 | } 871 | 872 | if (PTR_IS_LOCKED(p_obj_copy)) { 873 | p_ws_obj_h = PTR_GET_WS_HEADER(p_obj_copy); 874 | 875 | th_id = WS_GET_THREAD_ID(p_ws_obj_h); 876 | 877 | if (th_id == self->uniq_id) { 878 | if (self->run_counter == WS_GET_RUN_COUNTER(p_ws_obj_h)) { 879 | // p_obj already locked by current execution of this thread. 880 | // => return copy 881 | TRACE_2(self, "[%ld] already locked by this thread. p_obj = %p th_id = %ld\n", 882 | self->local_version, p_obj, th_id); 883 | *p_p_obj = p_obj_copy; 884 | return 1; 885 | } 886 | 887 | TRACE_1(self, "[%ld] already locked by another execution of this thread -> fail and sync. p_obj = %p th_id = %ld\n", 888 | self->local_version, p_obj, th_id); 889 | // p_obj is locked by another execution of this thread. 890 | self->is_sync++; 891 | return 0; 892 | } 893 | 894 | // p_obj already locked by another thread. 895 | // => send sync request to the other thread 896 | // => in the meantime -> sync this thread 897 | TRACE_1(self, "[%ld] already locked by another thread -> fail and request sync. p_obj = %p th_id = %ld\n", 898 | self->local_version, p_obj, th_id); 899 | 900 | rlu_send_sync_request(th_id); 901 | self->is_sync++; 902 | return 0; 903 | } 904 | 905 | // p_obj is free 906 | 907 | // Indicate that write-set is updated 908 | if (self->is_write_detected == 0) { 909 | self->is_write_detected = 1; 910 | self->is_check_locks = 1; 911 | } 912 | 913 | // Add write-set header for the object 914 | p_obj_copy = rlu_add_ws_obj_header_to_write_set(self, p_obj, obj_size); 915 | 916 | // Try lock p_obj -> install pointer to copy 917 | if (!TRY_CAS_PTR_OBJ_COPY(p_obj, p_obj_copy)) { 918 | TRACE_1(self, "[%ld] CAS failed\n", self->local_version); 919 | return 0; 920 | } 921 | 922 | // Locked successfully 923 | 924 | // Copy object to write-set 925 | rlu_add_obj_copy_to_write_set(self, p_obj, obj_size); 926 | 927 | RLU_ASSERT_MSG(GET_COPY(p_obj) == p_obj_copy, 928 | self, "p_obj_copy = %p my_p_obj_copy = %p\n", 929 | GET_COPY(p_obj), p_obj_copy); 930 | 931 | TRACE_2(self, "[%ld] p_obj = %p is locked, p_obj_copy = %p , th_id = %ld\n", 932 | self->local_version, p_obj, GET_COPY(p_obj), GET_THREAD_ID(p_obj)); 933 | 934 | *p_p_obj = p_obj_copy; 935 | return 1; 936 | 937 | } 938 | 939 | void rlu_lock(rlu_thread_data_t *self, intptr_t **p_p_obj, unsigned int obj_size) { 940 | RLU_ASSERT(self->type == RLU_TYPE_COARSE_GRAINED); 941 | 942 | RLU_ASSERT(rlu_try_lock(self, p_p_obj, obj_size) != 0); 943 | } 944 | 945 | void rlu_abort(rlu_thread_data_t *self) { 946 | self->n_aborts++; 947 | 948 | rlu_unregister_thread(self); 949 | 950 | if (self->is_write_detected) { 951 | self->is_write_detected = 0; 952 | rlu_unlock_objs(self, self->ws_tail_counter); 953 | 954 | rlu_release_writer_locks(self, self->ws_cur_id); 955 | rlu_reset_write_set(self, self->ws_tail_counter); 956 | } else { 957 | rlu_release_writer_locks(self, self->ws_cur_id); 958 | rlu_reset_writer_locks(self, self->ws_cur_id); 959 | } 960 | 961 | rlu_sync_checkpoint(self); 962 | 963 | } 964 | 965 | int rlu_cmp_ptrs(intptr_t *p_obj_1, intptr_t *p_obj_2) { 966 | if (p_obj_1 != NULL) { 967 | p_obj_1 = (intptr_t *)FORCE_ACTUAL(p_obj_1); 968 | } 969 | 970 | if (p_obj_2 != NULL) { 971 | p_obj_2 = (intptr_t *)FORCE_ACTUAL(p_obj_2); 972 | } 973 | 974 | return p_obj_1 == p_obj_2; 975 | } 976 | 977 | void rlu_assign_pointer(intptr_t **p_ptr, intptr_t *p_obj) { 978 | if (p_obj != NULL) { 979 | p_obj = (intptr_t *)FORCE_ACTUAL(p_obj); 980 | } 981 | 982 | *p_ptr = p_obj; 983 | } 984 | 985 | 986 | -------------------------------------------------------------------------------- /rlu.h: -------------------------------------------------------------------------------- 1 | ///////////////////////////////////////////////////////////////////////////////////////// 2 | // 3 | // 4 | // 5 | ///////////////////////////////////////////////////////////////////////////////////////// 6 | 7 | #ifndef RLU_H 8 | #define RLU_H 1 9 | 10 | ///////////////////////////////////////////////////////////////////////////////////////// 11 | // INCLUDES 12 | ///////////////////////////////////////////////////////////////////////////////////////// 13 | #ifndef KERNEL 14 | # include 15 | # include 16 | #else /* KERNEL */ 17 | typedef long intptr_t; // Hack for kernel that does not have intptr_t 18 | #endif /* KERNEL */ 19 | 20 | ///////////////////////////////////////////////////////////////////////////////////////// 21 | // DEFINES - CONFIGURATION 22 | ///////////////////////////////////////////////////////////////////////////////////////// 23 | #define RLU_TYPE_FINE_GRAINED (1) 24 | #define RLU_TYPE_COARSE_GRAINED (2) 25 | 26 | #define RLU_MAX_THREADS (128) 27 | 28 | #define RLU_MAX_WRITE_SETS (200) // Minimum value is 2 29 | #define RLU_MAX_FREE_NODES (100000) 30 | 31 | #define RLU_MAX_WRITE_SET_BUFFER_SIZE (100000) 32 | 33 | #define RLU_MAX_NESTED_WRITER_LOCKS (20) 34 | #define RLU_MAX_WRITER_LOCKS (20000) 35 | 36 | #define RLU_GENERAL_WRITER_LOCK (RLU_MAX_WRITER_LOCKS-1) 37 | 38 | //#define RLU_ENABLE_TRACE_1 39 | //#define RLU_ENABLE_TRACE_2 40 | //#define RLU_ENABLE_TRACE_3 41 | 42 | ///////////////////////////////////////////////////////////////////////////////////////// 43 | // DEFINES - INTERNAL 44 | ///////////////////////////////////////////////////////////////////////////////////////// 45 | 46 | #define rlu_likely(x) __builtin_expect ((x), 1) 47 | 48 | #define RLU_DEFAULT_PADDING (16) 49 | 50 | #define RLU_OBJ_HEADER_SIZE (sizeof(rlu_obj_header_t)) 51 | #define RLU_MOVE_PTR_BACK(p_obj, offset) ((intptr_t *)(((volatile unsigned char *)p_obj) - offset)) 52 | #define RLU_OBJ_TO_H(p_obj) ((volatile rlu_obj_header_t *)RLU_MOVE_PTR_BACK(p_obj, RLU_OBJ_HEADER_SIZE)) 53 | 54 | #define RLU_GET_COPY(p_obj) (RLU_OBJ_TO_H(p_obj)->p_obj_copy) 55 | #define RLU_PTR_IS_LOCKED(p_obj_copy) (p_obj_copy != NULL) 56 | 57 | #define RLU_IS_UNLOCKED(p_obj) (!RLU_PTR_IS_LOCKED(RLU_GET_COPY(p_obj))) 58 | 59 | ///////////////////////////////////////////////////////////////////////////////////////// 60 | // TYPES 61 | ///////////////////////////////////////////////////////////////////////////////////////// 62 | typedef size_t obj_size_t; 63 | 64 | typedef struct rlu_obj_header { 65 | volatile intptr_t *p_obj_copy; 66 | } rlu_obj_header_t; 67 | 68 | typedef struct rlu_ws_obj_header { 69 | volatile intptr_t *p_obj_actual; 70 | volatile obj_size_t obj_size; 71 | volatile unsigned long run_counter; 72 | volatile long thread_id; 73 | } rlu_ws_obj_header_t; 74 | 75 | typedef struct writer_locks { 76 | long size; 77 | long ids[RLU_MAX_NESTED_WRITER_LOCKS]; 78 | } writer_locks_t; 79 | 80 | typedef struct obj_list { 81 | volatile writer_locks_t writer_locks; 82 | unsigned int num_of_objs; 83 | volatile intptr_t *p_cur; 84 | volatile unsigned char buffer[RLU_MAX_WRITE_SET_BUFFER_SIZE]; 85 | } obj_list_t; 86 | 87 | typedef struct wait_entry { 88 | volatile unsigned char is_wait; 89 | volatile unsigned long run_counter; 90 | } wait_entry_t; 91 | 92 | typedef struct rlu_thread_data { 93 | 94 | long padding_0[RLU_DEFAULT_PADDING]; 95 | 96 | long uniq_id; 97 | 98 | char is_check_locks; 99 | char is_write_detected; 100 | char is_steal; 101 | int type; 102 | int max_write_sets; 103 | 104 | long padding_1[RLU_DEFAULT_PADDING]; 105 | 106 | volatile unsigned long run_counter; 107 | volatile long local_version; 108 | volatile long local_commit_version; 109 | volatile long is_no_quiescence; 110 | volatile long is_sync; 111 | 112 | long padding_2[RLU_DEFAULT_PADDING]; 113 | 114 | volatile long writer_version; 115 | 116 | long padding_3[RLU_DEFAULT_PADDING]; 117 | 118 | wait_entry_t q_threads[RLU_MAX_THREADS]; 119 | 120 | long ws_head_counter; 121 | long ws_wb_counter; 122 | long ws_tail_counter; 123 | long ws_cur_id; 124 | volatile obj_list_t obj_write_set[RLU_MAX_WRITE_SETS]; 125 | 126 | long padding_4[RLU_DEFAULT_PADDING]; 127 | 128 | long free_nodes_size; 129 | intptr_t *free_nodes[RLU_MAX_FREE_NODES]; 130 | 131 | long padding_5[RLU_DEFAULT_PADDING]; 132 | 133 | long n_starts; 134 | long n_finish; 135 | long n_writers; 136 | long n_writer_writeback; 137 | long n_pure_readers; 138 | long n_aborts; 139 | long n_steals; 140 | long n_writer_sync_waits; 141 | long n_writeback_q_iters; 142 | long n_sync_requests; 143 | long n_sync_and_writeback; 144 | 145 | long padding_6[RLU_DEFAULT_PADDING]; 146 | 147 | } rlu_thread_data_t; 148 | 149 | ///////////////////////////////////////////////////////////////////////////////////////// 150 | // EXTERNAL FUNCTIONS 151 | ///////////////////////////////////////////////////////////////////////////////////////// 152 | 153 | void rlu_init(int type, int max_write_sets); 154 | void rlu_finish(void); 155 | void rlu_print_stats(void); 156 | 157 | void rlu_thread_init(rlu_thread_data_t *self); 158 | void rlu_thread_finish(rlu_thread_data_t *self); 159 | 160 | intptr_t *rlu_alloc(obj_size_t obj_size); 161 | void rlu_free(rlu_thread_data_t *self, intptr_t *p_obj); 162 | 163 | void rlu_reader_lock(rlu_thread_data_t *self); 164 | void rlu_reader_unlock(rlu_thread_data_t *self); 165 | 166 | int rlu_try_lock(rlu_thread_data_t *self, intptr_t **p_p_obj, size_t obj_size); 167 | void rlu_abort(rlu_thread_data_t *self); 168 | 169 | int rlu_try_writer_lock(rlu_thread_data_t *self, int writer_lock_id); 170 | void rlu_lock(rlu_thread_data_t *self, intptr_t **p_p_obj, unsigned int obj_size); 171 | 172 | intptr_t *rlu_deref_slow_path(rlu_thread_data_t *self, intptr_t *p_obj); 173 | 174 | int rlu_cmp_ptrs(intptr_t *p_obj_1, intptr_t *p_obj_2); 175 | void rlu_assign_pointer(intptr_t **p_ptr, intptr_t *p_obj); 176 | 177 | void rlu_sync_checkpoint(rlu_thread_data_t *self); 178 | 179 | ///////////////////////////////////////////////////////////////////////////////////////// 180 | // EXTERNAL MACROS 181 | ///////////////////////////////////////////////////////////////////////////////////////// 182 | 183 | #define RLU_INIT(type, max_write_sets) rlu_init(type, max_write_sets); 184 | #define RLU_FINISH() rlu_finish(); 185 | #define RLU_PRINT_STATS() rlu_print_stats() 186 | 187 | #define RLU_THREAD_INIT(self) rlu_thread_init(self) 188 | #define RLU_THREAD_FINISH(self) rlu_thread_finish(self) 189 | 190 | #define RLU_READER_LOCK(self) rlu_reader_lock(self) 191 | #define RLU_READER_UNLOCK(self) rlu_reader_unlock(self) 192 | 193 | #define RLU_ALLOC(obj_size) ((void *)rlu_alloc(obj_size)) 194 | #define RLU_FREE(self, p_obj) rlu_free(self, (intptr_t *)p_obj) 195 | 196 | #define RLU_TRY_WRITER_LOCK(self, writer_lock_id) rlu_try_writer_lock(self, writer_lock_id) 197 | #define RLU_LOCK(self, p_p_obj) rlu_lock(self, (intptr_t **)p_p_obj, sizeof(**p_p_obj)) 198 | 199 | #define RLU_TRY_LOCK(self, p_p_obj) rlu_try_lock(self, (intptr_t **)p_p_obj, sizeof(**p_p_obj)) 200 | #define RLU_ABORT(self) rlu_abort(self) 201 | 202 | #define RLU_IS_SAME_PTRS(p_obj_1, p_obj_2) rlu_cmp_ptrs((intptr_t *)p_obj_1, (intptr_t *)p_obj_2) 203 | #define RLU_ASSIGN_PTR(self, p_ptr, p_obj) rlu_assign_pointer((intptr_t **)p_ptr, (intptr_t *)p_obj) 204 | 205 | #define RLU_DEREF(self, p_obj) RLU_DEREF_INTERNAL(self, (intptr_t *)p_obj) 206 | 207 | #define RLU_DEREF_INTERNAL(self, p_obj) ({ \ 208 | intptr_t *p_cur_obj; \ 209 | if (rlu_likely(self->is_check_locks == 0)) { \ 210 | p_cur_obj = p_obj; \ 211 | } else { \ 212 | if (rlu_likely((p_obj != NULL) && RLU_IS_UNLOCKED(p_obj))) { \ 213 | p_cur_obj = p_obj; \ 214 | } else { \ 215 | p_cur_obj = rlu_deref_slow_path(self, p_obj); \ 216 | } \ 217 | }; p_cur_obj; }) 218 | 219 | #endif // RLU_H 220 | -------------------------------------------------------------------------------- /run_tests.py: -------------------------------------------------------------------------------- 1 | 2 | import time 3 | import sys 4 | import os 5 | 6 | IS_NUMA = 1 7 | IS_2_SOCKET = 0 8 | IS_PERF = 0 9 | 10 | CMD_PARAMS = '-a -w%d -b%d -d%d -u%d -i%d -r%d -n%d' 11 | 12 | PERF_FILE = "__perf_output.file" 13 | CMD_PREFIX_PERF = "perf stat -d -o %s" % (PERF_FILE,) 14 | 15 | CMD_BASE_HARRIS = './bench-harris' 16 | CMD_BASE_HP_HARRIS = './bench-hp-harris' 17 | CMD_BASE_RCU = './bench-rcu' 18 | CMD_BASE_RLU = './bench-rlu' 19 | 20 | OUTPUT_FILENAME = '___temp.file' 21 | 22 | W_OUTPUT_FILENAME = '__w_check.txt' 23 | 24 | CMD_PREFIX_LIBS = 'export LD_PRELOAD=\\"$GPERFTOOLS_LIB/libtcmalloc_minimal.so\\"' 25 | 26 | CMD_NUMA_BIND_TO_CPU_0 = 'numactl --cpunodebind=0 ' 27 | CMD_NUMA_BIND_TO_CPU_1 = 'numactl --cpunodebind=1 ' 28 | CMD_NUMA_BIND_TO_CPU_0_1 = 'numactl --cpunodebind=0,1 ' 29 | CMD_NUMA_PREFIX_8 = 'taskset -c 0-7 ' 30 | CMD_NUMA_PREFIX_10 = 'taskset -c 0-9 ' 31 | CMD_NUMA_PREFIX_12 = 'taskset -c 0-11 ' 32 | CMD_NUMA_PREFIX_14 = 'taskset -c 0-13 ' 33 | CMD_NUMA_PREFIX_16 = 'taskset -c 0-15 ' 34 | 35 | CMD_BASE = { 36 | 'harris' : CMD_BASE_HARRIS, 37 | 'hp_harris' : CMD_BASE_HP_HARRIS, 38 | 'rcu' : CMD_BASE_RCU, 39 | 'rlu' : CMD_BASE_RLU, 40 | } 41 | 42 | result_keys = [ 43 | '#ops :', 44 | '#update ops :', 45 | 't_writer_writebacks =', 46 | 't_writeback_q_iters =', 47 | 'a_writeback_q_iters =', 48 | 't_pure_readers =', 49 | 't_steals =', 50 | 't_aborts =', 51 | 't_sync_requests =', 52 | 't_sync_and_writeback =', 53 | ] 54 | 55 | perf_result_keys = [ 56 | 'instructions #', 57 | 'branches #', 58 | 'branch-misses #', 59 | 'L1-dcache-loads #', 60 | 'L1-dcache-load-misses #', 61 | ] 62 | 63 | def cmd_numa_prefix(threads_num): 64 | if (IS_2_SOCKET): 65 | if (threads_num <= 36): 66 | return CMD_NUMA_BIND_TO_CPU_1 67 | 68 | return CMD_NUMA_BIND_TO_CPU_0_1 69 | 70 | if (threads_num <= 8): 71 | return CMD_NUMA_PREFIX_8 72 | 73 | if (threads_num <= 10): 74 | return CMD_NUMA_PREFIX_10 75 | 76 | if (threads_num <= 12): 77 | return CMD_NUMA_PREFIX_12 78 | 79 | if (threads_num <= 14): 80 | return CMD_NUMA_PREFIX_14 81 | 82 | if (threads_num <= 16): 83 | return CMD_NUMA_PREFIX_16 84 | 85 | print 'cmd_numa_prefix: ERROR th_num = %d' % (threads_num,) 86 | 87 | 88 | def extract_data(output_data, key_str): 89 | data = output_data.split(key_str)[1].split()[0].strip() 90 | 91 | if (data.find('nan') != -1): 92 | return 0 93 | 94 | if (key_str == 'L1-dcache-load-misses #') or (key_str == 'branch-misses #'): 95 | data = data.strip('%') 96 | 97 | return float(data) 98 | 99 | 100 | def extract_keys(output_data): 101 | d = {} 102 | 103 | for key in result_keys: 104 | d[key] = extract_data(output_data, key) 105 | 106 | if IS_PERF: 107 | for key in perf_result_keys: 108 | d[key] = extract_data(output_data, key) 109 | 110 | return d 111 | 112 | 113 | def print_keys(dict_keys): 114 | print '=================================' 115 | for key in result_keys: 116 | print '%s %.2f' % (key, dict_keys[key]) 117 | 118 | if IS_PERF: 119 | for key in perf_result_keys: 120 | print '%s %.2f' % (key, dict_keys[key]) 121 | 122 | 123 | def run_test(runs_per_test, alg_type, cmd): 124 | 125 | ops_total = 0 126 | total_operations = 0 127 | aborts_total = 0 128 | total_combiners = 0 129 | total_num_of_waiting = 0 130 | total_additional_readers = 0 131 | total_additional_writers = 0 132 | 133 | cmd_prefix = 'bash -c "' + CMD_PREFIX_LIBS + ' ; ' 134 | if (IS_PERF): 135 | cmd_prefix += CMD_PREFIX_PERF + ' ' 136 | 137 | full_cmd = cmd_prefix + cmd + '"' 138 | 139 | print full_cmd 140 | 141 | total_dict_keys = {} 142 | for key in result_keys: 143 | total_dict_keys[key] = 0 144 | 145 | for i in xrange(runs_per_test): 146 | print 'run %d ' % (i,) 147 | 148 | 149 | if (IS_PERF): 150 | try: 151 | os.unlink(PERF_FILE) 152 | except OSError: 153 | pass 154 | 155 | os.system('w >> %s' % (W_OUTPUT_FILENAME,)) 156 | time.sleep(1) 157 | os.system(full_cmd + ' > ' + OUTPUT_FILENAME) 158 | os.system('w >> %s' % (W_OUTPUT_FILENAME,)) 159 | 160 | time.sleep(1) 161 | f = open(OUTPUT_FILENAME, 'rb') 162 | output_data = f.read() 163 | f.close() 164 | os.unlink(OUTPUT_FILENAME) 165 | 166 | if (IS_PERF): 167 | f = open(PERF_FILE, 'rb'); 168 | output_data += f.read() 169 | f.close() 170 | os.unlink(PERF_FILE) 171 | 172 | print "------------------------------------" 173 | print output_data 174 | print "------------------------------------" 175 | 176 | dict_keys = extract_keys(output_data) 177 | 178 | print '=================================' 179 | for key in dict_keys.keys(): 180 | total_dict_keys[key] += dict_keys[key] 181 | 182 | print_keys(dict_keys) 183 | 184 | for key in total_dict_keys.keys(): 185 | total_dict_keys[key] /= runs_per_test 186 | 187 | return total_dict_keys 188 | 189 | def print_run_results(f_out, rlu_max_ws, update_ratio, th_num, dict_keys): 190 | 191 | f_out.write('\n%.2f %.2f %.2f' % (rlu_max_ws, update_ratio, th_num)); 192 | 193 | for key in result_keys: 194 | f_out.write(' %.2f' % dict_keys[key]) 195 | 196 | if IS_PERF: 197 | for key in perf_result_keys: 198 | f_out.write(' %.2f' % dict_keys[key]) 199 | 200 | f_out.flush() 201 | 202 | 203 | def execute(runs_per_test, 204 | rlu_max_ws, 205 | buckets, 206 | duration, 207 | alg_type, 208 | update_ratio, 209 | initial_size, 210 | range_size, 211 | output_filename, 212 | th_num_list): 213 | 214 | 215 | f_w = open(W_OUTPUT_FILENAME, 'wb'); 216 | f_w.close() 217 | 218 | f_out = open(output_filename, 'wb') 219 | 220 | cmd_header = '[%s] ' % (alg_type,) + CMD_BASE[alg_type] + ' ' + CMD_PARAMS % ( 221 | rlu_max_ws, 222 | buckets, 223 | duration, 224 | update_ratio, 225 | initial_size, 226 | range_size, 227 | 0) 228 | 229 | f_out.write(cmd_header + '\n') 230 | f_out.flush() 231 | 232 | results = [] 233 | for th_num in th_num_list: 234 | 235 | 236 | cmd = CMD_BASE[alg_type] + ' ' + CMD_PARAMS % ( 237 | rlu_max_ws, 238 | buckets, 239 | duration, 240 | update_ratio, 241 | initial_size, 242 | range_size, 243 | th_num) 244 | 245 | if (IS_NUMA): 246 | cmd = cmd_numa_prefix(th_num) + cmd 247 | 248 | print '-------------------------------' 249 | print '[%d] %s ' % (th_num, cmd) 250 | 251 | dict_keys = run_test(runs_per_test, alg_type, cmd) 252 | 253 | results.append(dict_keys) 254 | 255 | print_run_results(f_out, rlu_max_ws, update_ratio, th_num, dict_keys) 256 | 257 | 258 | f_out.write('\n\n') 259 | f_out.flush() 260 | f_out.close() 261 | 262 | 263 | print 'DONE: written output to %s' % (output_filename,) 264 | 265 | 266 | if '__main__' == __name__: 267 | try: 268 | 269 | param_num = 10 270 | 271 | prog_name, runs_per_test, rlu_max_ws, buckets, duration, alg_type, update_ratio, initial_size, range_size, output_filename = sys.argv[:param_num] 272 | th_num_list = sys.argv[param_num:] 273 | 274 | runs_per_test = int(runs_per_test) 275 | rlu_max_ws = int(rlu_max_ws) 276 | buckets = int(buckets) 277 | duration = int(duration) 278 | update_ratio = int(update_ratio) 279 | initial_size = int(initial_size) 280 | range_size = int(range_size) 281 | for i, th_num in enumerate(th_num_list): 282 | th_num_list[i] = int(th_num) 283 | 284 | except ValueError, e: 285 | raise e 286 | sys.exit('USAGE: %s [runs_per_test] [rlu_max_ws] [buckets] [duration] [alg_type] [update_ratio] [initial_size] [range_size] [output_filename] [[thread num list]]' % (sys.argv[0],)) 287 | 288 | 289 | execute(runs_per_test, rlu_max_ws, buckets, duration, alg_type, update_ratio, initial_size, range_size, output_filename, th_num_list) 290 | 291 | --------------------------------------------------------------------------------