├── .gitignore ├── CONTRIBUTORS ├── LICENSE ├── Makefile ├── README.md ├── TODO ├── rdtscp.h ├── tap.c ├── tap.h ├── uslab.c ├── uslab.h ├── uslab_bench.c └── uslab_test.c /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.a 3 | *.so 4 | .*.swp 5 | uslab_bench 6 | uslab_test 7 | jemalloc 8 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | Devon H. O'Dell 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS=-ggdb3 -O3 -Wall --std=gnu99 2 | CC=gcc 3 | AR=ar 4 | 5 | .PHONY: all 6 | all: static shared 7 | 8 | .PHONY: static 9 | static: 10 | $(CC) $(CFLAGS) -c uslab.c -o uslab.o 11 | ar rcs libuslab.a uslab.o 12 | rm uslab.o 13 | 14 | .PHONY: shared 15 | shared: 16 | $(CC) $(CFLAGS) -fPIC -c uslab.c -o uslab.o 17 | $(CC) -shared -o libuslab.so uslab.o 18 | rm uslab.o 19 | 20 | .PHONY: clean 21 | clean: 22 | rm -f libuslab.a libuslab.so uslab.o uslab_bench uslab_test 23 | 24 | uslab_bench: static 25 | $(CC) $(CFLAGS) uslab_bench.c -o uslab_bench -Ijemalloc/include -Ljemalloc/lib -L. -luslab -ljemalloc -lpthread -static 26 | 27 | uslab_test: static 28 | $(CC) $(CFLAGS) uslab_test.c tap.c -o uslab_test -L. -luslab -lpthread -static 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # uslab 2 | 3 | Uslab is a standalone, lock-free slab allocator library supporting both 4 | short-lived allocations as well as persistent storage. It is expected to shine 5 | as a bounded buffer of fixed-size objects. 6 | 7 | ## Design 8 | 9 | Two major properties of uslab allow it to be small, simple, and lock-free. 10 | First, freed memory is never actually returned to the operating system. 11 | Instead, anything explicitly freed becomes the head of a freelist (which is 12 | thus implemented as a stack). The freelist is maintained by reusing the areas 13 | we previously allocated to store pointers to additional items in the list. 14 | 15 | Some ancillary assumptions are made. Memory must start zeroed, and therefore 16 | must be either mapped with `MAP_ANONYMOUS` or from a file on a RAM-backed disk 17 | store. Because memory is zeroed, any item appearing on a freelist chain with 18 | a 0 value implies that the immediately adjacent "node" is also free. Because 19 | of this assumption, we do not use offsets (and consequently, if using the slab 20 | for persistent memory storage, the slab must be mapped at a fixed address). 21 | 22 | The slab is designed to be safe with many concurrently allocating threads with 23 | many concurrently freeing threads. Items must be freed only once per 24 | corresponding allocation. Double frees will result in a corrupted free stack, 25 | likely creating a loop in the stack that ends up resulting in undefined 26 | behavior. 27 | 28 | The slab is ABA-safe. It must be, because it is possible for pre-emption to 29 | pause a thread that has observed `slab->first_free->next_free`. During this 30 | paused period, another thread may actually become the owner of the object 31 | we're paused reading. If at least 1 free occurs before the winning thread 32 | frees the paused thread's `slab->first_free`, `slab->first_free->next_free` 33 | will no longer be consistent and the stack will be corrupted. 34 | 35 | ## Building 36 | 37 | Uslab has been tested on Linux and requires 38 | [Concurrency Kit](http://concurrencykit.org) to build. To build, run `make`. 39 | It is not designed to be a drop-in memory allocator replacement. 40 | 41 | ## API 42 | 43 | ### struct uslab_pt 44 | 45 | Consuming applications must define a pointer to a thread-local 46 | `struct uslab_pt` called `uslab_pt`. This is a per-thread region of the slab 47 | used to reduce contention. 48 | 49 | ```c 50 | __thread struct uslab_pt *uslab_pt 51 | ``` 52 | 53 | ### struct uslab 54 | 55 | Structure describing an slab and allocated at the head of the slab. Details 56 | of the structure are managed by the library. The code holding the reference to 57 | the slab must treat it as immutable. 58 | 59 | ### Creating an Arena 60 | 61 | ```c 62 | struct uslab *uslab_create_anonymous(void *base, size_t size_class, uint64_t nelem); 63 | struct uslab *uslab_create_heap(size_t size_class, uint64_t nelem); 64 | struct uslab *uslab_create_ramdisk(const char *path, void *base, size_t size_class, uint64_t nelem); 65 | ``` 66 | 67 | Three methods exist for creating an slab: 68 | 69 | * From an anonymous `mmap(2)` region, using `uslab_create_anonymous`. 70 | * From the heap (using `calloc(3)`), using `uslab_create_heap`. 71 | * From a sparse file on a memory disk, using `uslab_create_ramdisk`. 72 | 73 | ### Allocating and Freeing 74 | 75 | ```c 76 | void *uslab_alloc(struct uslab *); 77 | void uslab_free(struct uslab *, void *p); 78 | ``` 79 | 80 | To allocate, pass the handle from your `uslab_create_*` call. To free, pass 81 | the handle and the pointer received from `uslab_alloc`. Simple. 82 | 83 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | * Possibly embed ABA counter in first_free so we can do CAS instead of CAS2. 2 | * Investigate use of rings in conjunction with madvise(2) or fallocate(2) to 3 | allow releasing pages back to OS 4 | * Port to other OSes 5 | * Investigate performance changing division to bit shift when possible. 6 | * Investigate and improve performance on architectures other than x86_64. 7 | -------------------------------------------------------------------------------- /rdtscp.h: -------------------------------------------------------------------------------- 1 | #ifndef _RDTSCP_H_ 2 | #define _RDTSCP_H_ 3 | 4 | #include 5 | 6 | static inline uint64_t 7 | rdtscp(void) 8 | { 9 | uint32_t eax, edx; 10 | 11 | __asm__ __volatile__("rdtscp" 12 | : "=a" (eax), "=d" (edx) 13 | : 14 | : "%ecx", "memory"); 15 | 16 | return (((uint64_t)edx << 32) | eax); 17 | } 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /tap.c: -------------------------------------------------------------------------------- 1 | /*- @nolint 2 | * Copyright (c) 2004 Nik Clayton 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions 7 | * are met: 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 | * SUCH DAMAGE. 25 | * 26 | * Modifications (c) Fastly. 27 | */ 28 | 29 | /* for vasnprintf in stdio.h */ 30 | #define _GNU_SOURCE 31 | 32 | #include "tap.h" 33 | 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | #define ph_ignore_result(x) ((void)(x)) 43 | 44 | static int use_color = 0; 45 | static int no_plan = 0; 46 | static int skip_all = 0; 47 | static int have_plan = 0; 48 | static unsigned int test_count = 0; /* Number of tests that have been run */ 49 | static unsigned int e_tests = 0; /* Expected number of tests to run */ 50 | static unsigned int failures = 0; /* Number of tests that failed */ 51 | static char *todo_msg = NULL; 52 | static const char *todo_msg_fixed = "libtap malloc issue"; 53 | static int todo = 0; 54 | static int test_died = 0; 55 | static struct timeval start_time; 56 | 57 | #include 58 | static pthread_mutex_t M = PTHREAD_MUTEX_INITIALIZER; 59 | # define LOCK pthread_mutex_lock(&M); 60 | # define UNLOCK pthread_mutex_unlock(&M); 61 | 62 | static void _expected_tests(unsigned int); 63 | static void _tap_init(void); 64 | static void _cleanup(void); 65 | 66 | /* 67 | * Generate a test result. 68 | * 69 | * ok -- boolean, indicates whether or not the test passed. 70 | * test_name -- the name of the test, may be NULL 71 | * test_comment -- a comment to print afterwards, may be NULL 72 | */ 73 | unsigned int 74 | _gen_result(int ok, const char *func, const char *file, unsigned int line, 75 | const char *test_name, ...) 76 | { 77 | va_list ap; 78 | char *local_test_name = NULL; 79 | char *c; 80 | int name_is_digits; 81 | 82 | LOCK; 83 | 84 | test_count++; 85 | 86 | /* Start by taking the test name and performing any printf() 87 | expansions on it */ 88 | if(test_name != NULL) { 89 | va_start(ap, test_name); 90 | ph_ignore_result(vasprintf(&local_test_name, test_name, ap)); 91 | va_end(ap); 92 | 93 | /* Make sure the test name contains more than digits 94 | and spaces. Emit an error message and exit if it 95 | does */ 96 | if(local_test_name) { 97 | name_is_digits = 1; 98 | for(c = local_test_name; *c != '\0'; c++) { 99 | if(!isdigit(*c) && !isspace(*c)) { 100 | name_is_digits = 0; 101 | break; 102 | } 103 | } 104 | 105 | if(name_is_digits) { 106 | diag(" You named your test '%s'. You shouldn't use numbers for your test names.", local_test_name); 107 | diag(" Very confusing."); 108 | } 109 | } 110 | } 111 | 112 | if (use_color) { 113 | printf("\x1b[%dm", ok ? 32 : 31); 114 | } 115 | 116 | if(!ok) { 117 | printf("not "); 118 | failures++; 119 | } 120 | 121 | printf("ok %d", test_count); 122 | 123 | if(test_name != NULL) { 124 | printf(" - "); 125 | 126 | /* Print the test name, escaping any '#' characters it 127 | might contain */ 128 | if(local_test_name != NULL) { 129 | flockfile(stdout); 130 | for(c = local_test_name; *c != '\0'; c++) { 131 | if(*c == '#') 132 | fputc('\\', stdout); 133 | fputc((int)*c, stdout); 134 | } 135 | funlockfile(stdout); 136 | } else { /* vasprintf() failed, use a fixed message */ 137 | printf("%s", todo_msg_fixed); 138 | } 139 | } 140 | 141 | /* If we're in a todo_start() block then flag the test as being 142 | TODO. todo_msg should contain the message to print at this 143 | point. If it's NULL then asprintf() failed, and we should 144 | use the fixed message. 145 | 146 | This is not counted as a failure, so decrement the counter if 147 | the test failed. */ 148 | if(todo) { 149 | printf(" # TODO %s", todo_msg ? todo_msg : todo_msg_fixed); 150 | if(!ok) 151 | failures--; 152 | } 153 | 154 | if (use_color) { 155 | printf("\x1b[0m"); 156 | } 157 | printf("\n"); 158 | 159 | if(!ok) { 160 | if(getenv("HARNESS_ACTIVE") != NULL) 161 | fputs("\n", stderr); 162 | 163 | diag(" Failed %stest (%s:%s() at line %d)", 164 | todo ? "(TODO) " : "", file, func, line); 165 | } 166 | free(local_test_name); 167 | 168 | UNLOCK; 169 | 170 | /* We only care (when testing) that ok is positive, but here we 171 | specifically only want to return 1 or 0 */ 172 | return ok ? 1 : 0; 173 | } 174 | 175 | /* 176 | * Initialise the TAP library. Will only do so once, however many times it's 177 | * called. 178 | */ 179 | void 180 | _tap_init(void) 181 | { 182 | static int run_once = 0; 183 | 184 | if(!run_once) { 185 | atexit(_cleanup); 186 | run_once = 1; 187 | use_color = isatty(STDOUT_FILENO); 188 | gettimeofday(&start_time, NULL); 189 | } 190 | } 191 | 192 | /* 193 | * Note that there's no plan. 194 | */ 195 | int 196 | plan_no_plan(void) 197 | { 198 | 199 | LOCK; 200 | 201 | _tap_init(); 202 | 203 | if(have_plan != 0) { 204 | fprintf(stderr, "You tried to plan twice!\n"); 205 | test_died = 1; 206 | UNLOCK; 207 | exit(255); 208 | } 209 | 210 | have_plan = 1; 211 | no_plan = 1; 212 | 213 | UNLOCK; 214 | 215 | return 1; 216 | } 217 | 218 | /* 219 | * Note that the plan is to skip all tests 220 | */ 221 | int 222 | plan_skip_all(char *reason) 223 | { 224 | 225 | LOCK; 226 | 227 | _tap_init(); 228 | 229 | skip_all = 1; 230 | 231 | printf("1..0"); 232 | 233 | if(reason != NULL) 234 | printf(" # SKIP %s", reason); 235 | 236 | printf("\n"); 237 | 238 | UNLOCK; 239 | 240 | exit(0); 241 | } 242 | 243 | /* 244 | * Note the number of tests that will be run. 245 | */ 246 | int 247 | plan_tests(unsigned int tests) 248 | { 249 | 250 | LOCK; 251 | 252 | _tap_init(); 253 | 254 | if(have_plan != 0) { 255 | fprintf(stderr, "You tried to plan twice!\n"); 256 | test_died = 1; 257 | UNLOCK; 258 | exit(255); 259 | } 260 | 261 | if(tests == 0) { 262 | fprintf(stderr, "You said to run 0 tests! You've got to run something.\n"); 263 | test_died = 1; 264 | UNLOCK; 265 | exit(255); 266 | } 267 | 268 | have_plan = 1; 269 | 270 | _expected_tests(tests); 271 | 272 | UNLOCK; 273 | 274 | return e_tests; 275 | } 276 | 277 | unsigned int 278 | diag(const char *fmt, ...) 279 | { 280 | va_list ap; 281 | 282 | if (use_color) { 283 | fprintf(stderr, "\x1b[1;30m"); 284 | } 285 | fputs("# ", stderr); 286 | 287 | va_start(ap, fmt); 288 | vfprintf(stderr, fmt, ap); 289 | va_end(ap); 290 | 291 | if (use_color) { 292 | fprintf(stderr, "\x1b[0m"); 293 | } 294 | fputs("\n", stderr); 295 | 296 | return 0; 297 | } 298 | 299 | void 300 | _expected_tests(unsigned int tests) 301 | { 302 | 303 | printf("1..%d\n", tests); 304 | e_tests = tests; 305 | } 306 | 307 | int 308 | skip(unsigned int n, char *fmt, ...) 309 | { 310 | va_list ap; 311 | char *skip_msg; 312 | 313 | LOCK; 314 | 315 | va_start(ap, fmt); 316 | ph_ignore_result(asprintf(&skip_msg, fmt, ap)); 317 | va_end(ap); 318 | 319 | while(n-- > 0) { 320 | test_count++; 321 | printf("ok %d # skip %s\n", test_count, 322 | skip_msg != NULL ? 323 | skip_msg : "libtap():malloc() failed"); 324 | } 325 | 326 | free(skip_msg); 327 | 328 | UNLOCK; 329 | 330 | return 1; 331 | } 332 | 333 | void 334 | todo_start(char *fmt, ...) 335 | { 336 | va_list ap; 337 | 338 | LOCK; 339 | 340 | va_start(ap, fmt); 341 | ph_ignore_result(vasprintf(&todo_msg, fmt, ap)); 342 | va_end(ap); 343 | 344 | todo = 1; 345 | 346 | UNLOCK; 347 | } 348 | 349 | void 350 | todo_end(void) 351 | { 352 | 353 | LOCK; 354 | 355 | todo = 0; 356 | free(todo_msg); 357 | 358 | UNLOCK; 359 | } 360 | 361 | int 362 | exit_status(void) 363 | { 364 | int r; 365 | 366 | LOCK; 367 | 368 | /* If there's no plan, just return the number of failures */ 369 | if(no_plan || !have_plan) { 370 | UNLOCK; 371 | return failures; 372 | } 373 | 374 | /* Ran too many tests? Return the number of tests that were run 375 | that shouldn't have been */ 376 | if(e_tests < test_count) { 377 | r = test_count - e_tests; 378 | UNLOCK; 379 | return r; 380 | } 381 | 382 | /* Return the number of tests that failed + the number of tests 383 | that weren't run */ 384 | r = failures + e_tests - test_count; 385 | UNLOCK; 386 | 387 | return r; 388 | } 389 | 390 | static uint64_t timeval_to_ms(struct timeval tv) 391 | { 392 | return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); 393 | } 394 | 395 | /* 396 | * Cleanup at the end of the run, produce any final output that might be 397 | * required. 398 | */ 399 | void 400 | _cleanup(void) 401 | { 402 | struct timeval end_time; 403 | uint64_t ms; 404 | 405 | gettimeofday(&end_time, NULL); 406 | ms = timeval_to_ms(end_time) - timeval_to_ms(start_time); 407 | 408 | LOCK; 409 | 410 | /* If plan_no_plan() wasn't called, and we don't have a plan, 411 | and we're not skipping everything, then something happened 412 | before we could produce any output */ 413 | if(!no_plan && !have_plan && !skip_all) { 414 | diag("Looks like your test died before it could output anything."); 415 | UNLOCK; 416 | return; 417 | } 418 | 419 | if(test_died) { 420 | diag("Looks like your test died just after %d.", test_count); 421 | UNLOCK; 422 | return; 423 | } 424 | 425 | 426 | /* No plan provided, but now we know how many tests were run, and can 427 | print the header at the end */ 428 | if(!skip_all && (no_plan || !have_plan)) { 429 | printf("1..%d\n", test_count); 430 | } 431 | 432 | if((have_plan && !no_plan) && e_tests < test_count) { 433 | diag("Looks like you planned %d %s but ran %d extra.", 434 | e_tests, e_tests == 1 ? "test" : "tests", test_count - e_tests); 435 | ok(0, "plan_tests() doesn't match number of test results"); 436 | UNLOCK; 437 | return; 438 | } 439 | 440 | if((have_plan || !no_plan) && e_tests > test_count) { 441 | diag("Looks like you planned %d %s but only ran %d.", 442 | e_tests, e_tests == 1 ? "test" : "tests", test_count); 443 | ok(0, "plan_tests() doesn't match number of test results"); 444 | UNLOCK; 445 | return; 446 | } 447 | 448 | if(failures) 449 | diag("Looks like you failed %d %s of %d.", 450 | failures, failures == 1 ? "test" : "tests", test_count); 451 | 452 | diag("ELAPSED: %" PRIu64 "ms\n", ms); 453 | 454 | UNLOCK; 455 | } 456 | -------------------------------------------------------------------------------- /tap.h: -------------------------------------------------------------------------------- 1 | /*- @nolint 2 | * Copyright (c) 2004 Nik Clayton 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions 7 | * are met: 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 | * SUCH DAMAGE. 25 | */ 26 | 27 | #if __STDC_VERSION__ >= 199901L /* __GNUC__ */ 28 | # define ok(e, ...) ((e) ? \ 29 | _gen_result(1, __func__, __FILE__, __LINE__, \ 30 | __VA_ARGS__) : \ 31 | _gen_result(0, __func__, __FILE__, __LINE__, \ 32 | __VA_ARGS__)) 33 | 34 | # define ok1(e) ((e) ? \ 35 | _gen_result(1, __func__, __FILE__, __LINE__, "%s", #e) : \ 36 | _gen_result(0, __func__, __FILE__, __LINE__, "%s", #e)) 37 | 38 | # define pass(...) ok(1, __VA_ARGS__); 39 | # define fail(...) ok(0, __VA_ARGS__); 40 | 41 | # define skip_start(test, n, ...) \ 42 | do { \ 43 | if((test)) { \ 44 | skip(n, __VA_ARGS__); \ 45 | continue; \ 46 | } 47 | 48 | /* '## __VA_ARGS__' is a gcc'ism. C99 doesn't allow the token pasting 49 | and requires the caller to add the final comma if they've ommitted 50 | the optional arguments */ 51 | #elif defined(__GNUC__) 52 | # define ok(e, test, ...) ((e) ? \ 53 | _gen_result(1, __func__, __FILE__, __LINE__, \ 54 | test, ## __VA_ARGS__) : \ 55 | _gen_result(0, __func__, __FILE__, __LINE__, \ 56 | test, ## __VA_ARGS__)) 57 | 58 | # define ok1(e) ((e) ? \ 59 | _gen_result(1, __func__, __FILE__, __LINE__, "%s", #e) : \ 60 | _gen_result(0, __func__, __FILE__, __LINE__, "%s", #e)) 61 | 62 | # define pass(test, ...) ok(1, test, ## __VA_ARGS__); 63 | # define fail(test, ...) ok(0, test, ## __VA_ARGS__); 64 | 65 | # define skip_start(test, n, fmt, ...) \ 66 | do { \ 67 | if((test)) { \ 68 | skip(n, fmt, ## __VA_ARGS__); \ 69 | continue; \ 70 | } 71 | #else /* __STDC_VERSION__ */ 72 | # error "Needs gcc or C99 compiler for variadic macros." 73 | #endif /* __STDC_VERSION__ */ 74 | 75 | #define skip_end() } while(0); 76 | 77 | unsigned int _gen_result(int, const char *, const char *, unsigned int, const char *, ...) 78 | #ifdef __GNUC__ 79 | __attribute__((format(printf, 5, 6))) 80 | #endif 81 | ; 82 | 83 | int plan_no_plan(void); 84 | int plan_skip_all(char *); 85 | int plan_tests(unsigned int); 86 | 87 | unsigned int diag(const char *, ...) 88 | #ifdef __GNUC__ 89 | __attribute__((format(printf, 1, 2))) 90 | #endif 91 | ; 92 | 93 | int skip(unsigned int, char *, ...) 94 | #ifdef __GNUC__ 95 | __attribute__((format(printf, 2, 3))) 96 | #endif 97 | ; 98 | 99 | void todo_start(char *, ...) 100 | #ifdef __GNUC__ 101 | __attribute__((format(printf, 1, 2))) 102 | #endif 103 | ; 104 | void todo_end(void); 105 | 106 | int exit_status(void); 107 | 108 | #define is(a, b) ok(a == b, "%s == %s", #a, #b) 109 | #define isnt(a, b) ok(a != b, "%s != %s", #a, #b) 110 | #define is_int(a, b) ok(a == b, "%s %d == %s %d", #a, (int)a, #b, (int)b) 111 | #define is_true(what) ok(what, #what) 112 | #define is_string(a, b) ok(!strcmp(a, b), "%s == %s", a, b) 113 | 114 | 115 | -------------------------------------------------------------------------------- /uslab.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Fastly, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Concurrent, in-line slab allocator implementation, safe for workloads with 17 | * a single concurrent process freeing and multiple concurrent processes 18 | * allocating. 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include 35 | 36 | #include "uslab.h" 37 | 38 | struct uslab * 39 | uslab_create_heap(size_t size_class, uint64_t nelem, uint64_t npt_slabs) 40 | { 41 | char *cur_slab, *cur_base; 42 | struct uslab *a; 43 | uint64_t i; 44 | 45 | if (((size_class * nelem) / npt_slabs) == 0) { 46 | return NULL; 47 | } 48 | 49 | a = calloc(1, (2 * PAGE_SIZE) + (size_class * nelem)); 50 | if (a == NULL) { 51 | return NULL; 52 | } 53 | 54 | cur_slab = ((char *)a) + PAGE_SIZE; 55 | a->slab0_base = cur_base = ((char *)a) + (2 * PAGE_SIZE); 56 | 57 | a->pt_base = (struct uslab_pt *)cur_slab; 58 | a->pt_size = (size_class * nelem) / npt_slabs; 59 | a->pt_slabs = npt_slabs; 60 | a->size_class = size_class; 61 | a->slab_len = size_class * nelem; 62 | 63 | for (i = 0; i < npt_slabs; i++) { 64 | struct uslab_pt *pt; 65 | 66 | pt = (struct uslab_pt *)cur_slab; 67 | pt->base = pt->first_free = cur_base; 68 | pt->size = a->pt_size; 69 | pt->offset = i; 70 | 71 | cur_slab += sizeof (*pt); 72 | cur_base += a->pt_size; 73 | } 74 | 75 | return a; 76 | } 77 | 78 | struct uslab * 79 | uslab_create_anonymous(void *base, size_t size_class, uint64_t nelem, 80 | uint64_t npt_slabs) 81 | { 82 | int mflags = MAP_ANONYMOUS | MAP_PRIVATE; 83 | char *cur_slab, *cur_base; 84 | struct uslab *a; 85 | uint64_t i; 86 | void *map; 87 | 88 | if (((size_class * nelem) / npt_slabs) == 0) { 89 | return NULL; 90 | } 91 | 92 | if (base != NULL) { 93 | mflags |= MAP_FIXED; 94 | } 95 | 96 | map = mmap(base, (2 * PAGE_SIZE) + (nelem * size_class), 97 | PROT_READ | PROT_WRITE, mflags, -1, 0); 98 | if (map == MAP_FAILED) { 99 | perror("mmap"); 100 | return NULL; 101 | } 102 | 103 | a = map; 104 | cur_slab = ((char *)a) + PAGE_SIZE; 105 | a->slab0_base = cur_base = ((char *)a) + (2 * PAGE_SIZE); 106 | 107 | a->pt_base = (struct uslab_pt *)cur_slab; 108 | a->pt_size = (size_class * nelem) / npt_slabs; 109 | a->pt_slabs = npt_slabs; 110 | a->size_class = size_class; 111 | a->slab_len = size_class * nelem; 112 | 113 | for (i = 0; i < npt_slabs; i++) { 114 | struct uslab_pt *pt; 115 | 116 | pt = (struct uslab_pt *)cur_slab; 117 | pt->base = pt->first_free = cur_base; 118 | pt->size = a->pt_size; 119 | pt->offset = i; 120 | 121 | cur_slab += sizeof (*pt); 122 | cur_base += a->pt_size; 123 | } 124 | 125 | return a; 126 | } 127 | 128 | static void 129 | uslab_close_fd(int fd) 130 | { 131 | int r; 132 | 133 | do { 134 | r = close(fd); 135 | } while (r == -1 && errno == EINTR); 136 | } 137 | 138 | struct uslab * 139 | uslab_create_ramdisk(const char *path, void *base, size_t size_class, 140 | uint64_t nelem, uint64_t npt_slabs) 141 | { 142 | int fd, r, mflags = MAP_SHARED; 143 | char *cur_slab, *cur_base; 144 | struct uslab *a; 145 | struct stat sb; 146 | bool opened; 147 | uint64_t i; 148 | void *map; 149 | 150 | if (((size_class * nelem) / npt_slabs) == 0) { 151 | return NULL; 152 | } 153 | 154 | opened = false; 155 | r = stat(path, &sb); 156 | if (r == -1 && errno == ENOENT) { 157 | const char z = 0; 158 | ssize_t s; 159 | off_t o; 160 | int e; 161 | 162 | if ((fd = open(path, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR)) == -1) { 163 | return NULL; 164 | } 165 | 166 | o = lseek(fd, (2 * PAGE_SIZE) + (nelem * size_class) - 1, SEEK_SET); 167 | if (o == -1) { 168 | e = errno; 169 | uslab_close_fd(fd); 170 | errno = e; 171 | return NULL; 172 | } 173 | 174 | do { 175 | s = write(fd, &z, 1); 176 | } while (s == -1 && errno == EINTR); 177 | 178 | if (s == -1) { 179 | e = errno; 180 | uslab_close_fd(fd); 181 | errno = e; 182 | return NULL; 183 | } 184 | 185 | o = lseek(fd, 0, SEEK_SET); 186 | if (o == -1) { 187 | e = errno; 188 | uslab_close_fd(fd); 189 | errno = e; 190 | return NULL; 191 | } 192 | 193 | sb.st_size = (2 * PAGE_SIZE) + (nelem * size_class); 194 | } else { 195 | if ((fd = open(path, O_RDWR, S_IRUSR | S_IWUSR)) == -1) { 196 | return NULL; 197 | } 198 | 199 | opened = true; 200 | } 201 | 202 | if (base != NULL) { 203 | mflags |= MAP_FIXED; 204 | } 205 | 206 | map = mmap(base, sb.st_size, PROT_READ | PROT_WRITE, mflags, fd, 0); 207 | uslab_close_fd(fd); 208 | 209 | if (map == MAP_FAILED) { 210 | return NULL; 211 | } 212 | 213 | a = map; 214 | 215 | cur_slab = ((char *)a) + PAGE_SIZE; 216 | 217 | a->slab0_base = cur_base = ((char *)a) + (2 * PAGE_SIZE); 218 | a->pt_base = (struct uslab_pt *)cur_slab; 219 | a->pt_size = (size_class * nelem) / npt_slabs; 220 | a->pt_slabs = npt_slabs; 221 | a->size_class = size_class; 222 | a->slab_len = size_class * nelem; 223 | 224 | for (i = 0; i < npt_slabs; i++) { 225 | struct uslab_pt *pt; 226 | 227 | pt = (struct uslab_pt *)cur_slab; 228 | pt->base = cur_base; 229 | if (opened == false) { 230 | pt->first_free = cur_base; 231 | } 232 | pt->size = a->pt_size; 233 | pt->offset = i; 234 | 235 | cur_slab += sizeof (*pt); 236 | cur_base += a->pt_size; 237 | } 238 | 239 | return a; 240 | } 241 | 242 | void 243 | uslab_destroy_heap(struct uslab *a) 244 | { 245 | 246 | free(a); 247 | } 248 | 249 | void 250 | uslab_destroy_map(struct uslab *a) 251 | { 252 | 253 | munmap(a, a->slab_len); 254 | } 255 | 256 | /* 257 | * When we begin, our slab is sparse and zeroed. Effectively, this means that 258 | * we obtain our memory either with mmap(2) and MAP_ANONYMOUS, by using 259 | * shm_open(3), ftruncate(2), and mmap(2), or the mmap(2)-backed file comes 260 | * from a RAM-backed storage that initializes to 0 on access. 261 | * 262 | * Our approach is to find the first free block. We then figure out what the 263 | * next free block will be. If the next free block is NULL, we know that the 264 | * block immediately following the block we've chosen is the next logically 265 | * free block. 266 | * 267 | * We are prone to ABA. If we read first_free, load the next_free from it, and 268 | * are subsequently pre-empted, another concurrent process could allocate and 269 | * then free our target. Additional allocations may have occurred which alter 270 | * the target's next_free member by the time it was freed. In this case, we 271 | * would end up in an inconsistent state. We solve this problem by doing a 272 | * CAS2 on our slab to update both the free block and a generation counter. 273 | */ 274 | void * 275 | uslab_alloc(struct uslab *a) 276 | { 277 | struct uslab_pt update, original, *slab, *oa; 278 | struct uslab_entry *target; 279 | char *next_free; 280 | 281 | if (uslab_pt == NULL) { 282 | uslab_pt = &a->pt_base[ck_pr_faa_64(&a->pt_ctr, 1) % a->pt_slabs]; 283 | } 284 | 285 | slab = uslab_pt; 286 | 287 | retry: 288 | /* If we're out of space, try to steal some memory from elsewhere */ 289 | if (slab->first_free >= slab->base + slab->size) { 290 | uint64_t i = 1; 291 | 292 | oa = slab; 293 | slab = &a->pt_base[(oa->offset + i) % a->pt_slabs]; 294 | while (slab != oa && slab->first_free >= slab->base + slab->size) { 295 | slab = &a->pt_base[(oa->offset + i++) % a->pt_slabs]; 296 | } 297 | 298 | /* OOM. */ 299 | if (slab == oa) { 300 | return NULL; 301 | } 302 | 303 | goto retry; 304 | } 305 | 306 | original.generation = ck_pr_load_ptr(&slab->generation); 307 | ck_pr_fence_load(); 308 | original.first_free = ck_pr_load_ptr(&slab->first_free); 309 | target = (struct uslab_entry *)original.first_free; 310 | ck_pr_fence_load(); 311 | 312 | if (target->next_free == 0) { 313 | /* 314 | * When this is the last block, this will put an address 315 | * outside the bounds of the slab into the first_free member. 316 | * If we succeed, no other threads could win the bad value as 317 | * first_free is ABA protected and checked to be within bounds. 318 | */ 319 | next_free = original.first_free + a->size_class; 320 | } else { 321 | next_free = target->next_free; 322 | } 323 | 324 | update.generation = original.generation + 1; 325 | update.first_free = next_free; 326 | 327 | while (ck_pr_cas_ptr_2_value(slab, &original, &update, &original) == false) { 328 | /* 329 | * We failed to get the optimistic allocation, and our new 330 | * first_free block is outside the bounds of this slab. 331 | * Revert to trying to steal one from elsewhere. 332 | */ 333 | if (slab->first_free >= slab->base + slab->size) { 334 | slab = &a->pt_base[(slab->offset + 1) % a->pt_slabs]; 335 | goto retry; 336 | } 337 | 338 | update.generation = original.generation + 1; 339 | target = (struct uslab_entry *)original.first_free; 340 | ck_pr_fence_load(); 341 | if (target->next_free == 0) { 342 | next_free = original.first_free + a->size_class; 343 | } else { 344 | next_free = target->next_free; 345 | } 346 | 347 | update.first_free = next_free; 348 | } 349 | ck_pr_add_64(&slab->used, a->size_class); 350 | 351 | return target; 352 | } 353 | 354 | /* 355 | * An slab free routine that is safe with one or more concurrent unique 356 | * freeing processes in the face of many concurrent allocating processes. We 357 | * don't need any CAS2 voodoo here because we do not rely on the value of 358 | * next_free for the entry we are attempting to replace at the head of our 359 | * stack. Additionally, it is impossible for us to observe the same value 360 | * at the time we read target and the time we try to write to it because 361 | * no other concurrent processes know about target. 362 | */ 363 | void 364 | uslab_free(struct uslab *a, void *p) 365 | { 366 | struct uslab_pt *allocated_slab; 367 | struct uslab_entry *e; 368 | char *target; 369 | 370 | /* Stupid. */ 371 | if (p == NULL) return; 372 | 373 | /* 374 | * We want to free these into the same section of the pool from which 375 | * they were allocated. 376 | */ 377 | allocated_slab = &a->pt_base[(((char *)p) - a->slab0_base) / a->pt_size]; 378 | 379 | do { 380 | e = p; 381 | target = ck_pr_load_ptr(&allocated_slab->first_free); 382 | e->next_free = target; 383 | ck_pr_fence_store(); 384 | } while (ck_pr_cas_ptr(&allocated_slab->first_free, target, e) == false); 385 | 386 | ck_pr_sub_64(&allocated_slab->used, a->size_class); 387 | } 388 | -------------------------------------------------------------------------------- /uslab.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Fastly, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | #ifndef _USLAB_H_ 19 | #define _USLAB_H_ 20 | 21 | #include 22 | #include 23 | 24 | struct uslab_pt { 25 | /* 26 | * first_free and generation *must* be contiguous so that CAS2 can 27 | * update both to avoid ABA conflicts on concurrent allocations. 28 | */ 29 | char *first_free; 30 | char *generation; 31 | size_t size; 32 | size_t used; 33 | size_t offset; 34 | 35 | char *base; 36 | /* 37 | * Keep this cacheline-sized, otherwise false sharing will kill 38 | * throughput in threads in adjacent uslabs. 39 | */ 40 | char pad[64 - 48]; 41 | }; 42 | 43 | extern __thread struct uslab_pt *uslab_pt; 44 | 45 | struct uslab_entry { 46 | char *next_free; 47 | }; 48 | 49 | struct uslab { 50 | struct uslab_pt *pt_base; 51 | char *slab0_base; 52 | 53 | uint64_t size_class; 54 | size_t slab_len; 55 | uint64_t pt_slabs; 56 | size_t pt_size; 57 | uint64_t pt_ctr; 58 | }; 59 | 60 | struct uslab *uslab_create_anonymous(void *base, size_t size_class, uint64_t nelem, uint64_t npt_slabs); 61 | struct uslab *uslab_create_heap(size_t size_class, uint64_t nelem, uint64_t npt_slabs); 62 | struct uslab *uslab_create_ramdisk(const char *path, void *base, size_t size_class, uint64_t nelem, uint64_t npt_slabs); 63 | 64 | void *uslab_alloc(struct uslab *); 65 | void uslab_free(struct uslab *, void *p); 66 | 67 | void uslab_destroy_heap(struct uslab *); 68 | void uslab_destroy_map(struct uslab *); 69 | 70 | #endif 71 | -------------------------------------------------------------------------------- /uslab_bench.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Fastly, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Benchmarks for uslab allocation. Records throughput and average latency 17 | * per-thread from 1..N threads for a stoachastic workload of M operations. 18 | */ 19 | 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #include "jemalloc/jemalloc.h" 33 | #include "uslab.h" 34 | #include "rdtscp.h" 35 | 36 | struct td_state { 37 | pthread_t pt; 38 | 39 | uint64_t n_ops; 40 | uint64_t tid; 41 | 42 | struct uslab *slab; 43 | void **ptrs; 44 | uint64_t top; 45 | 46 | uint64_t n_allocs_completed; 47 | uint64_t n_frees_completed; 48 | uint64_t tdelta; 49 | }; 50 | 51 | struct td_state *state; 52 | __thread struct uslab_pt *uslab_pt = NULL; 53 | 54 | void * 55 | bench_td_jemalloc(void *arg) 56 | { 57 | struct td_state *a; 58 | uint64_t st, et; 59 | 60 | a = arg; 61 | 62 | st = rdtscp(); 63 | for (uint64_t i = 0; i < a->n_ops; i++) { 64 | a->ptrs[i] = je_malloc(sizeof (void *)); 65 | a->n_allocs_completed++; 66 | } 67 | 68 | for (uint64_t i = 0; i < a->n_ops; i++) { 69 | je_free(a->ptrs[i]); 70 | a->n_frees_completed++; 71 | } 72 | et = rdtscp(); 73 | 74 | a->tdelta = et - st; 75 | 76 | return NULL; 77 | } 78 | void * 79 | bench_td_malloc(void *arg) 80 | { 81 | struct td_state *a; 82 | uint64_t st, et; 83 | 84 | a = arg; 85 | 86 | st = rdtscp(); 87 | for (uint64_t i = 0; i < a->n_ops; i++) { 88 | a->ptrs[i] = malloc(sizeof (void *)); 89 | a->n_allocs_completed++; 90 | } 91 | 92 | for (uint64_t i = 0; i < a->n_ops; i++) { 93 | free(a->ptrs[i]); 94 | a->n_frees_completed++; 95 | } 96 | et = rdtscp(); 97 | 98 | a->tdelta = et - st; 99 | 100 | return NULL; 101 | } 102 | 103 | void * 104 | bench_td_uslab(void *arg) 105 | { 106 | struct td_state *a; 107 | uint64_t st, et; 108 | 109 | a = arg; 110 | 111 | st = rdtscp(); 112 | for (uint64_t i = 0; i < a->n_ops; i++) { 113 | a->ptrs[i] = uslab_alloc(a->slab); 114 | a->n_allocs_completed++; 115 | } 116 | 117 | for (uint64_t i = 0; i < a->n_ops; i++) { 118 | uslab_free(a->slab, a->ptrs[i]); 119 | a->n_frees_completed++; 120 | } 121 | et = rdtscp(); 122 | 123 | a->tdelta = et - st; 124 | 125 | return NULL; 126 | } 127 | 128 | void 129 | usage(void) 130 | { 131 | 132 | fprintf(stderr, "uslab_bench -t N -n N\n" 133 | "\t-a N:\tNumber of slabs to use\n" 134 | "\t-n N:\tNumber of operations to complete per thread\n" 135 | "\t-t N:\tNumber of threads to test up to\n"); 136 | exit(EX_USAGE); 137 | } 138 | 139 | int 140 | main(int argc, char **argv) 141 | { 142 | unsigned long n_tds, n_ops, n_slabs; 143 | struct uslab *slab; 144 | uint64_t td_total; 145 | int opt; 146 | 147 | n_slabs = n_tds = 2; 148 | n_ops = 10 * 1000 * 1000; 149 | 150 | while ((opt = getopt(argc, argv, "a:n:t:")) != -1) { 151 | switch (opt) { 152 | case 'a': 153 | errno = 0; 154 | n_slabs = strtoul(optarg, NULL, 0); 155 | if (errno != 0) { 156 | usage(); 157 | } 158 | break; 159 | case 'n': 160 | errno = 0; 161 | n_ops = strtoul(optarg, NULL, 0); 162 | if (errno != 0) { 163 | usage(); 164 | } 165 | break; 166 | case 't': 167 | errno = 0; 168 | n_tds = strtoul(optarg, NULL, 0); 169 | if (errno != 0) { 170 | usage(); 171 | } 172 | break; 173 | default: 174 | usage(); 175 | break; 176 | } 177 | } 178 | 179 | n_slabs = MIN(n_slabs, n_tds); 180 | td_total = 0; 181 | 182 | state = calloc(n_tds, sizeof (*state)); 183 | //slab = uslab_create_anonymous(NULL, sizeof (void *), n_ops * n_tds, n_slabs); 184 | slab = uslab_create_heap(sizeof (void *), n_ops * n_tds, n_slabs); 185 | 186 | for (unsigned long i = 0; i < n_tds; i++) { 187 | state[i].n_ops = n_ops; 188 | state[i].tid = i; 189 | state[i].ptrs = calloc(n_ops, sizeof (void *)); 190 | state[i].slab = slab; 191 | } 192 | 193 | for (unsigned long i = 0; i < n_tds; i++) { 194 | pthread_create(&state[i].pt, NULL, bench_td_uslab, &state[i]); 195 | } 196 | 197 | for (unsigned long i = 0; i < n_tds; i++) { 198 | pthread_join(state[i].pt, NULL); 199 | } 200 | 201 | 202 | for (unsigned long i = 0; i < n_tds; i++) { 203 | fprintf(stderr, "Thread %lu:\n" 204 | "\tn_allocs: %" PRIu64 "\n" 205 | "\tn_frees: %" PRIu64 "\n" 206 | "\tcycles: %" PRIu64 "\n", 207 | i, state[i].n_allocs_completed, 208 | state[i].n_frees_completed, state[i].tdelta); 209 | td_total += state[i].tdelta; 210 | state[i].n_allocs_completed = state[i].n_frees_completed = state[i].tdelta = 0; 211 | } 212 | fprintf(stderr, "td_total: %" PRIu64 "\n\n", td_total); 213 | td_total = 0; 214 | 215 | uslab_destroy_heap(slab); 216 | 217 | for (unsigned long i = 0; i < n_tds; i++) { 218 | pthread_create(&state[i].pt, NULL, bench_td_malloc, &state[i]); 219 | } 220 | 221 | for (unsigned long i = 0; i < n_tds; i++) { 222 | pthread_join(state[i].pt, NULL); 223 | } 224 | 225 | 226 | for (unsigned long i = 0; i < n_tds; i++) { 227 | fprintf(stderr, "Thread %lu:\n" 228 | "\tn_allocs: %" PRIu64 "\n" 229 | "\tn_frees: %" PRIu64 "\n" 230 | "\tcycles: %" PRIu64 "\n", 231 | i, state[i].n_allocs_completed, 232 | state[i].n_frees_completed, state[i].tdelta); 233 | td_total += state[i].tdelta; 234 | state[i].n_allocs_completed = state[i].n_frees_completed = state[i].tdelta = 0; 235 | } 236 | fprintf(stderr, "td_total: %" PRIu64 "\n\n", td_total); 237 | td_total = 0; 238 | 239 | for (unsigned long i = 0; i < n_tds; i++) { 240 | pthread_create(&state[i].pt, NULL, bench_td_jemalloc, &state[i]); 241 | } 242 | 243 | for (unsigned long i = 0; i < n_tds; i++) { 244 | pthread_join(state[i].pt, NULL); 245 | } 246 | 247 | 248 | for (unsigned long i = 0; i < n_tds; i++) { 249 | fprintf(stderr, "Thread %lu:\n" 250 | "\tn_allocs: %" PRIu64 "\n" 251 | "\tn_frees: %" PRIu64 "\n" 252 | "\tcycles: %" PRIu64 "\n", 253 | i, state[i].n_allocs_completed, 254 | state[i].n_frees_completed, state[i].tdelta); 255 | td_total += state[i].tdelta; 256 | } 257 | fprintf(stderr, "td_total: %" PRIu64 "\n\n", td_total); 258 | 259 | return EX_OK; 260 | } 261 | -------------------------------------------------------------------------------- /uslab_test.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Fastly, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include "uslab.h" 29 | #include "tap.h" 30 | 31 | __thread struct uslab_pt *uslab_pt = NULL; 32 | 33 | int 34 | main(void) 35 | { 36 | struct stat sb; 37 | int rv; 38 | 39 | plan_no_plan(); 40 | 41 | if ((rv = stat("tmp", &sb)) == -1) { 42 | if (errno != ENOENT) { 43 | perror("stat"); 44 | return -1; 45 | } 46 | } else { 47 | if (!S_ISDIR(sb.st_mode)) { 48 | fprintf(stderr, "tmp not a directory"); 49 | return -1; 50 | } else { 51 | umount("tmp"); 52 | if (rmdir("tmp") == -1) { 53 | perror("rmdir"); 54 | return -1; 55 | } 56 | } 57 | } 58 | 59 | mkdir("tmp", S_IRUSR | S_IWUSR | S_IXUSR); 60 | rv = mount(NULL, "tmp", "tmpfs", MS_NODEV | MS_NOATIME | MS_NODIRATIME | 61 | MS_NOSUID | MS_NOEXEC, "size=1g"); 62 | if (rv == -1) { 63 | perror("mount"); 64 | return -1; 65 | } 66 | 67 | 68 | /* Test that opening a ramdisk persists data between unmaps */ 69 | { 70 | char *base = (char *)0x4f000000; 71 | uintptr_t *p, *q; 72 | struct uslab *a; 73 | 74 | unlink("tmp/8"); 75 | 76 | a = uslab_create_ramdisk("tmp/8", base, 8, 1, 1); 77 | isnt(a, NULL); 78 | is((char *)a, base); 79 | 80 | p = uslab_alloc(a); 81 | q = (uintptr_t *)0x4f002000; 82 | is(p, q); 83 | *p = (uintptr_t)p; 84 | is(*p, *q); 85 | 86 | uslab_destroy_map(a); 87 | 88 | a = uslab_create_ramdisk("tmp/8", base, 8, 1, 1); 89 | isnt(a, NULL); 90 | is((char *)a, base); 91 | 92 | q = uslab_alloc(a); 93 | is(q, NULL); 94 | 95 | is(*p, (uintptr_t)p); 96 | 97 | uslab_destroy_map(a); 98 | unlink("tmp/8"); 99 | } 100 | 101 | /* Test ramdisk-backed sparse behavior */ 102 | { 103 | char *base = (char *)0x5f000000; 104 | uintptr_t *p, *q, *r; 105 | struct uslab *a; 106 | 107 | unlink("tmp/8"); 108 | 109 | /* Otherwise we keep remembering our old crap */ 110 | uslab_pt = NULL; 111 | a = uslab_create_ramdisk("tmp/8", base, 8, 1024UL*1024UL*1024UL*1024UL, 1); 112 | isnt(a, NULL); 113 | is((char *)a, base); 114 | 115 | p = uslab_alloc(a); 116 | q = (uintptr_t *)0x5f002000; 117 | is(p, q); 118 | *p = (uintptr_t)p; 119 | is(*p, *q); 120 | 121 | uslab_destroy_map(a); 122 | 123 | a = uslab_create_ramdisk("tmp/8", base, 8, 1024UL*1024UL*1024UL*1024UL, 1); 124 | isnt(a, NULL); 125 | 126 | r = q; 127 | q = uslab_alloc(a); 128 | is(q, r + 1); 129 | 130 | is(*p, (uintptr_t)p); 131 | 132 | uslab_destroy_map(a); 133 | unlink("tmp/8"); 134 | } 135 | 136 | /* 137 | * Test that allocation fails when we have nothing else to allocate, 138 | * and succeeds when we free. 139 | */ 140 | { 141 | struct uslab *a; 142 | void *p, *q; 143 | 144 | uslab_pt = NULL; 145 | a = uslab_create_heap(8, 1, 1); 146 | isnt(a, NULL); 147 | 148 | q = p = uslab_alloc(a); 149 | isnt(p, NULL); 150 | 151 | p = uslab_alloc(a); 152 | is(p, NULL); 153 | 154 | p = uslab_alloc(a); 155 | is(p, NULL); 156 | 157 | uslab_free(a, q); 158 | 159 | p = uslab_alloc(a); 160 | isnt(p, NULL); 161 | 162 | uslab_destroy_heap(a); 163 | } 164 | 165 | /* Test that we can "steal" from other arenas when we're out of mem */ 166 | { 167 | struct uslab *a; 168 | void *p; 169 | 170 | uslab_pt = NULL; 171 | a = uslab_create_heap(8, 2, 2); 172 | isnt(a, NULL); 173 | 174 | p = uslab_alloc(a); 175 | isnt(p, NULL); 176 | p = uslab_alloc(a); 177 | isnt(p, NULL); 178 | p = uslab_alloc(a); 179 | is(p, NULL); 180 | 181 | uslab_destroy_heap(a); 182 | } 183 | 184 | return 0; 185 | } 186 | --------------------------------------------------------------------------------