├── tests ├── .gitignore ├── Makefile └── test.c ├── README.md ├── LICENSE ├── srpmalloc.h └── srpmalloc.c /tests/.gitignore: -------------------------------------------------------------------------------- 1 | test 2 | *.exe 3 | *.o 4 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | CFLAGS=-Wall -Wextra -DENABLE_ASSERTS -O2 3 | test: 4 | $(CC) $(CFLAGS) test.c ../srpmalloc.c -o test 5 | ./test 6 | .PHONY: test 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # srpmalloc - Small rpmalloc 2 | 3 | This is a fork of [rpmalloc](https://github.com/mjansson/rpmalloc), with the intent to be 4 | used in single threaded applications only, with old C99 compilers, and in old OS. 5 | 6 | Summary of the changes: 7 | 8 | * Works with single thread applications only 9 | * Works with C99 compilers (C11 is not a required anymore) 10 | * Remove thread safety 11 | * Remove atomics usage 12 | * Remove use of thread locals 13 | * Remove statistics APIs 14 | * Remove first class heaps APIs 15 | * Remove APIs validations 16 | * Remove huge page support 17 | * Remove global cache support 18 | * Remove malloc override support 19 | 20 | By removing all this, it's much smaller, and works in old C compilers and Linux 21 | distributions, it was confirmed to work for example in Debian 4 with GCC 4.1, 22 | the original rpmalloc does not support many old Debian distributions due to use 23 | of C11 atomics. 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | 26 | 27 | You can also use this software under the MIT license if public domain 28 | is not recognized in your country 29 | 30 | 31 | The MIT License (MIT) 32 | 33 | Copyright (c) 2021 Eduardo Bart (https://github.com/edubart) 34 | Copyright (c) 2017 Mattias Jansson 35 | 36 | Permission is hereby granted, free of charge, to any person obtaining 37 | a copy of this software and associated documentation files (the 38 | "Software"), to deal in the Software without restriction, including 39 | without limitation the rights to use, copy, modify, merge, publish, 40 | distribute, sublicense, and/or sell copies of the Software, and to 41 | permit persons to whom the Software is furnished to do so, subject to 42 | the following conditions: 43 | 44 | The above copyright notice and this permission notice shall be 45 | included in all copies or substantial portions of the Software. 46 | 47 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 48 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 49 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 50 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 51 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 52 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 53 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 54 | -------------------------------------------------------------------------------- /srpmalloc.h: -------------------------------------------------------------------------------- 1 | /* srpmalloc.h - Small rpmalloc - Public Domain - 2021 Eduardo Bart (https://github.com/edubart) 2 | * rpmalloc.c - Memory allocator - Public Domain - 2016-2020 Mattias Jansson 3 | * This library is a fork of rpmalloc to be used with single thread applications 4 | * and with old C compilers. 5 | * 6 | * This library provides a cross-platform malloc implementation in C99. 7 | * The latest source code is always available at 8 | * 9 | * https://github.com/edubart/srpmalloc 10 | * 11 | * This library is put in the public domain; you can redistribute it and/or modify it without any restrictions. 12 | */ 13 | 14 | #ifndef RPMALLOC_H 15 | #define RPMALLOC_H 16 | 17 | #include 18 | 19 | #ifdef __cplusplus 20 | extern "C" { 21 | #endif 22 | 23 | #if (defined(__clang__) && __clang__ >= 4) || (defined(__GNUC__) && __GNUC__ >= 5) 24 | # define RPMALLOC_EXPORT extern __attribute__((visibility("default"))) 25 | # define RPMALLOC_ALLOCATOR 26 | # define RPMALLOC_ATTRIB_MALLOC __attribute__((__malloc__)) 27 | # define RPMALLOC_ATTRIB_ALLOC_SIZE(size) __attribute__((alloc_size(size))) 28 | # define RPMALLOC_ATTRIB_ALLOC_SIZE2(count, size) __attribute__((alloc_size(count, size))) 29 | #elif defined(_MSC_VER) 30 | # define RPMALLOC_EXPORT extern 31 | # define RPMALLOC_ALLOCATOR __declspec(allocator) __declspec(restrict) 32 | # define RPMALLOC_ATTRIB_MALLOC 33 | # define RPMALLOC_ATTRIB_ALLOC_SIZE(size) 34 | # define RPMALLOC_ATTRIB_ALLOC_SIZE2(count,size) 35 | #else 36 | # define RPMALLOC_EXPORT extern 37 | # define RPMALLOC_ALLOCATOR 38 | # define RPMALLOC_ATTRIB_MALLOC 39 | # define RPMALLOC_ATTRIB_ALLOC_SIZE(size) 40 | # define RPMALLOC_ATTRIB_ALLOC_SIZE2(count,size) 41 | #endif 42 | 43 | typedef enum rpmalloc_flags_t { 44 | //! Flag to rpaligned_realloc to not preserve content in reallocation 45 | RPMALLOC_NO_PRESERVE = 1, 46 | //! Flag to rpaligned_realloc to fail and return null pointer if grow cannot be done in-place, 47 | // in which case the original pointer is still valid (just like a call to realloc which failes to allocate 48 | // a new block). 49 | RPMALLOC_GROW_OR_FAIL = 2, 50 | } rpmalloc_flags_t; 51 | 52 | typedef struct rpmalloc_config_t { 53 | //! Map memory pages for the given number of bytes. The returned address MUST be 54 | // aligned to the rpmalloc span size, which will always be a power of two. 55 | // Optionally the function can store an alignment offset in the offset variable 56 | // in case it performs alignment and the returned pointer is offset from the 57 | // actual start of the memory region due to this alignment. The alignment offset 58 | // will be passed to the memory unmap function. The alignment offset MUST NOT be 59 | // larger than 65535 (storable in an uint16_t), if it is you must use natural 60 | // alignment to shift it into 16 bits. If you set a memory_map function, you 61 | // must also set a memory_unmap function or else the default implementation will 62 | // be used for both. 63 | void* (*memory_map)(size_t size, size_t* offset); 64 | //! Unmap the memory pages starting at address and spanning the given number of bytes. 65 | // If release is set to non-zero, the unmap is for an entire span range as returned by 66 | // a previous call to memory_map and that the entire range should be released. The 67 | // release argument holds the size of the entire span range. If release is set to 0, 68 | // the unmap is a partial decommit of a subset of the mapped memory range. 69 | // If you set a memory_unmap function, you must also set a memory_map function or 70 | // else the default implementation will be used for both. 71 | void (*memory_unmap)(void* address, size_t size, size_t offset, size_t release); 72 | //! Called when an assert fails, if asserts are enabled. Will use the standard assert() 73 | // if this is not set. 74 | void (*error_callback)(const char* message); 75 | //! Called when a call to map memory pages fails (out of memory). If this callback is 76 | // not set or returns zero the library will return a null pointer in the allocation 77 | // call. If this callback returns non-zero the map call will be retried. The argument 78 | // passed is the number of bytes that was requested in the map call. Only used if 79 | // the default system memory map function is used (memory_map callback is not set). 80 | int (*map_fail_callback)(size_t size); 81 | //! Number of spans to map at each request to map new virtual memory blocks. This can 82 | // be used to minimize the system call overhead at the cost of virtual memory address 83 | // space. The extra mapped pages will not be written until actually used, so physical 84 | // committed memory should not be affected in the default implementation. Will be 85 | // aligned to a multiple of spans that match memory page size in case of huge pages. 86 | size_t span_map_count; 87 | } rpmalloc_config_t; 88 | 89 | //! Initialize allocator with default configuration 90 | RPMALLOC_EXPORT int 91 | rpmalloc_initialize(void); 92 | 93 | //! Initialize allocator with given configuration 94 | RPMALLOC_EXPORT int 95 | rpmalloc_initialize_config(const rpmalloc_config_t* config); 96 | 97 | //! Get allocator configuration 98 | RPMALLOC_EXPORT const rpmalloc_config_t* 99 | rpmalloc_config(void); 100 | 101 | //! Finalize allocator 102 | RPMALLOC_EXPORT void 103 | rpmalloc_finalize(void); 104 | 105 | //! Initialize allocator for calling thread 106 | RPMALLOC_EXPORT void 107 | rpmalloc_thread_initialize(void); 108 | 109 | //! Finalize allocator for calling thread 110 | RPMALLOC_EXPORT void 111 | rpmalloc_thread_finalize(int release_caches); 112 | 113 | //! Query if allocator is initialized for calling thread 114 | RPMALLOC_EXPORT int 115 | rpmalloc_is_thread_initialized(void); 116 | 117 | //! Allocate a memory block of at least the given size 118 | RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 119 | rpmalloc(size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(1); 120 | 121 | //! Free the given memory block 122 | RPMALLOC_EXPORT void 123 | rpfree(void* ptr); 124 | 125 | //! Allocate a memory block of at least the given size and zero initialize it 126 | RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 127 | rpcalloc(size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(1, 2); 128 | 129 | //! Reallocate the given block to at least the given size 130 | RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 131 | rprealloc(void* ptr, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2); 132 | 133 | //! Reallocate the given block to at least the given size and alignment, 134 | // with optional control flags (see RPMALLOC_NO_PRESERVE). 135 | // Alignment must be a power of two and a multiple of sizeof(void*), 136 | // and should ideally be less than memory page size. A caveat of rpmalloc 137 | // internals is that this must also be strictly less than the span size (default 64KiB) 138 | RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 139 | rpaligned_realloc(void* ptr, size_t alignment, size_t size, size_t oldsize, unsigned int flags) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(3); 140 | 141 | //! Allocate a memory block of at least the given size and alignment. 142 | // Alignment must be a power of two and a multiple of sizeof(void*), 143 | // and should ideally be less than memory page size. A caveat of rpmalloc 144 | // internals is that this must also be strictly less than the span size (default 64KiB) 145 | RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 146 | rpaligned_alloc(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2); 147 | 148 | //! Allocate a memory block of at least the given size and alignment, and zero initialize it. 149 | // Alignment must be a power of two and a multiple of sizeof(void*), 150 | // and should ideally be less than memory page size. A caveat of rpmalloc 151 | // internals is that this must also be strictly less than the span size (default 64KiB) 152 | RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 153 | rpaligned_calloc(size_t alignment, size_t num, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE2(2, 3); 154 | 155 | //! Allocate a memory block of at least the given size and alignment. 156 | // Alignment must be a power of two and a multiple of sizeof(void*), 157 | // and should ideally be less than memory page size. A caveat of rpmalloc 158 | // internals is that this must also be strictly less than the span size (default 64KiB) 159 | RPMALLOC_EXPORT RPMALLOC_ALLOCATOR void* 160 | rpmemalign(size_t alignment, size_t size) RPMALLOC_ATTRIB_MALLOC RPMALLOC_ATTRIB_ALLOC_SIZE(2); 161 | 162 | //! Allocate a memory block of at least the given size and alignment. 163 | // Alignment must be a power of two and a multiple of sizeof(void*), 164 | // and should ideally be less than memory page size. A caveat of rpmalloc 165 | // internals is that this must also be strictly less than the span size (default 64KiB) 166 | RPMALLOC_EXPORT int 167 | rpposix_memalign(void** memptr, size_t alignment, size_t size); 168 | 169 | //! Query the usable size of the given memory block (from given pointer to the end of block) 170 | RPMALLOC_EXPORT size_t 171 | rpmalloc_usable_size(void* ptr); 172 | 173 | #ifdef __cplusplus 174 | } 175 | #endif 176 | 177 | #endif 178 | -------------------------------------------------------------------------------- /tests/test.c: -------------------------------------------------------------------------------- 1 | #if defined(_WIN32) && !defined(_CRT_SECURE_NO_WARNINGS) 2 | # define _CRT_SECURE_NO_WARNINGS 3 | #endif 4 | #ifdef __gnu_linux__ 5 | #define _GNU_SOURCE 6 | #endif 7 | 8 | #include "../srpmalloc.h" 9 | extern int 10 | test_run(int argc, char** argv); 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #define pointer_offset(ptr, ofs) (void*)((char*)(ptr) + (ptrdiff_t)(ofs)) 20 | #define pointer_diff(first, second) (ptrdiff_t)((const char*)(first) - (const char*)(second)) 21 | 22 | static int _test_failed; 23 | 24 | static int 25 | test_fail_cb(const char* reason, const char* file, int line) { 26 | fprintf(stderr, "FAIL: %s @ %s:%d\n", reason, file, line); 27 | fflush(stderr); 28 | _test_failed = 1; 29 | return -1; 30 | } 31 | 32 | #define test_fail(msg) test_fail_cb(msg, __FILE__, __LINE__) 33 | 34 | static int 35 | test_alloc(void) { 36 | unsigned int iloop = 0; 37 | unsigned int ipass = 0; 38 | unsigned int icheck = 0; 39 | unsigned int id = 0; 40 | void* addr[8142]; 41 | char data[20000]; 42 | unsigned int datasize[7] = { 473, 39, 195, 24, 73, 376, 245 }; 43 | size_t wanted_usable_size; 44 | 45 | rpmalloc_initialize(); 46 | 47 | //Query the small granularity 48 | void* zero_alloc = rpmalloc(0); 49 | size_t small_granularity = rpmalloc_usable_size(zero_alloc); 50 | rpfree(zero_alloc); 51 | 52 | for (id = 0; id < 20000; ++id) 53 | data[id] = (char)(id % 139 + id % 17); 54 | 55 | //Verify that blocks are 16 byte size aligned 56 | void* testptr = rpmalloc(16); 57 | if (rpmalloc_usable_size(testptr) != 16) 58 | return test_fail("Bad base alloc usable size"); 59 | rpfree(testptr); 60 | testptr = rpmalloc(32); 61 | if (rpmalloc_usable_size(testptr) != 32) 62 | return test_fail("Bad base alloc usable size"); 63 | rpfree(testptr); 64 | testptr = rpmalloc(128); 65 | if (rpmalloc_usable_size(testptr) != 128) 66 | return test_fail("Bad base alloc usable size"); 67 | rpfree(testptr); 68 | for (iloop = 0; iloop <= 1024; ++iloop) { 69 | testptr = rpmalloc(iloop); 70 | wanted_usable_size = iloop ? small_granularity * ((iloop + (small_granularity - 1)) / small_granularity) : small_granularity; 71 | if (rpmalloc_usable_size(testptr) != wanted_usable_size) { 72 | printf("For %u wanted %zu got %zu\n", iloop, wanted_usable_size, rpmalloc_usable_size(testptr)); 73 | return test_fail("Bad base alloc usable size"); 74 | } 75 | rpfree(testptr); 76 | } 77 | 78 | //Verify medium block sizes (until class merging kicks in) 79 | for (iloop = 1025; iloop <= 6000; ++iloop) { 80 | testptr = rpmalloc(iloop); 81 | wanted_usable_size = 512 * ((iloop / 512) + ((iloop % 512) ? 1 : 0)); 82 | if (rpmalloc_usable_size(testptr) != wanted_usable_size) 83 | return test_fail("Bad medium alloc usable size"); 84 | rpfree(testptr); 85 | } 86 | 87 | //Large reallocation test 88 | testptr = rpmalloc(253000); 89 | testptr = rprealloc(testptr, 151); 90 | wanted_usable_size = (small_granularity * ((151 + (small_granularity - 1)) / small_granularity)); 91 | if (rpmalloc_usable_size(testptr) != wanted_usable_size) 92 | return test_fail("Bad usable size"); 93 | if (rpmalloc_usable_size(pointer_offset(testptr, 16)) != (wanted_usable_size - 16)) 94 | return test_fail("Bad offset usable size"); 95 | rpfree(testptr); 96 | 97 | //Reallocation tests 98 | for (iloop = 1; iloop < 24; ++iloop) { 99 | size_t size = 37 * iloop; 100 | testptr = rpmalloc(size); 101 | *((uintptr_t*)testptr) = 0x12345678; 102 | wanted_usable_size = small_granularity * ((size / small_granularity) + ((size % small_granularity) ? 1 : 0)); 103 | if (rpmalloc_usable_size(testptr) != wanted_usable_size) 104 | return test_fail("Bad usable size (alloc)"); 105 | testptr = rprealloc(testptr, size + 16); 106 | if (rpmalloc_usable_size(testptr) < (wanted_usable_size + 16)) 107 | return test_fail("Bad usable size (realloc)"); 108 | if (*((uintptr_t*)testptr) != 0x12345678) 109 | return test_fail("Data not preserved on realloc"); 110 | rpfree(testptr); 111 | 112 | testptr = rpaligned_alloc(128, size); 113 | *((uintptr_t*)testptr) = 0x12345678; 114 | wanted_usable_size = small_granularity * ((size / small_granularity) + ((size % small_granularity) ? 1 : 0)); 115 | if (rpmalloc_usable_size(testptr) < wanted_usable_size) 116 | return test_fail("Bad usable size (aligned alloc)"); 117 | if (rpmalloc_usable_size(testptr) > (wanted_usable_size + 128)) 118 | return test_fail("Bad usable size (aligned alloc)"); 119 | testptr = rpaligned_realloc(testptr, 128, size + 32, 0, 0); 120 | if (rpmalloc_usable_size(testptr) < (wanted_usable_size + 32)) 121 | return test_fail("Bad usable size (aligned realloc)"); 122 | if (*((uintptr_t*)testptr) != 0x12345678) 123 | return test_fail("Data not preserved on realloc"); 124 | if (rpaligned_realloc(testptr, 128, size * 1024 * 4, 0, RPMALLOC_GROW_OR_FAIL)) 125 | return test_fail("Realloc with grow-or-fail did not fail as expected"); 126 | void* unaligned = rprealloc(testptr, size); 127 | if (unaligned != testptr) { 128 | ptrdiff_t diff = pointer_diff(testptr, unaligned); 129 | if (diff < 0) 130 | return test_fail("Bad realloc behaviour"); 131 | if (diff >= 128) 132 | return test_fail("Bad realloc behaviour"); 133 | } 134 | rpfree(testptr); 135 | } 136 | 137 | static size_t alignment[5] = { 0, 32, 64, 128, 256 }; 138 | for (iloop = 0; iloop < 5; ++iloop) { 139 | for (ipass = 0; ipass < 128 * 1024; ++ipass) { 140 | size_t this_alignment = alignment[iloop]; 141 | char* baseptr = rpaligned_alloc(this_alignment, ipass); 142 | if (this_alignment && ((uintptr_t)baseptr & (this_alignment - 1))) 143 | return test_fail("Alignment failed"); 144 | rpfree(baseptr); 145 | } 146 | } 147 | for (iloop = 0; iloop < 64; ++iloop) { 148 | for (ipass = 0; ipass < 8142; ++ipass) { 149 | size_t this_alignment = alignment[ipass % 5]; 150 | size_t size = iloop + ipass + datasize[(iloop + ipass) % 7]; 151 | char* baseptr = rpaligned_alloc(this_alignment, size); 152 | if (this_alignment && ((uintptr_t)baseptr & (this_alignment - 1))) 153 | return test_fail("Alignment failed"); 154 | for (size_t ibyte = 0; ibyte < size; ++ibyte) 155 | baseptr[ibyte] = (char)(ibyte & 0xFF); 156 | 157 | size_t resize = (iloop * ipass + datasize[(iloop + ipass) % 7]) & 0x2FF; 158 | size_t capsize = (size > resize ? resize : size); 159 | baseptr = rprealloc(baseptr, resize); 160 | for (size_t ibyte = 0; ibyte < capsize; ++ibyte) { 161 | if (baseptr[ibyte] != (char)(ibyte & 0xFF)) 162 | return test_fail("Data not preserved on realloc"); 163 | } 164 | 165 | size_t alignsize = (iloop * ipass + datasize[(iloop + ipass * 3) % 7]) & 0x2FF; 166 | this_alignment = alignment[(ipass + 1) % 5]; 167 | capsize = (capsize > alignsize ? alignsize : capsize); 168 | baseptr = rpaligned_realloc(baseptr, this_alignment, alignsize, resize, 0); 169 | for (size_t ibyte = 0; ibyte < capsize; ++ibyte) { 170 | if (baseptr[ibyte] != (char)(ibyte & 0xFF)) 171 | return test_fail("Data not preserved on realloc"); 172 | } 173 | 174 | rpfree(baseptr); 175 | } 176 | } 177 | 178 | for (iloop = 0; iloop < 64; ++iloop) { 179 | for (ipass = 0; ipass < 8142; ++ipass) { 180 | addr[ipass] = rpmalloc(500); 181 | if (addr[ipass] == 0) 182 | return test_fail("Allocation failed"); 183 | 184 | memcpy(addr[ipass], data + ipass, 500); 185 | 186 | for (icheck = 0; icheck < ipass; ++icheck) { 187 | if (addr[icheck] == addr[ipass]) 188 | return test_fail("Bad allocation result"); 189 | if (addr[icheck] < addr[ipass]) { 190 | if (pointer_offset(addr[icheck], 500) > addr[ipass]) 191 | return test_fail("Bad allocation result"); 192 | } 193 | else if (addr[icheck] > addr[ipass]) { 194 | if (pointer_offset(addr[ipass], 500) > addr[icheck]) 195 | return test_fail("Bad allocation result"); 196 | } 197 | } 198 | } 199 | 200 | for (ipass = 0; ipass < 8142; ++ipass) { 201 | if (memcmp(addr[ipass], data + ipass, 500)) 202 | return test_fail("Data corruption"); 203 | } 204 | 205 | for (ipass = 0; ipass < 8142; ++ipass) 206 | rpfree(addr[ipass]); 207 | } 208 | 209 | for (iloop = 0; iloop < 64; ++iloop) { 210 | for (ipass = 0; ipass < 1024; ++ipass) { 211 | unsigned int cursize = datasize[ipass%7] + ipass; 212 | 213 | addr[ipass] = rpmalloc(cursize); 214 | if (addr[ipass] == 0) 215 | return test_fail("Allocation failed"); 216 | 217 | memcpy(addr[ipass], data + ipass, cursize); 218 | 219 | for (icheck = 0; icheck < ipass; ++icheck) { 220 | if (addr[icheck] == addr[ipass]) 221 | return test_fail("Identical pointer returned from allocation"); 222 | if (addr[icheck] < addr[ipass]) { 223 | if (pointer_offset(addr[icheck], rpmalloc_usable_size(addr[icheck])) > addr[ipass]) 224 | return test_fail("Invalid pointer inside another block returned from allocation"); 225 | } 226 | else if (addr[icheck] > addr[ipass]) { 227 | if (pointer_offset(addr[ipass], rpmalloc_usable_size(addr[ipass])) > addr[icheck]) 228 | return test_fail("Invalid pointer inside another block returned from allocation"); 229 | } 230 | } 231 | } 232 | 233 | for (ipass = 0; ipass < 1024; ++ipass) { 234 | unsigned int cursize = datasize[ipass%7] + ipass; 235 | if (memcmp(addr[ipass], data + ipass, cursize)) 236 | return test_fail("Data corruption"); 237 | } 238 | 239 | for (ipass = 0; ipass < 1024; ++ipass) 240 | rpfree(addr[ipass]); 241 | } 242 | 243 | for (iloop = 0; iloop < 128; ++iloop) { 244 | for (ipass = 0; ipass < 1024; ++ipass) { 245 | addr[ipass] = rpmalloc(500); 246 | if (addr[ipass] == 0) 247 | return test_fail("Allocation failed"); 248 | 249 | memcpy(addr[ipass], data + ipass, 500); 250 | 251 | for (icheck = 0; icheck < ipass; ++icheck) { 252 | if (addr[icheck] == addr[ipass]) 253 | return test_fail("Identical pointer returned from allocation"); 254 | if (addr[icheck] < addr[ipass]) { 255 | if (pointer_offset(addr[icheck], 500) > addr[ipass]) 256 | return test_fail("Invalid pointer inside another block returned from allocation"); 257 | } 258 | else if (addr[icheck] > addr[ipass]) { 259 | if (pointer_offset(addr[ipass], 500) > addr[icheck]) 260 | return test_fail("Invalid pointer inside another block returned from allocation"); 261 | } 262 | } 263 | } 264 | 265 | for (ipass = 0; ipass < 1024; ++ipass) { 266 | if (memcmp(addr[ipass], data + ipass, 500)) 267 | return test_fail("Data corruption"); 268 | } 269 | 270 | for (ipass = 0; ipass < 1024; ++ipass) 271 | rpfree(addr[ipass]); 272 | } 273 | 274 | rpmalloc_finalize(); 275 | 276 | for (iloop = 0; iloop < 2048; iloop += 16) { 277 | rpmalloc_initialize(); 278 | addr[0] = rpmalloc(iloop); 279 | if (!addr[0]) 280 | return test_fail("Allocation failed"); 281 | rpfree(addr[0]); 282 | rpmalloc_finalize(); 283 | } 284 | 285 | for (iloop = 2048; iloop < (64 * 1024); iloop += 512) { 286 | rpmalloc_initialize(); 287 | addr[0] = rpmalloc(iloop); 288 | if (!addr[0]) 289 | return test_fail("Allocation failed"); 290 | rpfree(addr[0]); 291 | rpmalloc_finalize(); 292 | } 293 | 294 | for (iloop = (64 * 1024); iloop < (2 * 1024 * 1024); iloop += 4096) { 295 | rpmalloc_initialize(); 296 | addr[0] = rpmalloc(iloop); 297 | if (!addr[0]) 298 | return test_fail("Allocation failed"); 299 | rpfree(addr[0]); 300 | rpmalloc_finalize(); 301 | } 302 | 303 | rpmalloc_initialize(); 304 | for (iloop = 0; iloop < (2 * 1024 * 1024); iloop += 16) { 305 | addr[0] = rpmalloc(iloop); 306 | if (!addr[0]) 307 | return test_fail("Allocation failed"); 308 | rpfree(addr[0]); 309 | } 310 | rpmalloc_finalize(); 311 | 312 | printf("Memory allocation tests passed\n"); 313 | 314 | return 0; 315 | } 316 | 317 | static int 318 | test_realloc(void) { 319 | srand((unsigned int)time(0)); 320 | 321 | rpmalloc_initialize(); 322 | 323 | size_t pointer_count = 4096; 324 | void** pointers = rpmalloc(sizeof(void*) * pointer_count); 325 | memset(pointers, 0, sizeof(void*) * pointer_count); 326 | 327 | size_t alignments[5] = {0, 16, 32, 64, 128}; 328 | 329 | for (size_t iloop = 0; iloop < 8000; ++iloop) { 330 | for (size_t iptr = 0; iptr < pointer_count; ++iptr) { 331 | if (iloop) 332 | rpfree(rprealloc(pointers[iptr], (size_t)rand() % 4096)); 333 | pointers[iptr] = rpaligned_alloc(alignments[(iptr + iloop) % 5], iloop + iptr); 334 | } 335 | } 336 | 337 | for (size_t iptr = 0; iptr < pointer_count; ++iptr) 338 | rpfree(pointers[iptr]); 339 | rpfree(pointers); 340 | 341 | size_t bigsize = 1024 * 1024; 342 | void* bigptr = rpmalloc(bigsize); 343 | while (bigsize < 3000000) { 344 | ++bigsize; 345 | bigptr = rprealloc(bigptr, bigsize); 346 | if (rpaligned_realloc(bigptr, 0, bigsize * 32, 0, RPMALLOC_GROW_OR_FAIL)) 347 | return test_fail("Reallocation with grow-or-fail did not fail as expected"); 348 | if (rpaligned_realloc(bigptr, 128, bigsize * 32, 0, RPMALLOC_GROW_OR_FAIL)) 349 | return test_fail("Reallocation with aligned grow-or-fail did not fail as expected"); 350 | } 351 | rpfree(bigptr); 352 | 353 | rpmalloc_finalize(); 354 | 355 | printf("Memory reallocation tests passed\n"); 356 | 357 | return 0; 358 | } 359 | 360 | static int 361 | test_superalign(void) { 362 | 363 | rpmalloc_initialize(); 364 | 365 | size_t alignment[] = { 2048, 4096, 8192, 16384, 32768 }; 366 | size_t sizes[] = { 187, 1057, 2436, 5234, 9235, 17984, 35783, 72436 }; 367 | 368 | for (size_t ipass = 0; ipass < 8; ++ipass) { 369 | for (size_t iloop = 0; iloop < 4096; ++iloop) { 370 | for (size_t ialign = 0, asize = sizeof(alignment) / sizeof(alignment[0]); ialign < asize; ++ialign) { 371 | for (size_t isize = 0, ssize = sizeof(sizes) / sizeof(sizes[0]); isize < ssize; ++isize) { 372 | size_t alloc_size = sizes[isize] + iloop + ipass; 373 | uint8_t* ptr = rpaligned_alloc(alignment[ialign], alloc_size); 374 | if (!ptr || ((uintptr_t)ptr & (alignment[ialign] - 1))) 375 | return test_fail("Super alignment allocation failed"); 376 | ptr[0] = 1; 377 | ptr[alloc_size - 1] = 1; 378 | rpfree(ptr); 379 | } 380 | } 381 | } 382 | } 383 | 384 | rpmalloc_finalize(); 385 | 386 | printf("Memory super aligned tests passed\n"); 387 | 388 | return 0; 389 | } 390 | 391 | typedef struct _allocator_thread_arg { 392 | unsigned int loops; 393 | unsigned int passes; //max 4096 394 | unsigned int datasize[32]; 395 | unsigned int num_datasize; //max 32 396 | int init_fini_each_loop; 397 | void** pointers; 398 | void** crossthread_pointers; 399 | } allocator_thread_arg_t; 400 | 401 | static int got_error; 402 | 403 | static void test_error_callback(const char* message) { 404 | printf("%s\n", message); 405 | (void)sizeof(message); 406 | got_error = 1; 407 | } 408 | 409 | static int test_error(void) { 410 | //printf("Detecting memory leak\n"); 411 | 412 | rpmalloc_config_t config = {0}; 413 | config.error_callback = test_error_callback; 414 | rpmalloc_initialize_config(&config); 415 | 416 | rpmalloc(10); 417 | 418 | rpmalloc_finalize(); 419 | 420 | if (!got_error) { 421 | printf("Leak not detected and reported as expected\n"); 422 | return -1; 423 | } 424 | 425 | printf("Error detection test passed\n"); 426 | return 0; 427 | } 428 | 429 | int test_run(int argc, char** argv) { 430 | (void)sizeof(argc); 431 | (void)sizeof(argv); 432 | if (test_alloc()) 433 | return -1; 434 | if (test_realloc()) 435 | return -1; 436 | if (test_superalign()) 437 | return -1; 438 | if (test_error()) 439 | return -1; 440 | printf("All tests passed\n"); 441 | return 0; 442 | } 443 | 444 | int main(int argc, char** argv) { 445 | return test_run(argc, argv); 446 | } 447 | -------------------------------------------------------------------------------- /srpmalloc.c: -------------------------------------------------------------------------------- 1 | /* srpmalloc.h - Small rpmalloc - Public Domain - 2021 Eduardo Bart (https://github.com/edubart) 2 | * rpmalloc.c - Memory allocator - Public Domain - 2016-2020 Mattias Jansson 3 | * This library is a fork of rpmalloc to be used with single thread applications 4 | * and with old C compilers. 5 | * 6 | * This library provides a cross-platform malloc implementation in C99. 7 | * The latest source code is always available at 8 | * 9 | * https://github.com/edubart/srpmalloc 10 | * 11 | * This library is put in the public domain; you can redistribute it and/or modify it without any restrictions. 12 | */ 13 | 14 | #include "srpmalloc.h" 15 | 16 | //////////// 17 | /// 18 | /// Build time configurable limits 19 | /// 20 | ////// 21 | 22 | #ifndef HEAP_ARRAY_SIZE 23 | //! Size of heap hashmap 24 | #define HEAP_ARRAY_SIZE 47 25 | #endif 26 | #ifndef ENABLE_ASSERTS 27 | //! Enable asserts 28 | #define ENABLE_ASSERTS 0 29 | #endif 30 | #ifndef DEFAULT_SPAN_MAP_COUNT 31 | //! Default number of spans to map in call to map more virtual memory (default values yield 4MiB here) 32 | #define DEFAULT_SPAN_MAP_COUNT 64 33 | #endif 34 | #ifndef GLOBAL_CACHE_MULTIPLIER 35 | //! Multiplier for global cache 36 | #define GLOBAL_CACHE_MULTIPLIER 8 37 | #endif 38 | 39 | #if defined(_WIN32) || defined(__WIN32__) || defined(_WIN64) 40 | # define PLATFORM_WINDOWS 1 41 | # define PLATFORM_POSIX 0 42 | #else 43 | # define PLATFORM_WINDOWS 0 44 | # define PLATFORM_POSIX 1 45 | #endif 46 | 47 | /// Platform and arch specifics 48 | #if PLATFORM_WINDOWS 49 | # ifndef WIN32_LEAN_AND_MEAN 50 | # define WIN32_LEAN_AND_MEAN 51 | # endif 52 | # include 53 | #else 54 | # include 55 | # include 56 | # include 57 | # include 58 | # if defined(__APPLE__) 59 | # include 60 | # if !TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR 61 | # include 62 | # include 63 | # endif 64 | # endif 65 | #endif 66 | 67 | #include 68 | #include 69 | #include 70 | 71 | #if PLATFORM_POSIX 72 | # include 73 | # include 74 | # ifndef MAP_UNINITIALIZED 75 | # define MAP_UNINITIALIZED 0 76 | # endif 77 | #endif 78 | #include 79 | 80 | #if ENABLE_ASSERTS 81 | # undef NDEBUG 82 | # if defined(_MSC_VER) && !defined(_DEBUG) 83 | # define _DEBUG 84 | # endif 85 | # include 86 | #define RPMALLOC_TOSTRING_M(x) #x 87 | #define RPMALLOC_TOSTRING(x) RPMALLOC_TOSTRING_M(x) 88 | #define rpmalloc_assert(truth, message) \ 89 | do { \ 90 | if (!(truth)) { \ 91 | if (_memory_config.error_callback) { \ 92 | _memory_config.error_callback( \ 93 | message " (" RPMALLOC_TOSTRING(truth) ") at " __FILE__ ":" RPMALLOC_TOSTRING(__LINE__)); \ 94 | } else { \ 95 | assert((truth) && message); \ 96 | } \ 97 | } \ 98 | } while (0) 99 | #else 100 | # define rpmalloc_assert(truth, message) do {} while(0) 101 | #endif 102 | 103 | 104 | #if defined(__GNUC__) 105 | #define EXPECTED(x) __builtin_expect((x), 1) 106 | #define UNEXPECTED(x) __builtin_expect((x), 0) 107 | #else 108 | #define EXPECTED(x) (x) 109 | #define UNEXPECTED(x) (x) 110 | #endif 111 | 112 | 113 | /// 114 | /// Preconfigured limits and sizes 115 | /// 116 | 117 | //! Granularity of a small allocation block (must be power of two) 118 | #define SMALL_GRANULARITY 16 119 | //! Small granularity shift count 120 | #define SMALL_GRANULARITY_SHIFT 4 121 | //! Number of small block size classes 122 | #define SMALL_CLASS_COUNT 65 123 | //! Maximum size of a small block 124 | #define SMALL_SIZE_LIMIT (SMALL_GRANULARITY * (SMALL_CLASS_COUNT - 1)) 125 | //! Granularity of a medium allocation block 126 | #define MEDIUM_GRANULARITY 512 127 | //! Medium granularity shift count 128 | #define MEDIUM_GRANULARITY_SHIFT 9 129 | //! Number of medium block size classes 130 | #define MEDIUM_CLASS_COUNT 61 131 | //! Total number of small + medium size classes 132 | #define SIZE_CLASS_COUNT (SMALL_CLASS_COUNT + MEDIUM_CLASS_COUNT) 133 | //! Number of large block size classes 134 | #define LARGE_CLASS_COUNT 63 135 | //! Maximum size of a medium block 136 | #define MEDIUM_SIZE_LIMIT (SMALL_SIZE_LIMIT + (MEDIUM_GRANULARITY * MEDIUM_CLASS_COUNT)) 137 | //! Maximum size of a large block 138 | #define LARGE_SIZE_LIMIT ((LARGE_CLASS_COUNT * _memory_span_size) - SPAN_HEADER_SIZE) 139 | //! Size of a span header (must be a multiple of SMALL_GRANULARITY and a power of two) 140 | #define SPAN_HEADER_SIZE 128 141 | //! Number of spans in thread cache 142 | #define MAX_THREAD_SPAN_CACHE 400 143 | //! Number of spans to transfer between thread and global cache 144 | #define THREAD_SPAN_CACHE_TRANSFER 64 145 | //! Number of spans in thread cache for large spans (must be greater than LARGE_CLASS_COUNT / 2) 146 | #define MAX_THREAD_SPAN_LARGE_CACHE 100 147 | //! Number of spans to transfer between thread and global cache for large spans 148 | #define THREAD_SPAN_LARGE_CACHE_TRANSFER 6 149 | 150 | #define pointer_offset(ptr, ofs) (void*)((char*)(ptr) + (ptrdiff_t)(ofs)) 151 | #define pointer_diff(first, second) (ptrdiff_t)((const char*)(first) - (const char*)(second)) 152 | 153 | #define SIZE_CLASS_LARGE SIZE_CLASS_COUNT 154 | #define SIZE_CLASS_HUGE ((uint32_t)-1) 155 | 156 | //////////// 157 | /// 158 | /// Data types 159 | /// 160 | ////// 161 | 162 | //! A memory heap, per thread 163 | typedef struct heap_t heap_t; 164 | //! Span of memory pages 165 | typedef struct span_t span_t; 166 | //! Span list 167 | typedef struct span_list_t span_list_t; 168 | //! Span active data 169 | typedef struct span_active_t span_active_t; 170 | //! Size class definition 171 | typedef struct size_class_t size_class_t; 172 | //! Global cache 173 | typedef struct global_cache_t global_cache_t; 174 | 175 | //! Flag indicating span is the first (master) span of a split superspan 176 | #define SPAN_FLAG_MASTER 1U 177 | //! Flag indicating span is a secondary (sub) span of a split superspan 178 | #define SPAN_FLAG_SUBSPAN 2U 179 | //! Flag indicating span has blocks with increased alignment 180 | #define SPAN_FLAG_ALIGNED_BLOCKS 4U 181 | //! Flag indicating an unmapped master span 182 | #define SPAN_FLAG_UNMAPPED_MASTER 8U 183 | 184 | // A span can either represent a single span of memory pages with size declared by span_map_count configuration variable, 185 | // or a set of spans in a continuous region, a super span. Any reference to the term "span" usually refers to both a single 186 | // span or a super span. A super span can further be divided into multiple spans (or this, super spans), where the first 187 | // (super)span is the master and subsequent (super)spans are subspans. The master span keeps track of how many subspans 188 | // that are still alive and mapped in virtual memory, and once all subspans and master have been unmapped the entire 189 | // superspan region is released and unmapped (on Windows for example, the entire superspan range has to be released 190 | // in the same call to release the virtual memory range, but individual subranges can be decommitted individually 191 | // to reduce physical memory use). 192 | struct span_t { 193 | //! Free list 194 | void* free_list; 195 | //! Total block count of size class 196 | uint32_t block_count; 197 | //! Size class 198 | uint32_t size_class; 199 | //! Index of last block initialized in free list 200 | uint32_t free_list_limit; 201 | //! Number of used blocks remaining when in partial state 202 | uint32_t used_count; 203 | //! Deferred free list 204 | void* free_list_deferred; 205 | //! Size of deferred free list, or list of spans when part of a cache list 206 | uint32_t list_size; 207 | //! Size of a block 208 | uint32_t block_size; 209 | //! Flags and counters 210 | uint32_t flags; 211 | //! Number of spans 212 | uint32_t span_count; 213 | //! Total span counter for master spans 214 | uint32_t total_spans; 215 | //! Offset from master span for subspans 216 | uint32_t offset_from_master; 217 | //! Remaining span counter, for master spans 218 | int32_t remaining_spans; 219 | //! Alignment offset 220 | uint32_t align_offset; 221 | //! Owning heap 222 | heap_t* heap; 223 | //! Next span 224 | span_t* next; 225 | //! Previous span 226 | span_t* prev; 227 | }; 228 | 229 | struct span_cache_t { 230 | size_t count; 231 | span_t* span[MAX_THREAD_SPAN_CACHE]; 232 | }; 233 | typedef struct span_cache_t span_cache_t; 234 | 235 | struct span_large_cache_t { 236 | size_t count; 237 | span_t* span[MAX_THREAD_SPAN_LARGE_CACHE]; 238 | }; 239 | typedef struct span_large_cache_t span_large_cache_t; 240 | 241 | struct heap_size_class_t { 242 | //! Free list of active span 243 | void* free_list; 244 | //! Double linked list of partially used spans with free blocks. 245 | // Previous span pointer in head points to tail span of list. 246 | span_t* partial_span; 247 | //! Early level cache of fully free spans 248 | span_t* cache; 249 | }; 250 | typedef struct heap_size_class_t heap_size_class_t; 251 | 252 | // Control structure for a heap, either a thread heap or a first class heap if enabled 253 | struct heap_t { 254 | //! Owning thread ID 255 | uintptr_t owner_thread; 256 | //! Free lists for each size class 257 | heap_size_class_t size_class[SIZE_CLASS_COUNT]; 258 | //! Arrays of fully freed spans, single span 259 | span_cache_t span_cache; 260 | //! List of deferred free spans (single linked list) 261 | void* span_free_deferred; 262 | //! Number of full spans 263 | size_t full_span_count; 264 | //! Mapped but unused spans 265 | span_t* span_reserve; 266 | //! Master span for mapped but unused spans 267 | span_t* span_reserve_master; 268 | //! Number of mapped but unused spans 269 | uint32_t spans_reserved; 270 | //! Child count 271 | int32_t child_count; 272 | //! Next heap in id list 273 | heap_t* next_heap; 274 | //! Next heap in orphan list 275 | heap_t* next_orphan; 276 | //! Heap ID 277 | int32_t id; 278 | //! Finalization state flag 279 | int finalize; 280 | //! Master heap owning the memory pages 281 | heap_t* master_heap; 282 | //! Arrays of fully freed spans, large spans with > 1 span count 283 | span_large_cache_t span_large_cache[LARGE_CLASS_COUNT - 1]; 284 | }; 285 | 286 | // Size class for defining a block size bucket 287 | struct size_class_t { 288 | //! Size of blocks in this class 289 | uint32_t block_size; 290 | //! Number of blocks in each chunk 291 | uint16_t block_count; 292 | //! Class index this class is merged with 293 | uint16_t class_idx; 294 | }; 295 | 296 | struct global_cache_t { 297 | //! Cache lock 298 | int32_t lock; 299 | //! Cache count 300 | uint32_t count; 301 | //! Cached spans 302 | span_t* span[GLOBAL_CACHE_MULTIPLIER * MAX_THREAD_SPAN_CACHE]; 303 | //! Unlimited cache overflow 304 | span_t* overflow; 305 | }; 306 | 307 | //////////// 308 | /// 309 | /// Global data 310 | /// 311 | ////// 312 | 313 | //! Default span size (64KiB) 314 | #define _memory_default_span_size (64 * 1024) 315 | #define _memory_default_span_size_shift 16 316 | #define _memory_default_span_mask (~((uintptr_t)(_memory_span_size - 1))) 317 | 318 | //! Initialized flag 319 | static int _rpmalloc_initialized; 320 | //! Configuration 321 | static rpmalloc_config_t _memory_config; 322 | //! Memory page size 323 | static size_t _memory_page_size; 324 | //! Shift to divide by page size 325 | static size_t _memory_page_size_shift; 326 | //! Granularity at which memory pages are mapped by OS 327 | static size_t _memory_map_granularity; 328 | //! Hardwired span size 329 | #define _memory_span_size _memory_default_span_size 330 | #define _memory_span_size_shift _memory_default_span_size_shift 331 | #define _memory_span_mask _memory_default_span_mask 332 | //! Number of spans to map in each map call 333 | static size_t _memory_span_map_count; 334 | //! Number of spans to keep reserved in each heap 335 | static size_t _memory_heap_reserve_count; 336 | //! Global size classes 337 | static size_class_t _memory_size_class[SIZE_CLASS_COUNT]; 338 | //! Run-time size limit of medium blocks 339 | static size_t _memory_medium_size_limit; 340 | //! Heap ID counter 341 | static int32_t _memory_heap_id; 342 | //! Global reserved spans 343 | static span_t* _memory_global_reserve; 344 | //! Global reserved count 345 | static size_t _memory_global_reserve_count; 346 | //! Global reserved master 347 | static span_t* _memory_global_reserve_master; 348 | //! All heaps 349 | static heap_t* _memory_heaps[HEAP_ARRAY_SIZE]; 350 | //! Orphaned heaps 351 | static heap_t* _memory_orphan_heaps; 352 | 353 | //////////// 354 | /// 355 | /// Thread local heap and ID 356 | /// 357 | ////// 358 | 359 | //! Current thread heap 360 | static heap_t* _memory_thread_heap; 361 | 362 | //! Set the current thread heap 363 | static void 364 | set_thread_heap(heap_t* heap) { 365 | _memory_thread_heap = heap; 366 | if (heap) 367 | heap->owner_thread = 0; 368 | } 369 | 370 | //////////// 371 | /// 372 | /// Low level memory map/unmap 373 | /// 374 | ////// 375 | 376 | //! Map more virtual memory 377 | // size is number of bytes to map 378 | // offset receives the offset in bytes from start of mapped region 379 | // returns address to start of mapped region to use 380 | static void* 381 | _rpmalloc_mmap(size_t size, size_t* offset) { 382 | rpmalloc_assert(!(size % _memory_page_size), "Invalid mmap size"); 383 | rpmalloc_assert(size >= _memory_page_size, "Invalid mmap size"); 384 | return _memory_config.memory_map(size, offset); 385 | } 386 | 387 | //! Unmap virtual memory 388 | // address is the memory address to unmap, as returned from _memory_map 389 | // size is the number of bytes to unmap, which might be less than full region for a partial unmap 390 | // offset is the offset in bytes to the actual mapped region, as set by _memory_map 391 | // release is set to 0 for partial unmap, or size of entire range for a full unmap 392 | static void 393 | _rpmalloc_unmap(void* address, size_t size, size_t offset, size_t release) { 394 | rpmalloc_assert(!release || (release >= size), "Invalid unmap size"); 395 | rpmalloc_assert(!release || (release >= _memory_page_size), "Invalid unmap size"); 396 | if (release) { 397 | rpmalloc_assert(!(release % _memory_page_size), "Invalid unmap size"); 398 | } 399 | _memory_config.memory_unmap(address, size, offset, release); 400 | } 401 | 402 | //! Default implementation to map new pages to virtual memory 403 | static void* 404 | _rpmalloc_mmap_os(size_t size, size_t* offset) { 405 | //Either size is a heap (a single page) or a (multiple) span - we only need to align spans, and only if larger than map granularity 406 | size_t padding = ((size >= _memory_span_size) && (_memory_span_size > _memory_map_granularity)) ? _memory_span_size : 0; 407 | rpmalloc_assert(size >= _memory_page_size, "Invalid mmap size"); 408 | #if PLATFORM_WINDOWS 409 | //Ok to MEM_COMMIT - according to MSDN, "actual physical pages are not allocated unless/until the virtual addresses are actually accessed" 410 | void* ptr = VirtualAlloc(0, size + padding, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); 411 | if (!ptr) { 412 | if (_memory_config.map_fail_callback) { 413 | if (_memory_config.map_fail_callback(size + padding)) 414 | return _rpmalloc_mmap_os(size, offset); 415 | } else { 416 | rpmalloc_assert(ptr, "Failed to map virtual memory block"); 417 | } 418 | return 0; 419 | } 420 | #else 421 | int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_UNINITIALIZED; 422 | # if defined(__APPLE__) && !TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR 423 | int fd = (int)VM_MAKE_TAG(240U); 424 | void* ptr = mmap(0, size + padding, PROT_READ | PROT_WRITE, flags, fd, 0); 425 | # else 426 | void* ptr = mmap(0, size + padding, PROT_READ | PROT_WRITE, flags, -1, 0); 427 | # endif 428 | if ((ptr == MAP_FAILED) || !ptr) { 429 | if (_memory_config.map_fail_callback) { 430 | if (_memory_config.map_fail_callback(size + padding)) 431 | return _rpmalloc_mmap_os(size, offset); 432 | } else if (errno != ENOMEM) { 433 | rpmalloc_assert((ptr != MAP_FAILED) && ptr, "Failed to map virtual memory block"); 434 | } 435 | return 0; 436 | } 437 | #endif 438 | if (padding) { 439 | size_t final_padding = padding - ((uintptr_t)ptr & ~_memory_span_mask); 440 | rpmalloc_assert(final_padding <= _memory_span_size, "Internal failure in padding"); 441 | rpmalloc_assert(final_padding <= padding, "Internal failure in padding"); 442 | rpmalloc_assert(!(final_padding % 8), "Internal failure in padding"); 443 | ptr = pointer_offset(ptr, final_padding); 444 | *offset = final_padding >> 3; 445 | } 446 | rpmalloc_assert((size < _memory_span_size) || !((uintptr_t)ptr & ~_memory_span_mask), "Internal failure in padding"); 447 | return ptr; 448 | } 449 | 450 | //! Default implementation to unmap pages from virtual memory 451 | static void 452 | _rpmalloc_unmap_os(void* address, size_t size, size_t offset, size_t release) { 453 | rpmalloc_assert(release || (offset == 0), "Invalid unmap size"); 454 | rpmalloc_assert(!release || (release >= _memory_page_size), "Invalid unmap size"); 455 | rpmalloc_assert(size >= _memory_page_size, "Invalid unmap size"); 456 | if (release && offset) { 457 | offset <<= 3; 458 | address = pointer_offset(address, -(int32_t)offset); 459 | if ((release >= _memory_span_size) && (_memory_span_size > _memory_map_granularity)) { 460 | //Padding is always one span size 461 | release += _memory_span_size; 462 | } 463 | } 464 | #if PLATFORM_WINDOWS 465 | if (!VirtualFree(address, release ? 0 : size, release ? MEM_RELEASE : MEM_DECOMMIT)) { 466 | rpmalloc_assert(0, "Failed to unmap virtual memory block"); 467 | } 468 | #else 469 | if (release) { 470 | if (munmap(address, release)) { 471 | rpmalloc_assert(0, "Failed to unmap virtual memory block"); 472 | } 473 | } else { 474 | #if defined(MADV_FREE_REUSABLE) 475 | int ret; 476 | while ((ret = madvise(address, size, MADV_FREE_REUSABLE)) == -1 && (errno == EAGAIN)) 477 | errno = 0; 478 | if ((ret == -1) && (errno != 0)) { 479 | #elif defined(MADV_DONTNEED) 480 | if (madvise(address, size, MADV_DONTNEED)) { 481 | #elif defined(MADV_PAGEOUT) 482 | if (madvise(address, size, MADV_PAGEOUT)) { 483 | #elif defined(MADV_FREE) 484 | if (madvise(address, size, MADV_FREE)) { 485 | #else 486 | if (posix_madvise(address, size, POSIX_MADV_DONTNEED)) { 487 | #endif 488 | rpmalloc_assert(0, "Failed to madvise virtual memory block as free"); 489 | } 490 | } 491 | #endif 492 | } 493 | 494 | static void 495 | _rpmalloc_span_mark_as_subspan_unless_master(span_t* master, span_t* subspan, size_t span_count); 496 | 497 | //! Use global reserved spans to fulfill a memory map request (reserve size must be checked by caller) 498 | static span_t* 499 | _rpmalloc_global_get_reserved_spans(size_t span_count) { 500 | span_t* span = _memory_global_reserve; 501 | _rpmalloc_span_mark_as_subspan_unless_master(_memory_global_reserve_master, span, span_count); 502 | _memory_global_reserve_count -= span_count; 503 | if (_memory_global_reserve_count) 504 | _memory_global_reserve = (span_t*)pointer_offset(span, span_count << _memory_span_size_shift); 505 | else 506 | _memory_global_reserve = 0; 507 | return span; 508 | } 509 | 510 | //! Store the given spans as global reserve (must only be called from within new heap allocation, not thread safe) 511 | static void 512 | _rpmalloc_global_set_reserved_spans(span_t* master, span_t* reserve, size_t reserve_span_count) { 513 | _memory_global_reserve_master = master; 514 | _memory_global_reserve_count = reserve_span_count; 515 | _memory_global_reserve = reserve; 516 | } 517 | 518 | 519 | //////////// 520 | /// 521 | /// Span linked list management 522 | /// 523 | ////// 524 | 525 | //! Add a span to double linked list at the head 526 | static void 527 | _rpmalloc_span_double_link_list_add(span_t** head, span_t* span) { 528 | if (*head) 529 | (*head)->prev = span; 530 | span->next = *head; 531 | *head = span; 532 | } 533 | 534 | //! Pop head span from double linked list 535 | static void 536 | _rpmalloc_span_double_link_list_pop_head(span_t** head, span_t* span) { 537 | rpmalloc_assert(*head == span, "Linked list corrupted"); 538 | span = *head; 539 | *head = span->next; 540 | } 541 | 542 | //! Remove a span from double linked list 543 | static void 544 | _rpmalloc_span_double_link_list_remove(span_t** head, span_t* span) { 545 | rpmalloc_assert(*head, "Linked list corrupted"); 546 | if (*head == span) { 547 | *head = span->next; 548 | } else { 549 | span_t* next_span = span->next; 550 | span_t* prev_span = span->prev; 551 | prev_span->next = next_span; 552 | if (EXPECTED(next_span != 0)) 553 | next_span->prev = prev_span; 554 | } 555 | } 556 | 557 | 558 | //////////// 559 | /// 560 | /// Span control 561 | /// 562 | ////// 563 | 564 | static void 565 | _rpmalloc_heap_cache_insert(heap_t* heap, span_t* span); 566 | 567 | static void 568 | _rpmalloc_heap_finalize(heap_t* heap); 569 | 570 | static void 571 | _rpmalloc_heap_set_reserved_spans(heap_t* heap, span_t* master, span_t* reserve, size_t reserve_span_count); 572 | 573 | //! Declare the span to be a subspan and store distance from master span and span count 574 | static void 575 | _rpmalloc_span_mark_as_subspan_unless_master(span_t* master, span_t* subspan, size_t span_count) { 576 | rpmalloc_assert((subspan != master) || (subspan->flags & SPAN_FLAG_MASTER), "Span master pointer and/or flag mismatch"); 577 | if (subspan != master) { 578 | subspan->flags = SPAN_FLAG_SUBSPAN; 579 | subspan->offset_from_master = (uint32_t)((uintptr_t)pointer_diff(subspan, master) >> _memory_span_size_shift); 580 | subspan->align_offset = 0; 581 | } 582 | subspan->span_count = (uint32_t)span_count; 583 | } 584 | 585 | //! Use reserved spans to fulfill a memory map request (reserve size must be checked by caller) 586 | static span_t* 587 | _rpmalloc_span_map_from_reserve(heap_t* heap, size_t span_count) { 588 | //Update the heap span reserve 589 | span_t* span = heap->span_reserve; 590 | heap->span_reserve = (span_t*)pointer_offset(span, span_count * _memory_span_size); 591 | heap->spans_reserved -= (uint32_t)span_count; 592 | 593 | _rpmalloc_span_mark_as_subspan_unless_master(heap->span_reserve_master, span, span_count); 594 | return span; 595 | } 596 | 597 | //! Get the aligned number of spans to map in based on wanted count, configured mapping granularity and the page size 598 | static size_t 599 | _rpmalloc_span_align_count(size_t span_count) { 600 | size_t request_count = (span_count > _memory_span_map_count) ? span_count : _memory_span_map_count; 601 | if ((_memory_page_size > _memory_span_size) && ((request_count * _memory_span_size) % _memory_page_size)) 602 | request_count += _memory_span_map_count - (request_count % _memory_span_map_count); 603 | return request_count; 604 | } 605 | 606 | //! Setup a newly mapped span 607 | static void 608 | _rpmalloc_span_initialize(span_t* span, size_t total_span_count, size_t span_count, size_t align_offset) { 609 | span->total_spans = (uint32_t)total_span_count; 610 | span->span_count = (uint32_t)span_count; 611 | span->align_offset = (uint32_t)align_offset; 612 | span->flags = SPAN_FLAG_MASTER; 613 | span->remaining_spans = (int32_t)total_span_count; 614 | } 615 | 616 | static void 617 | _rpmalloc_span_unmap(span_t* span); 618 | 619 | //! Map an aligned set of spans, taking configured mapping granularity and the page size into account 620 | static span_t* 621 | _rpmalloc_span_map_aligned_count(heap_t* heap, size_t span_count) { 622 | //If we already have some, but not enough, reserved spans, release those to heap cache and map a new 623 | //full set of spans. Otherwise we would waste memory if page size > span size (huge pages) 624 | size_t aligned_span_count = _rpmalloc_span_align_count(span_count); 625 | size_t align_offset = 0; 626 | span_t* span = (span_t*)_rpmalloc_mmap(aligned_span_count * _memory_span_size, &align_offset); 627 | if (!span) 628 | return 0; 629 | _rpmalloc_span_initialize(span, aligned_span_count, span_count, align_offset); 630 | if (aligned_span_count > span_count) { 631 | span_t* reserved_spans = (span_t*)pointer_offset(span, span_count * _memory_span_size); 632 | size_t reserved_count = aligned_span_count - span_count; 633 | if (heap->spans_reserved) { 634 | _rpmalloc_span_mark_as_subspan_unless_master(heap->span_reserve_master, heap->span_reserve, heap->spans_reserved); 635 | _rpmalloc_heap_cache_insert(heap, heap->span_reserve); 636 | } 637 | if (reserved_count > _memory_heap_reserve_count) { 638 | size_t remain_count = reserved_count - _memory_heap_reserve_count; 639 | reserved_count = _memory_heap_reserve_count; 640 | span_t* remain_span = (span_t*)pointer_offset(reserved_spans, reserved_count * _memory_span_size); 641 | if (_memory_global_reserve) { 642 | _rpmalloc_span_mark_as_subspan_unless_master(_memory_global_reserve_master, _memory_global_reserve, _memory_global_reserve_count); 643 | _rpmalloc_span_unmap(_memory_global_reserve); 644 | } 645 | _rpmalloc_global_set_reserved_spans(span, remain_span, remain_count); 646 | } 647 | _rpmalloc_heap_set_reserved_spans(heap, span, reserved_spans, reserved_count); 648 | } 649 | return span; 650 | } 651 | 652 | //! Map in memory pages for the given number of spans (or use previously reserved pages) 653 | static span_t* 654 | _rpmalloc_span_map(heap_t* heap, size_t span_count) { 655 | if (span_count <= heap->spans_reserved) 656 | return _rpmalloc_span_map_from_reserve(heap, span_count); 657 | span_t* span = 0; 658 | int use_global_reserve = (_memory_page_size > _memory_span_size) || (_memory_span_map_count > _memory_heap_reserve_count); 659 | if (use_global_reserve) { 660 | if (_memory_global_reserve_count >= span_count) { 661 | size_t reserve_count = (!heap->spans_reserved ? _memory_heap_reserve_count : span_count); 662 | if (_memory_global_reserve_count < reserve_count) 663 | reserve_count = _memory_global_reserve_count; 664 | span = _rpmalloc_global_get_reserved_spans(reserve_count); 665 | if (span) { 666 | if (reserve_count > span_count) { 667 | span_t* reserved_span = (span_t*)pointer_offset(span, span_count << _memory_span_size_shift); 668 | _rpmalloc_heap_set_reserved_spans(heap, _memory_global_reserve_master, reserved_span, reserve_count - span_count); 669 | } 670 | // Already marked as subspan in _rpmalloc_global_get_reserved_spans 671 | span->span_count = (uint32_t)span_count; 672 | } 673 | } 674 | } 675 | if (!span) 676 | span = _rpmalloc_span_map_aligned_count(heap, span_count); 677 | return span; 678 | } 679 | 680 | //! Unmap memory pages for the given number of spans (or mark as unused if no partial unmappings) 681 | static void 682 | _rpmalloc_span_unmap(span_t* span) { 683 | rpmalloc_assert((span->flags & SPAN_FLAG_MASTER) || (span->flags & SPAN_FLAG_SUBSPAN), "Span flag corrupted"); 684 | rpmalloc_assert(!(span->flags & SPAN_FLAG_MASTER) || !(span->flags & SPAN_FLAG_SUBSPAN), "Span flag corrupted"); 685 | 686 | int is_master = !!(span->flags & SPAN_FLAG_MASTER); 687 | span_t* master = is_master ? span : ((span_t*)pointer_offset(span, -(intptr_t)((uintptr_t)span->offset_from_master * _memory_span_size))); 688 | rpmalloc_assert(is_master || (span->flags & SPAN_FLAG_SUBSPAN), "Span flag corrupted"); 689 | rpmalloc_assert(master->flags & SPAN_FLAG_MASTER, "Span flag corrupted"); 690 | 691 | size_t span_count = span->span_count; 692 | if (!is_master) { 693 | //Directly unmap subspans (unless huge pages, in which case we defer and unmap entire page range with master) 694 | rpmalloc_assert(span->align_offset == 0, "Span align offset corrupted"); 695 | if (_memory_span_size >= _memory_page_size) 696 | _rpmalloc_unmap(span, span_count * _memory_span_size, 0, 0); 697 | } else { 698 | //Special double flag to denote an unmapped master 699 | //It must be kept in memory since span header must be used 700 | span->flags |= SPAN_FLAG_MASTER | SPAN_FLAG_SUBSPAN | SPAN_FLAG_UNMAPPED_MASTER; 701 | } 702 | 703 | master->remaining_spans -= (int32_t)span_count; 704 | if (master->remaining_spans <= 0) { 705 | //Everything unmapped, unmap the master span with release flag to unmap the entire range of the super span 706 | rpmalloc_assert(!!(master->flags & SPAN_FLAG_MASTER) && !!(master->flags & SPAN_FLAG_SUBSPAN), "Span flag corrupted"); 707 | size_t unmap_count = master->span_count; 708 | if (_memory_span_size < _memory_page_size) 709 | unmap_count = master->total_spans; 710 | _rpmalloc_unmap(master, unmap_count * _memory_span_size, master->align_offset, (size_t)master->total_spans * _memory_span_size); 711 | } 712 | } 713 | 714 | //! Move the span (used for small or medium allocations) to the heap thread cache 715 | static void 716 | _rpmalloc_span_release_to_cache(heap_t* heap, span_t* span) { 717 | rpmalloc_assert(heap == span->heap, "Span heap pointer corrupted"); 718 | rpmalloc_assert(span->size_class < SIZE_CLASS_COUNT, "Invalid span size class"); 719 | rpmalloc_assert(span->span_count == 1, "Invalid span count"); 720 | if (!heap->finalize) { 721 | if (heap->size_class[span->size_class].cache) 722 | _rpmalloc_heap_cache_insert(heap, heap->size_class[span->size_class].cache); 723 | heap->size_class[span->size_class].cache = span; 724 | } else { 725 | _rpmalloc_span_unmap(span); 726 | } 727 | } 728 | 729 | //! Initialize a (partial) free list up to next system memory page, while reserving the first block 730 | //! as allocated, returning number of blocks in list 731 | static uint32_t 732 | free_list_partial_init(void** list, void** first_block, void* page_start, void* block_start, uint32_t block_count, uint32_t block_size) { 733 | rpmalloc_assert(block_count, "Internal failure"); 734 | *first_block = block_start; 735 | if (block_count > 1) { 736 | void* free_block = pointer_offset(block_start, block_size); 737 | void* block_end = pointer_offset(block_start, (size_t)block_size * block_count); 738 | //If block size is less than half a memory page, bound init to next memory page boundary 739 | if (block_size < (_memory_page_size >> 1)) { 740 | void* page_end = pointer_offset(page_start, _memory_page_size); 741 | if (page_end < block_end) 742 | block_end = page_end; 743 | } 744 | *list = free_block; 745 | block_count = 2; 746 | void* next_block = pointer_offset(free_block, block_size); 747 | while (next_block < block_end) { 748 | *((void**)free_block) = next_block; 749 | free_block = next_block; 750 | ++block_count; 751 | next_block = pointer_offset(next_block, block_size); 752 | } 753 | *((void**)free_block) = 0; 754 | } else { 755 | *list = 0; 756 | } 757 | return block_count; 758 | } 759 | 760 | //! Initialize an unused span (from cache or mapped) to be new active span, putting the initial free list in heap class free list 761 | static void* 762 | _rpmalloc_span_initialize_new(heap_t* heap, heap_size_class_t* heap_size_class, span_t* span, uint32_t class_idx) { 763 | rpmalloc_assert(span->span_count == 1, "Internal failure"); 764 | size_class_t* size_class = _memory_size_class + class_idx; 765 | span->size_class = class_idx; 766 | span->heap = heap; 767 | span->flags &= ~SPAN_FLAG_ALIGNED_BLOCKS; 768 | span->block_size = size_class->block_size; 769 | span->block_count = size_class->block_count; 770 | span->free_list = 0; 771 | span->list_size = 0; 772 | span->free_list_deferred = 0; 773 | 774 | //Setup free list. Only initialize one system page worth of free blocks in list 775 | void* block; 776 | span->free_list_limit = free_list_partial_init(&heap_size_class->free_list, &block, 777 | span, pointer_offset(span, SPAN_HEADER_SIZE), size_class->block_count, size_class->block_size); 778 | //Link span as partial if there remains blocks to be initialized as free list, or full if fully initialized 779 | if (span->free_list_limit < span->block_count) { 780 | _rpmalloc_span_double_link_list_add(&heap_size_class->partial_span, span); 781 | span->used_count = span->free_list_limit; 782 | } else { 783 | ++heap->full_span_count; 784 | span->used_count = span->block_count; 785 | } 786 | return block; 787 | } 788 | 789 | static void 790 | _rpmalloc_span_extract_free_list_deferred(span_t* span) { 791 | // We need acquire semantics on the CAS operation since we are interested in the list size 792 | // Refer to _rpmalloc_deallocate_defer_small_or_medium for further comments on this dependency 793 | span->free_list = span->free_list_deferred; 794 | span->used_count -= span->list_size; 795 | span->list_size = 0; 796 | span->free_list_deferred = 0; 797 | } 798 | 799 | static int 800 | _rpmalloc_span_is_fully_utilized(span_t* span) { 801 | rpmalloc_assert(span->free_list_limit <= span->block_count, "Span free list corrupted"); 802 | return !span->free_list && (span->free_list_limit >= span->block_count); 803 | } 804 | 805 | static int 806 | _rpmalloc_span_finalize(heap_t* heap, size_t iclass, span_t* span, span_t** list_head) { 807 | void* free_list = heap->size_class[iclass].free_list; 808 | span_t* class_span = (span_t*)((uintptr_t)free_list & _memory_span_mask); 809 | if (span == class_span) { 810 | // Adopt the heap class free list back into the span free list 811 | void* block = span->free_list; 812 | void* last_block = 0; 813 | while (block) { 814 | last_block = block; 815 | block = *((void**)block); 816 | } 817 | uint32_t free_count = 0; 818 | block = free_list; 819 | while (block) { 820 | ++free_count; 821 | block = *((void**)block); 822 | } 823 | if (last_block) { 824 | *((void**)last_block) = free_list; 825 | } else { 826 | span->free_list = free_list; 827 | } 828 | heap->size_class[iclass].free_list = 0; 829 | span->used_count -= free_count; 830 | } 831 | //If this assert triggers you have memory leaks 832 | rpmalloc_assert(span->list_size == span->used_count, "Memory leak detected"); 833 | if (span->list_size == span->used_count) { 834 | // This function only used for spans in double linked lists 835 | if (list_head) 836 | _rpmalloc_span_double_link_list_remove(list_head, span); 837 | _rpmalloc_span_unmap(span); 838 | return 1; 839 | } 840 | return 0; 841 | } 842 | 843 | 844 | //////////// 845 | /// 846 | /// Heap control 847 | /// 848 | ////// 849 | 850 | static void 851 | _rpmalloc_deallocate_huge(span_t*); 852 | 853 | //! Store the given spans as reserve in the given heap 854 | static void 855 | _rpmalloc_heap_set_reserved_spans(heap_t* heap, span_t* master, span_t* reserve, size_t reserve_span_count) { 856 | heap->span_reserve_master = master; 857 | heap->span_reserve = reserve; 858 | heap->spans_reserved = (uint32_t)reserve_span_count; 859 | } 860 | 861 | //! Adopt the deferred span cache list, optionally extracting the first single span for immediate re-use 862 | static void 863 | _rpmalloc_heap_cache_adopt_deferred(heap_t* heap, span_t** single_span) { 864 | span_t* span = (span_t*)((void*)heap->span_free_deferred); 865 | while (span) { 866 | span_t* next_span = (span_t*)span->free_list; 867 | rpmalloc_assert(span->heap == heap, "Span heap pointer corrupted"); 868 | if (EXPECTED(span->size_class < SIZE_CLASS_COUNT)) { 869 | rpmalloc_assert(heap->full_span_count, "Heap span counter corrupted"); 870 | --heap->full_span_count; 871 | if (single_span && !*single_span) 872 | *single_span = span; 873 | else 874 | _rpmalloc_heap_cache_insert(heap, span); 875 | } else { 876 | if (span->size_class == SIZE_CLASS_HUGE) { 877 | _rpmalloc_deallocate_huge(span); 878 | } else { 879 | rpmalloc_assert(span->size_class == SIZE_CLASS_LARGE, "Span size class invalid"); 880 | rpmalloc_assert(heap->full_span_count, "Heap span counter corrupted"); 881 | --heap->full_span_count; 882 | uint32_t idx = span->span_count - 1; 883 | if (!idx && single_span && !*single_span) 884 | *single_span = span; 885 | else 886 | _rpmalloc_heap_cache_insert(heap, span); 887 | } 888 | } 889 | span = next_span; 890 | } 891 | } 892 | 893 | static void 894 | _rpmalloc_heap_unmap(heap_t* heap) { 895 | if (!heap->master_heap) { 896 | if ((heap->finalize > 1) && !heap->child_count) { 897 | span_t* span = (span_t*)((uintptr_t)heap & _memory_span_mask); 898 | _rpmalloc_span_unmap(span); 899 | } 900 | } else { 901 | heap->master_heap->child_count -= 1; 902 | if (heap->master_heap->child_count == 0) { 903 | _rpmalloc_heap_unmap(heap->master_heap); 904 | } 905 | } 906 | } 907 | 908 | static void 909 | _rpmalloc_heap_global_finalize(heap_t* heap) { 910 | if (heap->finalize++ > 1) { 911 | --heap->finalize; 912 | return; 913 | } 914 | 915 | _rpmalloc_heap_finalize(heap); 916 | 917 | for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) { 918 | span_cache_t* span_cache; 919 | if (!iclass) 920 | span_cache = &heap->span_cache; 921 | else 922 | span_cache = (span_cache_t*)(heap->span_large_cache + (iclass - 1)); 923 | for (size_t ispan = 0; ispan < span_cache->count; ++ispan) 924 | _rpmalloc_span_unmap(span_cache->span[ispan]); 925 | span_cache->count = 0; 926 | } 927 | 928 | if (heap->full_span_count) { 929 | --heap->finalize; 930 | return; 931 | } 932 | 933 | for (size_t iclass = 0; iclass < SIZE_CLASS_COUNT; ++iclass) { 934 | if (heap->size_class[iclass].free_list || heap->size_class[iclass].partial_span) { 935 | --heap->finalize; 936 | return; 937 | } 938 | } 939 | //Heap is now completely free, unmap and remove from heap list 940 | size_t list_idx = (size_t)heap->id % HEAP_ARRAY_SIZE; 941 | heap_t* list_heap = _memory_heaps[list_idx]; 942 | if (list_heap == heap) { 943 | _memory_heaps[list_idx] = heap->next_heap; 944 | } else { 945 | while (list_heap->next_heap != heap) 946 | list_heap = list_heap->next_heap; 947 | list_heap->next_heap = heap->next_heap; 948 | } 949 | 950 | _rpmalloc_heap_unmap(heap); 951 | } 952 | 953 | //! Insert a single span into thread heap cache, releasing to global cache if overflow 954 | static void 955 | _rpmalloc_heap_cache_insert(heap_t* heap, span_t* span) { 956 | if (UNEXPECTED(heap->finalize != 0)) { 957 | _rpmalloc_span_unmap(span); 958 | _rpmalloc_heap_global_finalize(heap); 959 | return; 960 | } 961 | size_t span_count = span->span_count; 962 | if (span_count == 1) { 963 | span_cache_t* span_cache = &heap->span_cache; 964 | span_cache->span[span_cache->count++] = span; 965 | if (span_cache->count == MAX_THREAD_SPAN_CACHE) { 966 | const size_t remain_count = MAX_THREAD_SPAN_CACHE - THREAD_SPAN_CACHE_TRANSFER; 967 | for (size_t ispan = 0; ispan < THREAD_SPAN_CACHE_TRANSFER; ++ispan) 968 | _rpmalloc_span_unmap(span_cache->span[remain_count + ispan]); 969 | span_cache->count = remain_count; 970 | } 971 | } else { 972 | size_t cache_idx = span_count - 2; 973 | span_large_cache_t* span_cache = heap->span_large_cache + cache_idx; 974 | span_cache->span[span_cache->count++] = span; 975 | const size_t cache_limit = (MAX_THREAD_SPAN_LARGE_CACHE - (span_count >> 1)); 976 | if (span_cache->count == cache_limit) { 977 | const size_t transfer_limit = 2 + (cache_limit >> 2); 978 | const size_t transfer_count = (THREAD_SPAN_LARGE_CACHE_TRANSFER <= transfer_limit ? THREAD_SPAN_LARGE_CACHE_TRANSFER : transfer_limit); 979 | const size_t remain_count = cache_limit - transfer_count; 980 | for (size_t ispan = 0; ispan < transfer_count; ++ispan) 981 | _rpmalloc_span_unmap(span_cache->span[remain_count + ispan]); 982 | span_cache->count = remain_count; 983 | } 984 | } 985 | } 986 | 987 | //! Extract the given number of spans from the different cache levels 988 | static span_t* 989 | _rpmalloc_heap_thread_cache_extract(heap_t* heap, size_t span_count) { 990 | span_t* span = 0; 991 | span_cache_t* span_cache; 992 | if (span_count == 1) 993 | span_cache = &heap->span_cache; 994 | else 995 | span_cache = (span_cache_t*)(heap->span_large_cache + (span_count - 2)); 996 | if (span_cache->count) { 997 | return span_cache->span[--span_cache->count]; 998 | } 999 | return span; 1000 | } 1001 | 1002 | static span_t* 1003 | _rpmalloc_heap_thread_cache_deferred_extract(heap_t* heap, size_t span_count) { 1004 | span_t* span = 0; 1005 | if (span_count == 1) { 1006 | _rpmalloc_heap_cache_adopt_deferred(heap, &span); 1007 | } else { 1008 | _rpmalloc_heap_cache_adopt_deferred(heap, 0); 1009 | span = _rpmalloc_heap_thread_cache_extract(heap, span_count); 1010 | } 1011 | return span; 1012 | } 1013 | 1014 | static span_t* 1015 | _rpmalloc_heap_reserved_extract(heap_t* heap, size_t span_count) { 1016 | if (heap->spans_reserved >= span_count) 1017 | return _rpmalloc_span_map(heap, span_count); 1018 | return 0; 1019 | } 1020 | 1021 | //! Extract a span from the global cache 1022 | static span_t* 1023 | _rpmalloc_heap_global_cache_extract(heap_t* heap, size_t span_count) { 1024 | (void)sizeof(heap); 1025 | (void)sizeof(span_count); 1026 | return 0; 1027 | } 1028 | 1029 | static void 1030 | _rpmalloc_inc_span_statistics(heap_t* heap, size_t span_count, uint32_t class_idx) { 1031 | (void)sizeof(heap); 1032 | (void)sizeof(span_count); 1033 | (void)sizeof(class_idx); 1034 | } 1035 | 1036 | //! Get a span from one of the cache levels (thread cache, reserved, global cache) or fallback to mapping more memory 1037 | static span_t* 1038 | _rpmalloc_heap_extract_new_span(heap_t* heap, heap_size_class_t* heap_size_class, size_t span_count, uint32_t class_idx) { 1039 | span_t* span; 1040 | if (heap_size_class && heap_size_class->cache) { 1041 | span = heap_size_class->cache; 1042 | heap_size_class->cache = (heap->span_cache.count ? heap->span_cache.span[--heap->span_cache.count] : 0); 1043 | _rpmalloc_inc_span_statistics(heap, span_count, class_idx); 1044 | return span; 1045 | } 1046 | (void)sizeof(class_idx); 1047 | // Allow 50% overhead to increase cache hits 1048 | size_t base_span_count = span_count; 1049 | size_t limit_span_count = (span_count > 2) ? (span_count + (span_count >> 1)) : span_count; 1050 | if (limit_span_count > LARGE_CLASS_COUNT) 1051 | limit_span_count = LARGE_CLASS_COUNT; 1052 | do { 1053 | span = _rpmalloc_heap_thread_cache_extract(heap, span_count); 1054 | if (EXPECTED(span != 0)) { 1055 | _rpmalloc_inc_span_statistics(heap, span_count, class_idx); 1056 | return span; 1057 | } 1058 | span = _rpmalloc_heap_thread_cache_deferred_extract(heap, span_count); 1059 | if (EXPECTED(span != 0)) { 1060 | _rpmalloc_inc_span_statistics(heap, span_count, class_idx); 1061 | return span; 1062 | } 1063 | span = _rpmalloc_heap_reserved_extract(heap, span_count); 1064 | if (EXPECTED(span != 0)) { 1065 | _rpmalloc_inc_span_statistics(heap, span_count, class_idx); 1066 | return span; 1067 | } 1068 | span = _rpmalloc_heap_global_cache_extract(heap, span_count); 1069 | if (EXPECTED(span != 0)) { 1070 | _rpmalloc_inc_span_statistics(heap, span_count, class_idx); 1071 | return span; 1072 | } 1073 | ++span_count; 1074 | } while (span_count <= limit_span_count); 1075 | //Final fallback, map in more virtual memory 1076 | span = _rpmalloc_span_map(heap, base_span_count); 1077 | _rpmalloc_inc_span_statistics(heap, base_span_count, class_idx); 1078 | return span; 1079 | } 1080 | 1081 | static void 1082 | _rpmalloc_heap_initialize(heap_t* heap) { 1083 | memset(heap, 0, sizeof(heap_t)); 1084 | //Get a new heap ID 1085 | _memory_heap_id += 1; 1086 | heap->id = _memory_heap_id + 1; 1087 | 1088 | //Link in heap in heap ID map 1089 | size_t list_idx = (size_t)heap->id % HEAP_ARRAY_SIZE; 1090 | heap->next_heap = _memory_heaps[list_idx]; 1091 | _memory_heaps[list_idx] = heap; 1092 | } 1093 | 1094 | static void 1095 | _rpmalloc_heap_orphan(heap_t* heap, int first_class) { 1096 | heap->owner_thread = (uintptr_t)-1; 1097 | (void)sizeof(first_class); 1098 | heap_t** heap_list = &_memory_orphan_heaps; 1099 | heap->next_orphan = *heap_list; 1100 | *heap_list = heap; 1101 | } 1102 | 1103 | //! Allocate a new heap from newly mapped memory pages 1104 | static heap_t* 1105 | _rpmalloc_heap_allocate_new(void) { 1106 | // Map in pages for a 16 heaps. If page size is greater than required size for this, map a page and 1107 | // use first part for heaps and remaining part for spans for allocations. Adds a lot of complexity, 1108 | // but saves a lot of memory on systems where page size > 64 spans (4MiB) 1109 | size_t heap_size = sizeof(heap_t); 1110 | size_t aligned_heap_size = 16 * ((heap_size + 15) / 16); 1111 | size_t request_heap_count = 16; 1112 | size_t heap_span_count = ((aligned_heap_size * request_heap_count) + sizeof(span_t) + _memory_span_size - 1) / _memory_span_size; 1113 | size_t block_size = _memory_span_size * heap_span_count; 1114 | size_t span_count = heap_span_count; 1115 | span_t* span = 0; 1116 | // If there are global reserved spans, use these first 1117 | if (_memory_global_reserve_count >= heap_span_count) { 1118 | span = _rpmalloc_global_get_reserved_spans(heap_span_count); 1119 | } 1120 | if (!span) { 1121 | if (_memory_page_size > block_size) { 1122 | span_count = _memory_page_size / _memory_span_size; 1123 | block_size = _memory_page_size; 1124 | // If using huge pages, make sure to grab enough heaps to avoid reallocating a huge page just to serve new heaps 1125 | size_t possible_heap_count = (block_size - sizeof(span_t)) / aligned_heap_size; 1126 | if (possible_heap_count >= (request_heap_count * 16)) 1127 | request_heap_count *= 16; 1128 | else if (possible_heap_count < request_heap_count) 1129 | request_heap_count = possible_heap_count; 1130 | heap_span_count = ((aligned_heap_size * request_heap_count) + sizeof(span_t) + _memory_span_size - 1) / _memory_span_size; 1131 | } 1132 | 1133 | size_t align_offset = 0; 1134 | span = (span_t*)_rpmalloc_mmap(block_size, &align_offset); 1135 | if (!span) 1136 | return 0; 1137 | 1138 | // Master span will contain the heaps 1139 | _rpmalloc_span_initialize(span, span_count, heap_span_count, align_offset); 1140 | } 1141 | 1142 | size_t remain_size = _memory_span_size - sizeof(span_t); 1143 | heap_t* heap = (heap_t*)pointer_offset(span, sizeof(span_t)); 1144 | _rpmalloc_heap_initialize(heap); 1145 | 1146 | // Put extra heaps as orphans 1147 | size_t num_heaps = remain_size / aligned_heap_size; 1148 | if (num_heaps < request_heap_count) 1149 | num_heaps = request_heap_count; 1150 | heap->child_count = (int32_t)num_heaps - 1; 1151 | heap_t* extra_heap = (heap_t*)pointer_offset(heap, aligned_heap_size); 1152 | while (num_heaps > 1) { 1153 | _rpmalloc_heap_initialize(extra_heap); 1154 | extra_heap->master_heap = heap; 1155 | _rpmalloc_heap_orphan(extra_heap, 1); 1156 | extra_heap = (heap_t*)pointer_offset(extra_heap, aligned_heap_size); 1157 | --num_heaps; 1158 | } 1159 | 1160 | if (span_count > heap_span_count) { 1161 | // Cap reserved spans 1162 | size_t remain_count = span_count - heap_span_count; 1163 | size_t reserve_count = (remain_count > _memory_heap_reserve_count ? _memory_heap_reserve_count : remain_count); 1164 | span_t* remain_span = (span_t*)pointer_offset(span, heap_span_count * _memory_span_size); 1165 | _rpmalloc_heap_set_reserved_spans(heap, span, remain_span, reserve_count); 1166 | 1167 | if (remain_count > reserve_count) { 1168 | // Set to global reserved spans 1169 | remain_span = (span_t*)pointer_offset(remain_span, reserve_count * _memory_span_size); 1170 | reserve_count = remain_count - reserve_count; 1171 | _rpmalloc_global_set_reserved_spans(span, remain_span, reserve_count); 1172 | } 1173 | } 1174 | 1175 | return heap; 1176 | } 1177 | 1178 | static heap_t* 1179 | _rpmalloc_heap_extract_orphan(heap_t** heap_list) { 1180 | heap_t* heap = *heap_list; 1181 | *heap_list = (heap ? heap->next_orphan : 0); 1182 | return heap; 1183 | } 1184 | 1185 | //! Allocate a new heap, potentially reusing a previously orphaned heap 1186 | static heap_t* 1187 | _rpmalloc_heap_allocate(int first_class) { 1188 | heap_t* heap = 0; 1189 | if (first_class == 0) 1190 | heap = _rpmalloc_heap_extract_orphan(&_memory_orphan_heaps); 1191 | if (!heap) 1192 | heap = _rpmalloc_heap_allocate_new(); 1193 | _rpmalloc_heap_cache_adopt_deferred(heap, 0); 1194 | return heap; 1195 | } 1196 | 1197 | static void 1198 | _rpmalloc_heap_release(void* heapptr, int first_class, int release_cache) { 1199 | heap_t* heap = (heap_t*)heapptr; 1200 | if (!heap) 1201 | return; 1202 | //Release thread cache spans back to global cache 1203 | _rpmalloc_heap_cache_adopt_deferred(heap, 0); 1204 | if (release_cache || heap->finalize) { 1205 | for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) { 1206 | span_cache_t* span_cache; 1207 | if (!iclass) 1208 | span_cache = &heap->span_cache; 1209 | else 1210 | span_cache = (span_cache_t*)(heap->span_large_cache + (iclass - 1)); 1211 | if (!span_cache->count) 1212 | continue; 1213 | for (size_t ispan = 0; ispan < span_cache->count; ++ispan) 1214 | _rpmalloc_span_unmap(span_cache->span[ispan]); 1215 | span_cache->count = 0; 1216 | } 1217 | } 1218 | 1219 | if (_memory_thread_heap == heap) 1220 | set_thread_heap(0); 1221 | 1222 | _rpmalloc_heap_orphan(heap, first_class); 1223 | } 1224 | 1225 | static void 1226 | _rpmalloc_heap_release_raw(void* heapptr, int release_cache) { 1227 | _rpmalloc_heap_release(heapptr, 0, release_cache); 1228 | } 1229 | 1230 | static void 1231 | _rpmalloc_heap_finalize(heap_t* heap) { 1232 | if (heap->spans_reserved) { 1233 | span_t* span = _rpmalloc_span_map(heap, heap->spans_reserved); 1234 | _rpmalloc_span_unmap(span); 1235 | heap->spans_reserved = 0; 1236 | } 1237 | 1238 | _rpmalloc_heap_cache_adopt_deferred(heap, 0); 1239 | 1240 | for (size_t iclass = 0; iclass < SIZE_CLASS_COUNT; ++iclass) { 1241 | if (heap->size_class[iclass].cache) 1242 | _rpmalloc_span_unmap(heap->size_class[iclass].cache); 1243 | heap->size_class[iclass].cache = 0; 1244 | span_t* span = heap->size_class[iclass].partial_span; 1245 | while (span) { 1246 | span_t* next = span->next; 1247 | _rpmalloc_span_finalize(heap, iclass, span, &heap->size_class[iclass].partial_span); 1248 | span = next; 1249 | } 1250 | // If class still has a free list it must be a full span 1251 | if (heap->size_class[iclass].free_list) { 1252 | span_t* class_span = (span_t*)((uintptr_t)heap->size_class[iclass].free_list & _memory_span_mask); 1253 | span_t** list = 0; 1254 | --heap->full_span_count; 1255 | if (!_rpmalloc_span_finalize(heap, iclass, class_span, list)) { 1256 | if (list) 1257 | _rpmalloc_span_double_link_list_remove(list, class_span); 1258 | _rpmalloc_span_double_link_list_add(&heap->size_class[iclass].partial_span, class_span); 1259 | } 1260 | } 1261 | } 1262 | 1263 | for (size_t iclass = 0; iclass < LARGE_CLASS_COUNT; ++iclass) { 1264 | span_cache_t* span_cache; 1265 | if (!iclass) 1266 | span_cache = &heap->span_cache; 1267 | else 1268 | span_cache = (span_cache_t*)(heap->span_large_cache + (iclass - 1)); 1269 | for (size_t ispan = 0; ispan < span_cache->count; ++ispan) 1270 | _rpmalloc_span_unmap(span_cache->span[ispan]); 1271 | span_cache->count = 0; 1272 | } 1273 | rpmalloc_assert(!heap->span_free_deferred, "Heaps still active during finalization"); 1274 | } 1275 | 1276 | 1277 | //////////// 1278 | /// 1279 | /// Allocation entry points 1280 | /// 1281 | ////// 1282 | 1283 | //! Pop first block from a free list 1284 | static void* 1285 | free_list_pop(void** list) { 1286 | void* block = *list; 1287 | *list = *((void**)block); 1288 | return block; 1289 | } 1290 | 1291 | //! Allocate a small/medium sized memory block from the given heap 1292 | static void* 1293 | _rpmalloc_allocate_from_heap_fallback(heap_t* heap, heap_size_class_t* heap_size_class, uint32_t class_idx) { 1294 | span_t* span = heap_size_class->partial_span; 1295 | if (EXPECTED(span != 0)) { 1296 | rpmalloc_assert(span->block_count == _memory_size_class[span->size_class].block_count, "Span block count corrupted"); 1297 | rpmalloc_assert(!_rpmalloc_span_is_fully_utilized(span), "Internal failure"); 1298 | void* block; 1299 | if (span->free_list) { 1300 | //Span local free list is not empty, swap to size class free list 1301 | block = free_list_pop(&span->free_list); 1302 | heap_size_class->free_list = span->free_list; 1303 | span->free_list = 0; 1304 | } else { 1305 | //If the span did not fully initialize free list, link up another page worth of blocks 1306 | void* block_start = pointer_offset(span, SPAN_HEADER_SIZE + ((size_t)span->free_list_limit * span->block_size)); 1307 | span->free_list_limit += free_list_partial_init(&heap_size_class->free_list, &block, 1308 | (void*)((uintptr_t)block_start & ~(_memory_page_size - 1)), block_start, 1309 | span->block_count - span->free_list_limit, span->block_size); 1310 | } 1311 | rpmalloc_assert(span->free_list_limit <= span->block_count, "Span block count corrupted"); 1312 | span->used_count = span->free_list_limit; 1313 | 1314 | //Swap in deferred free list if present 1315 | if (span->free_list_deferred) 1316 | _rpmalloc_span_extract_free_list_deferred(span); 1317 | 1318 | //If span is still not fully utilized keep it in partial list and early return block 1319 | if (!_rpmalloc_span_is_fully_utilized(span)) 1320 | return block; 1321 | 1322 | //The span is fully utilized, unlink from partial list and add to fully utilized list 1323 | _rpmalloc_span_double_link_list_pop_head(&heap_size_class->partial_span, span); 1324 | ++heap->full_span_count; 1325 | return block; 1326 | } 1327 | 1328 | //Find a span in one of the cache levels 1329 | span = _rpmalloc_heap_extract_new_span(heap, heap_size_class, 1, class_idx); 1330 | if (EXPECTED(span != 0)) { 1331 | //Mark span as owned by this heap and set base data, return first block 1332 | return _rpmalloc_span_initialize_new(heap, heap_size_class, span, class_idx); 1333 | } 1334 | 1335 | return 0; 1336 | } 1337 | 1338 | //! Allocate a small sized memory block from the given heap 1339 | static void* 1340 | _rpmalloc_allocate_small(heap_t* heap, size_t size) { 1341 | rpmalloc_assert(heap, "No thread heap"); 1342 | //Small sizes have unique size classes 1343 | const uint32_t class_idx = (uint32_t)((size + (SMALL_GRANULARITY - 1)) >> SMALL_GRANULARITY_SHIFT); 1344 | heap_size_class_t* heap_size_class = heap->size_class + class_idx; 1345 | if (EXPECTED(heap_size_class->free_list != 0)) 1346 | return free_list_pop(&heap_size_class->free_list); 1347 | return _rpmalloc_allocate_from_heap_fallback(heap, heap_size_class, class_idx); 1348 | } 1349 | 1350 | //! Allocate a medium sized memory block from the given heap 1351 | static void* 1352 | _rpmalloc_allocate_medium(heap_t* heap, size_t size) { 1353 | rpmalloc_assert(heap, "No thread heap"); 1354 | //Calculate the size class index and do a dependent lookup of the final class index (in case of merged classes) 1355 | const uint32_t base_idx = (uint32_t)(SMALL_CLASS_COUNT + ((size - (SMALL_SIZE_LIMIT + 1)) >> MEDIUM_GRANULARITY_SHIFT)); 1356 | const uint32_t class_idx = _memory_size_class[base_idx].class_idx; 1357 | heap_size_class_t* heap_size_class = heap->size_class + class_idx; 1358 | if (EXPECTED(heap_size_class->free_list != 0)) 1359 | return free_list_pop(&heap_size_class->free_list); 1360 | return _rpmalloc_allocate_from_heap_fallback(heap, heap_size_class, class_idx); 1361 | } 1362 | 1363 | //! Allocate a large sized memory block from the given heap 1364 | static void* 1365 | _rpmalloc_allocate_large(heap_t* heap, size_t size) { 1366 | rpmalloc_assert(heap, "No thread heap"); 1367 | //Calculate number of needed max sized spans (including header) 1368 | //Since this function is never called if size > LARGE_SIZE_LIMIT 1369 | //the span_count is guaranteed to be <= LARGE_CLASS_COUNT 1370 | size += SPAN_HEADER_SIZE; 1371 | size_t span_count = size >> _memory_span_size_shift; 1372 | if (size & (_memory_span_size - 1)) 1373 | ++span_count; 1374 | 1375 | //Find a span in one of the cache levels 1376 | span_t* span = _rpmalloc_heap_extract_new_span(heap, 0, span_count, SIZE_CLASS_LARGE); 1377 | if (!span) 1378 | return span; 1379 | 1380 | //Mark span as owned by this heap and set base data 1381 | rpmalloc_assert(span->span_count >= span_count, "Internal failure"); 1382 | span->size_class = SIZE_CLASS_LARGE; 1383 | span->heap = heap; 1384 | ++heap->full_span_count; 1385 | 1386 | return pointer_offset(span, SPAN_HEADER_SIZE); 1387 | } 1388 | 1389 | //! Allocate a huge block by mapping memory pages directly 1390 | static void* 1391 | _rpmalloc_allocate_huge(heap_t* heap, size_t size) { 1392 | rpmalloc_assert(heap, "No thread heap"); 1393 | _rpmalloc_heap_cache_adopt_deferred(heap, 0); 1394 | size += SPAN_HEADER_SIZE; 1395 | size_t num_pages = size >> _memory_page_size_shift; 1396 | if (size & (_memory_page_size - 1)) 1397 | ++num_pages; 1398 | size_t align_offset = 0; 1399 | span_t* span = (span_t*)_rpmalloc_mmap(num_pages * _memory_page_size, &align_offset); 1400 | if (!span) 1401 | return span; 1402 | 1403 | //Store page count in span_count 1404 | span->size_class = SIZE_CLASS_HUGE; 1405 | span->span_count = (uint32_t)num_pages; 1406 | span->align_offset = (uint32_t)align_offset; 1407 | span->heap = heap; 1408 | ++heap->full_span_count; 1409 | 1410 | return pointer_offset(span, SPAN_HEADER_SIZE); 1411 | } 1412 | 1413 | //! Allocate a block of the given size 1414 | static void* 1415 | _rpmalloc_allocate(heap_t* heap, size_t size) { 1416 | if (EXPECTED(size <= SMALL_SIZE_LIMIT)) 1417 | return _rpmalloc_allocate_small(heap, size); 1418 | else if (size <= _memory_medium_size_limit) 1419 | return _rpmalloc_allocate_medium(heap, size); 1420 | else if (size <= LARGE_SIZE_LIMIT) 1421 | return _rpmalloc_allocate_large(heap, size); 1422 | return _rpmalloc_allocate_huge(heap, size); 1423 | } 1424 | 1425 | static void* 1426 | _rpmalloc_aligned_allocate(heap_t* heap, size_t alignment, size_t size) { 1427 | if (alignment <= SMALL_GRANULARITY) 1428 | return _rpmalloc_allocate(heap, size); 1429 | 1430 | if ((alignment <= SPAN_HEADER_SIZE) && (size < _memory_medium_size_limit)) { 1431 | // If alignment is less or equal to span header size (which is power of two), 1432 | // and size aligned to span header size multiples is less than size + alignment, 1433 | // then use natural alignment of blocks to provide alignment 1434 | size_t multiple_size = size ? (size + (SPAN_HEADER_SIZE - 1)) & ~(uintptr_t)(SPAN_HEADER_SIZE - 1) : SPAN_HEADER_SIZE; 1435 | rpmalloc_assert(!(multiple_size % SPAN_HEADER_SIZE), "Failed alignment calculation"); 1436 | if (multiple_size <= (size + alignment)) 1437 | return _rpmalloc_allocate(heap, multiple_size); 1438 | } 1439 | 1440 | void* ptr = 0; 1441 | size_t align_mask = alignment - 1; 1442 | if (alignment <= _memory_page_size) { 1443 | ptr = _rpmalloc_allocate(heap, size + alignment); 1444 | if ((uintptr_t)ptr & align_mask) { 1445 | ptr = (void*)(((uintptr_t)ptr & ~(uintptr_t)align_mask) + alignment); 1446 | //Mark as having aligned blocks 1447 | span_t* span = (span_t*)((uintptr_t)ptr & _memory_span_mask); 1448 | span->flags |= SPAN_FLAG_ALIGNED_BLOCKS; 1449 | } 1450 | return ptr; 1451 | } 1452 | 1453 | // Fallback to mapping new pages for this request. Since pointers passed 1454 | // to rpfree must be able to reach the start of the span by bitmasking of 1455 | // the address with the span size, the returned aligned pointer from this 1456 | // function must be with a span size of the start of the mapped area. 1457 | // In worst case this requires us to loop and map pages until we get a 1458 | // suitable memory address. It also means we can never align to span size 1459 | // or greater, since the span header will push alignment more than one 1460 | // span size away from span start (thus causing pointer mask to give us 1461 | // an invalid span start on free) 1462 | if (alignment & align_mask) { 1463 | errno = EINVAL; 1464 | return 0; 1465 | } 1466 | if (alignment >= _memory_span_size) { 1467 | errno = EINVAL; 1468 | return 0; 1469 | } 1470 | 1471 | size_t extra_pages = alignment / _memory_page_size; 1472 | 1473 | // Since each span has a header, we will at least need one extra memory page 1474 | size_t num_pages = 1 + (size / _memory_page_size); 1475 | if (size & (_memory_page_size - 1)) 1476 | ++num_pages; 1477 | 1478 | if (extra_pages > num_pages) 1479 | num_pages = 1 + extra_pages; 1480 | 1481 | size_t original_pages = num_pages; 1482 | size_t limit_pages = (_memory_span_size / _memory_page_size) * 2; 1483 | if (limit_pages < (original_pages * 2)) 1484 | limit_pages = original_pages * 2; 1485 | 1486 | size_t mapped_size, align_offset; 1487 | span_t* span; 1488 | 1489 | retry: 1490 | align_offset = 0; 1491 | mapped_size = num_pages * _memory_page_size; 1492 | 1493 | span = (span_t*)_rpmalloc_mmap(mapped_size, &align_offset); 1494 | if (!span) { 1495 | errno = ENOMEM; 1496 | return 0; 1497 | } 1498 | ptr = pointer_offset(span, SPAN_HEADER_SIZE); 1499 | 1500 | if ((uintptr_t)ptr & align_mask) 1501 | ptr = (void*)(((uintptr_t)ptr & ~(uintptr_t)align_mask) + alignment); 1502 | 1503 | if (((size_t)pointer_diff(ptr, span) >= _memory_span_size) || 1504 | (pointer_offset(ptr, size) > pointer_offset(span, mapped_size)) || 1505 | (((uintptr_t)ptr & _memory_span_mask) != (uintptr_t)span)) { 1506 | _rpmalloc_unmap(span, mapped_size, align_offset, mapped_size); 1507 | ++num_pages; 1508 | if (num_pages > limit_pages) { 1509 | errno = EINVAL; 1510 | return 0; 1511 | } 1512 | goto retry; 1513 | } 1514 | 1515 | //Store page count in span_count 1516 | span->size_class = SIZE_CLASS_HUGE; 1517 | span->span_count = (uint32_t)num_pages; 1518 | span->align_offset = (uint32_t)align_offset; 1519 | span->heap = heap; 1520 | ++heap->full_span_count; 1521 | 1522 | return ptr; 1523 | } 1524 | 1525 | 1526 | //////////// 1527 | /// 1528 | /// Deallocation entry points 1529 | /// 1530 | ////// 1531 | 1532 | //! Deallocate the given small/medium memory block in the current thread local heap 1533 | static void 1534 | _rpmalloc_deallocate_direct_small_or_medium(span_t* span, void* block) { 1535 | heap_t* heap = span->heap; 1536 | //Add block to free list 1537 | if (UNEXPECTED(_rpmalloc_span_is_fully_utilized(span))) { 1538 | span->used_count = span->block_count; 1539 | _rpmalloc_span_double_link_list_add(&heap->size_class[span->size_class].partial_span, span); 1540 | --heap->full_span_count; 1541 | } 1542 | *((void**)block) = span->free_list; 1543 | --span->used_count; 1544 | span->free_list = block; 1545 | if (UNEXPECTED(span->used_count == span->list_size)) { 1546 | _rpmalloc_span_double_link_list_remove(&heap->size_class[span->size_class].partial_span, span); 1547 | _rpmalloc_span_release_to_cache(heap, span); 1548 | } 1549 | } 1550 | 1551 | static void 1552 | _rpmalloc_deallocate_small_or_medium(span_t* span, void* p) { 1553 | if (span->flags & SPAN_FLAG_ALIGNED_BLOCKS) { 1554 | //Realign pointer to block start 1555 | void* blocks_start = pointer_offset(span, SPAN_HEADER_SIZE); 1556 | uint32_t block_offset = (uint32_t)pointer_diff(p, blocks_start); 1557 | p = pointer_offset(p, -(int32_t)(block_offset % span->block_size)); 1558 | } 1559 | _rpmalloc_deallocate_direct_small_or_medium(span, p); 1560 | } 1561 | 1562 | //! Deallocate the given large memory block to the current heap 1563 | static void 1564 | _rpmalloc_deallocate_large(span_t* span) { 1565 | rpmalloc_assert(span->size_class == SIZE_CLASS_LARGE, "Bad span size class"); 1566 | rpmalloc_assert(!(span->flags & SPAN_FLAG_MASTER) || !(span->flags & SPAN_FLAG_SUBSPAN), "Span flag corrupted"); 1567 | rpmalloc_assert((span->flags & SPAN_FLAG_MASTER) || (span->flags & SPAN_FLAG_SUBSPAN), "Span flag corrupted"); 1568 | rpmalloc_assert(span->heap->full_span_count, "Heap span counter corrupted"); 1569 | --span->heap->full_span_count; 1570 | heap_t* heap = span->heap; 1571 | rpmalloc_assert(heap, "No thread heap"); 1572 | const int set_as_reserved = ((span->span_count > 1) && (heap->span_cache.count == 0) && !heap->finalize && !heap->spans_reserved); 1573 | if (set_as_reserved) { 1574 | heap->span_reserve = span; 1575 | heap->spans_reserved = span->span_count; 1576 | if (span->flags & SPAN_FLAG_MASTER) { 1577 | heap->span_reserve_master = span; 1578 | } else { //SPAN_FLAG_SUBSPAN 1579 | span_t* master = (span_t*)pointer_offset(span, -(intptr_t)((size_t)span->offset_from_master * _memory_span_size)); 1580 | heap->span_reserve_master = master; 1581 | rpmalloc_assert(master->flags & SPAN_FLAG_MASTER, "Span flag corrupted"); 1582 | rpmalloc_assert(master->remaining_spans >= (int32_t)span->span_count, "Master span count corrupted"); 1583 | } 1584 | } else { 1585 | //Insert into cache list 1586 | _rpmalloc_heap_cache_insert(heap, span); 1587 | } 1588 | } 1589 | 1590 | //! Deallocate the given huge span 1591 | static void 1592 | _rpmalloc_deallocate_huge(span_t* span) { 1593 | rpmalloc_assert(span->heap, "No span heap"); 1594 | rpmalloc_assert(span->heap->full_span_count, "Heap span counter corrupted"); 1595 | --span->heap->full_span_count; 1596 | 1597 | //Oversized allocation, page count is stored in span_count 1598 | size_t num_pages = span->span_count; 1599 | _rpmalloc_unmap(span, num_pages * _memory_page_size, span->align_offset, num_pages * _memory_page_size); 1600 | } 1601 | 1602 | //! Deallocate the given block 1603 | static void 1604 | _rpmalloc_deallocate(void* p) { 1605 | //Grab the span (always at start of span, using span alignment) 1606 | span_t* span = (span_t*)((uintptr_t)p & _memory_span_mask); 1607 | if (UNEXPECTED(!span)) 1608 | return; 1609 | if (EXPECTED(span->size_class < SIZE_CLASS_COUNT)) 1610 | _rpmalloc_deallocate_small_or_medium(span, p); 1611 | else if (span->size_class == SIZE_CLASS_LARGE) 1612 | _rpmalloc_deallocate_large(span); 1613 | else 1614 | _rpmalloc_deallocate_huge(span); 1615 | } 1616 | 1617 | //////////// 1618 | /// 1619 | /// Reallocation entry points 1620 | /// 1621 | ////// 1622 | 1623 | static size_t 1624 | _rpmalloc_usable_size(void* p); 1625 | 1626 | //! Reallocate the given block to the given size 1627 | static void* 1628 | _rpmalloc_reallocate(heap_t* heap, void* p, size_t size, size_t oldsize, unsigned int flags) { 1629 | if (p) { 1630 | //Grab the span using guaranteed span alignment 1631 | span_t* span = (span_t*)((uintptr_t)p & _memory_span_mask); 1632 | if (EXPECTED(span->size_class < SIZE_CLASS_COUNT)) { 1633 | //Small/medium sized block 1634 | rpmalloc_assert(span->span_count == 1, "Span counter corrupted"); 1635 | void* blocks_start = pointer_offset(span, SPAN_HEADER_SIZE); 1636 | uint32_t block_offset = (uint32_t)pointer_diff(p, blocks_start); 1637 | uint32_t block_idx = block_offset / span->block_size; 1638 | void* block = pointer_offset(blocks_start, (size_t)block_idx * span->block_size); 1639 | if (!oldsize) 1640 | oldsize = (size_t)((ptrdiff_t)span->block_size - pointer_diff(p, block)); 1641 | if ((size_t)span->block_size >= size) { 1642 | //Still fits in block, never mind trying to save memory, but preserve data if alignment changed 1643 | if ((p != block) && !(flags & RPMALLOC_NO_PRESERVE)) 1644 | memmove(block, p, oldsize); 1645 | return block; 1646 | } 1647 | } else if (span->size_class == SIZE_CLASS_LARGE) { 1648 | //Large block 1649 | size_t total_size = size + SPAN_HEADER_SIZE; 1650 | size_t num_spans = total_size >> _memory_span_size_shift; 1651 | if (total_size & (_memory_span_mask - 1)) 1652 | ++num_spans; 1653 | size_t current_spans = span->span_count; 1654 | void* block = pointer_offset(span, SPAN_HEADER_SIZE); 1655 | if (!oldsize) 1656 | oldsize = (current_spans * _memory_span_size) - (size_t)pointer_diff(p, block) - SPAN_HEADER_SIZE; 1657 | if ((current_spans >= num_spans) && (total_size >= (oldsize / 2))) { 1658 | //Still fits in block, never mind trying to save memory, but preserve data if alignment changed 1659 | if ((p != block) && !(flags & RPMALLOC_NO_PRESERVE)) 1660 | memmove(block, p, oldsize); 1661 | return block; 1662 | } 1663 | } else { 1664 | //Oversized block 1665 | size_t total_size = size + SPAN_HEADER_SIZE; 1666 | size_t num_pages = total_size >> _memory_page_size_shift; 1667 | if (total_size & (_memory_page_size - 1)) 1668 | ++num_pages; 1669 | //Page count is stored in span_count 1670 | size_t current_pages = span->span_count; 1671 | void* block = pointer_offset(span, SPAN_HEADER_SIZE); 1672 | if (!oldsize) 1673 | oldsize = (current_pages * _memory_page_size) - (size_t)pointer_diff(p, block) - SPAN_HEADER_SIZE; 1674 | if ((current_pages >= num_pages) && (num_pages >= (current_pages / 2))) { 1675 | //Still fits in block, never mind trying to save memory, but preserve data if alignment changed 1676 | if ((p != block) && !(flags & RPMALLOC_NO_PRESERVE)) 1677 | memmove(block, p, oldsize); 1678 | return block; 1679 | } 1680 | } 1681 | } else { 1682 | oldsize = 0; 1683 | } 1684 | 1685 | if (!!(flags & RPMALLOC_GROW_OR_FAIL)) 1686 | return 0; 1687 | 1688 | //Size is greater than block size, need to allocate a new block and deallocate the old 1689 | //Avoid hysteresis by overallocating if increase is small (below 37%) 1690 | size_t lower_bound = oldsize + (oldsize >> 2) + (oldsize >> 3); 1691 | size_t new_size = (size > lower_bound) ? size : ((size > oldsize) ? lower_bound : size); 1692 | void* block = _rpmalloc_allocate(heap, new_size); 1693 | if (p && block) { 1694 | if (!(flags & RPMALLOC_NO_PRESERVE)) 1695 | memcpy(block, p, oldsize < new_size ? oldsize : new_size); 1696 | _rpmalloc_deallocate(p); 1697 | } 1698 | 1699 | return block; 1700 | } 1701 | 1702 | static void* 1703 | _rpmalloc_aligned_reallocate(heap_t* heap, void* ptr, size_t alignment, size_t size, size_t oldsize, 1704 | unsigned int flags) { 1705 | if (alignment <= SMALL_GRANULARITY) 1706 | return _rpmalloc_reallocate(heap, ptr, size, oldsize, flags); 1707 | 1708 | int no_alloc = !!(flags & RPMALLOC_GROW_OR_FAIL); 1709 | size_t usablesize = (ptr ? _rpmalloc_usable_size(ptr) : 0); 1710 | if ((usablesize >= size) && !((uintptr_t)ptr & (alignment - 1))) { 1711 | if (no_alloc || (size >= (usablesize / 2))) 1712 | return ptr; 1713 | } 1714 | // Aligned alloc marks span as having aligned blocks 1715 | void* block = (!no_alloc ? _rpmalloc_aligned_allocate(heap, alignment, size) : 0); 1716 | if (EXPECTED(block != 0)) { 1717 | if (!(flags & RPMALLOC_NO_PRESERVE) && ptr) { 1718 | if (!oldsize) 1719 | oldsize = usablesize; 1720 | memcpy(block, ptr, oldsize < size ? oldsize : size); 1721 | } 1722 | _rpmalloc_deallocate(ptr); 1723 | } 1724 | return block; 1725 | } 1726 | 1727 | 1728 | //////////// 1729 | /// 1730 | /// Initialization, finalization and utility 1731 | /// 1732 | ////// 1733 | 1734 | //! Get the usable size of the given block 1735 | static size_t 1736 | _rpmalloc_usable_size(void* p) { 1737 | //Grab the span using guaranteed span alignment 1738 | span_t* span = (span_t*)((uintptr_t)p & _memory_span_mask); 1739 | if (span->size_class < SIZE_CLASS_COUNT) { 1740 | //Small/medium block 1741 | void* blocks_start = pointer_offset(span, SPAN_HEADER_SIZE); 1742 | return span->block_size - ((size_t)pointer_diff(p, blocks_start) % span->block_size); 1743 | } 1744 | if (span->size_class == SIZE_CLASS_LARGE) { 1745 | //Large block 1746 | size_t current_spans = span->span_count; 1747 | return (current_spans * _memory_span_size) - (size_t)pointer_diff(p, span); 1748 | } 1749 | //Oversized block, page count is stored in span_count 1750 | size_t current_pages = span->span_count; 1751 | return (current_pages * _memory_page_size) - (size_t)pointer_diff(p, span); 1752 | } 1753 | 1754 | //! Adjust and optimize the size class properties for the given class 1755 | static void 1756 | _rpmalloc_adjust_size_class(size_t iclass) { 1757 | size_t block_size = _memory_size_class[iclass].block_size; 1758 | size_t block_count = (_memory_span_size - SPAN_HEADER_SIZE) / block_size; 1759 | 1760 | _memory_size_class[iclass].block_count = (uint16_t)block_count; 1761 | _memory_size_class[iclass].class_idx = (uint16_t)iclass; 1762 | 1763 | //Check if previous size classes can be merged 1764 | if (iclass >= SMALL_CLASS_COUNT) { 1765 | size_t prevclass = iclass; 1766 | while (prevclass > 0) { 1767 | --prevclass; 1768 | //A class can be merged if number of pages and number of blocks are equal 1769 | if (_memory_size_class[prevclass].block_count == _memory_size_class[iclass].block_count) 1770 | memcpy(_memory_size_class + prevclass, _memory_size_class + iclass, sizeof(_memory_size_class[iclass])); 1771 | else 1772 | break; 1773 | } 1774 | } 1775 | } 1776 | 1777 | //! Initialize the allocator and setup global data 1778 | int 1779 | rpmalloc_initialize(void) { 1780 | if (_rpmalloc_initialized) { 1781 | rpmalloc_thread_initialize(); 1782 | return 0; 1783 | } 1784 | return rpmalloc_initialize_config(0); 1785 | } 1786 | 1787 | int 1788 | rpmalloc_initialize_config(const rpmalloc_config_t* config) { 1789 | if (_rpmalloc_initialized) { 1790 | rpmalloc_thread_initialize(); 1791 | return 0; 1792 | } 1793 | _rpmalloc_initialized = 1; 1794 | 1795 | if (config) 1796 | memcpy(&_memory_config, config, sizeof(rpmalloc_config_t)); 1797 | else 1798 | memset(&_memory_config, 0, sizeof(rpmalloc_config_t)); 1799 | 1800 | if (!_memory_config.memory_map || !_memory_config.memory_unmap) { 1801 | _memory_config.memory_map = _rpmalloc_mmap_os; 1802 | _memory_config.memory_unmap = _rpmalloc_unmap_os; 1803 | } 1804 | 1805 | #if PLATFORM_WINDOWS 1806 | SYSTEM_INFO system_info; 1807 | memset(&system_info, 0, sizeof(system_info)); 1808 | GetSystemInfo(&system_info); 1809 | _memory_map_granularity = system_info.dwAllocationGranularity; 1810 | #else 1811 | _memory_map_granularity = (size_t)sysconf(_SC_PAGESIZE); 1812 | #endif 1813 | 1814 | _memory_page_size = 0; 1815 | if (!_memory_page_size) { 1816 | #if PLATFORM_WINDOWS 1817 | _memory_page_size = system_info.dwPageSize; 1818 | #else 1819 | _memory_page_size = _memory_map_granularity; 1820 | #endif 1821 | } 1822 | 1823 | size_t min_span_size = 256; 1824 | size_t max_page_size; 1825 | #if UINTPTR_MAX > 0xFFFFFFFF 1826 | max_page_size = 4096ULL * 1024ULL * 1024ULL; 1827 | #else 1828 | max_page_size = 4 * 1024 * 1024; 1829 | #endif 1830 | if (_memory_page_size < min_span_size) 1831 | _memory_page_size = min_span_size; 1832 | if (_memory_page_size > max_page_size) 1833 | _memory_page_size = max_page_size; 1834 | _memory_page_size_shift = 0; 1835 | size_t page_size_bit = _memory_page_size; 1836 | while (page_size_bit != 1) { 1837 | ++_memory_page_size_shift; 1838 | page_size_bit >>= 1; 1839 | } 1840 | _memory_page_size = ((size_t)1 << _memory_page_size_shift); 1841 | 1842 | _memory_span_map_count = ( _memory_config.span_map_count ? _memory_config.span_map_count : DEFAULT_SPAN_MAP_COUNT); 1843 | if ((_memory_span_size * _memory_span_map_count) < _memory_page_size) 1844 | _memory_span_map_count = (_memory_page_size / _memory_span_size); 1845 | if ((_memory_page_size >= _memory_span_size) && ((_memory_span_map_count * _memory_span_size) % _memory_page_size)) 1846 | _memory_span_map_count = (_memory_page_size / _memory_span_size); 1847 | _memory_heap_reserve_count = (_memory_span_map_count > DEFAULT_SPAN_MAP_COUNT) ? DEFAULT_SPAN_MAP_COUNT : _memory_span_map_count; 1848 | 1849 | _memory_config.span_map_count = _memory_span_map_count; 1850 | 1851 | //Setup all small and medium size classes 1852 | size_t iclass = 0; 1853 | _memory_size_class[iclass].block_size = SMALL_GRANULARITY; 1854 | _rpmalloc_adjust_size_class(iclass); 1855 | for (iclass = 1; iclass < SMALL_CLASS_COUNT; ++iclass) { 1856 | size_t size = iclass * SMALL_GRANULARITY; 1857 | _memory_size_class[iclass].block_size = (uint32_t)size; 1858 | _rpmalloc_adjust_size_class(iclass); 1859 | } 1860 | //At least two blocks per span, then fall back to large allocations 1861 | _memory_medium_size_limit = (_memory_span_size - SPAN_HEADER_SIZE) >> 1; 1862 | if (_memory_medium_size_limit > MEDIUM_SIZE_LIMIT) 1863 | _memory_medium_size_limit = MEDIUM_SIZE_LIMIT; 1864 | for (iclass = 0; iclass < MEDIUM_CLASS_COUNT; ++iclass) { 1865 | size_t size = SMALL_SIZE_LIMIT + ((iclass + 1) * MEDIUM_GRANULARITY); 1866 | if (size > _memory_medium_size_limit) 1867 | break; 1868 | _memory_size_class[SMALL_CLASS_COUNT + iclass].block_size = (uint32_t)size; 1869 | _rpmalloc_adjust_size_class(SMALL_CLASS_COUNT + iclass); 1870 | } 1871 | 1872 | _memory_orphan_heaps = 0; 1873 | memset(_memory_heaps, 0, sizeof(_memory_heaps)); 1874 | 1875 | //Initialize this thread 1876 | rpmalloc_thread_initialize(); 1877 | return 0; 1878 | } 1879 | 1880 | //! Finalize the allocator 1881 | void 1882 | rpmalloc_finalize(void) { 1883 | rpmalloc_thread_finalize(1); 1884 | 1885 | if (_memory_global_reserve) { 1886 | _memory_global_reserve_master->remaining_spans -= (int32_t)_memory_global_reserve_count; 1887 | _memory_global_reserve_master = 0; 1888 | _memory_global_reserve_count = 0; 1889 | _memory_global_reserve = 0; 1890 | } 1891 | 1892 | //Free all thread caches and fully free spans 1893 | for (size_t list_idx = 0; list_idx < HEAP_ARRAY_SIZE; ++list_idx) { 1894 | heap_t* heap = _memory_heaps[list_idx]; 1895 | while (heap) { 1896 | heap_t* next_heap = heap->next_heap; 1897 | heap->finalize = 1; 1898 | _rpmalloc_heap_global_finalize(heap); 1899 | heap = next_heap; 1900 | } 1901 | } 1902 | 1903 | _rpmalloc_initialized = 0; 1904 | } 1905 | 1906 | //! Initialize thread, assign heap 1907 | void 1908 | rpmalloc_thread_initialize(void) { 1909 | if (!_memory_thread_heap) { 1910 | heap_t* heap = _rpmalloc_heap_allocate(0); 1911 | if (heap) { 1912 | set_thread_heap(heap); 1913 | } 1914 | } 1915 | } 1916 | 1917 | //! Finalize thread, orphan heap 1918 | void 1919 | rpmalloc_thread_finalize(int release_caches) { 1920 | heap_t* heap = _memory_thread_heap; 1921 | if (heap) 1922 | _rpmalloc_heap_release_raw(heap, release_caches); 1923 | set_thread_heap(0); 1924 | } 1925 | 1926 | int 1927 | rpmalloc_is_thread_initialized(void) { 1928 | return (_memory_thread_heap != 0) ? 1 : 0; 1929 | } 1930 | 1931 | const rpmalloc_config_t* 1932 | rpmalloc_config(void) { 1933 | return &_memory_config; 1934 | } 1935 | 1936 | // Extern interface 1937 | 1938 | RPMALLOC_ALLOCATOR void* 1939 | rpmalloc(size_t size) { 1940 | return _rpmalloc_allocate(_memory_thread_heap, size); 1941 | } 1942 | 1943 | void 1944 | rpfree(void* ptr) { 1945 | _rpmalloc_deallocate(ptr); 1946 | } 1947 | 1948 | RPMALLOC_ALLOCATOR void* 1949 | rpcalloc(size_t num, size_t size) { 1950 | size_t total; 1951 | total = num * size; 1952 | void* block = _rpmalloc_allocate(_memory_thread_heap, total); 1953 | if (block) 1954 | memset(block, 0, total); 1955 | return block; 1956 | } 1957 | 1958 | RPMALLOC_ALLOCATOR void* 1959 | rprealloc(void* ptr, size_t size) { 1960 | return _rpmalloc_reallocate(_memory_thread_heap, ptr, size, 0, 0); 1961 | } 1962 | 1963 | RPMALLOC_ALLOCATOR void* 1964 | rpaligned_realloc(void* ptr, size_t alignment, size_t size, size_t oldsize, 1965 | unsigned int flags) { 1966 | return _rpmalloc_aligned_reallocate(_memory_thread_heap, ptr, alignment, size, oldsize, flags); 1967 | } 1968 | 1969 | RPMALLOC_ALLOCATOR void* 1970 | rpaligned_alloc(size_t alignment, size_t size) { 1971 | return _rpmalloc_aligned_allocate(_memory_thread_heap, alignment, size); 1972 | } 1973 | 1974 | RPMALLOC_ALLOCATOR void* 1975 | rpaligned_calloc(size_t alignment, size_t num, size_t size) { 1976 | size_t total; 1977 | total = num * size; 1978 | void* block = rpaligned_alloc(alignment, total); 1979 | if (block) 1980 | memset(block, 0, total); 1981 | return block; 1982 | } 1983 | 1984 | RPMALLOC_ALLOCATOR void* 1985 | rpmemalign(size_t alignment, size_t size) { 1986 | return rpaligned_alloc(alignment, size); 1987 | } 1988 | 1989 | int 1990 | rpposix_memalign(void **memptr, size_t alignment, size_t size) { 1991 | if (memptr) 1992 | *memptr = rpaligned_alloc(alignment, size); 1993 | else 1994 | return EINVAL; 1995 | return *memptr ? 0 : ENOMEM; 1996 | } 1997 | 1998 | size_t 1999 | rpmalloc_usable_size(void* ptr) { 2000 | return (ptr ? _rpmalloc_usable_size(ptr) : 0); 2001 | } 2002 | --------------------------------------------------------------------------------