├── .gitignore ├── README.md ├── araqsgc.nim ├── license.txt ├── mimalloc ├── alloc-aligned.c ├── alloc-override-osx.c ├── alloc-override-win.c ├── alloc-override.c ├── alloc-posix.c ├── alloc.c ├── heap.c ├── init.c ├── memory.c ├── mimalloc-atomic.h ├── mimalloc-internal.h ├── mimalloc-types.h ├── mimalloc.h ├── options.c ├── os.c ├── page-queue.c ├── page.c ├── segment.c ├── static.c └── stats.c └── tests └── gcbench.nim /.gitignore: -------------------------------------------------------------------------------- 1 | nimcache/ 2 | *.exe 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # araqsgc 2 | A new GC for Nim. Supports shared memory, gives complete control over what is happening. Easily tweakable for your needs. 3 | 4 | ## Abandoned 5 | 6 | This project is abandoned, all ideas and developments have become part of Nim's ``--gc:orc`` mode. 7 | 8 | -------------------------------------------------------------------------------- /araqsgc.nim: -------------------------------------------------------------------------------- 1 | ## Shared memory GC for Nim. 2 | ## Does not hide anything from you, you remain in complete control. 3 | ## (c) 2019 Andreas Rumpf 4 | 5 | when not defined(gcDestructors): 6 | {.error: "Compile with --gc:destructors".} 7 | 8 | # Note: Do not compile the -override.*.c files, 9 | # thankfully we don't need them. 10 | {.compile: "mimalloc/alloc-aligned.c".} 11 | 12 | {.compile: "mimalloc/alloc.c".} 13 | {.compile: "mimalloc/heap.c".} 14 | {.compile: "mimalloc/init.c".} 15 | {.compile: "mimalloc/memory.c".} 16 | {.compile: "mimalloc/os.c".} 17 | {.compile: "mimalloc/page.c".} 18 | {.compile: "mimalloc/segment.c".} 19 | {.compile: "mimalloc/stats.c".} 20 | 21 | # options.c seems silly, find a way to avoid it: 22 | {.compile: "mimalloc/options.c".} 23 | 24 | proc mi_zalloc_small(size: int): pointer {.importc, cdecl.} 25 | proc mi_free(p: pointer) {.importc, cdecl.} 26 | 27 | type 28 | MiHeap = object 29 | VisitHeapCallback = proc (heap: ptr MiHeap, area: pointer, blck: pointer, 30 | blockSize: int, arg: pointer): bool {. 31 | cdecl, tags: [], raises: [], gcsafe.} 32 | 33 | proc mi_heap_visit_blocks(heap: ptr MiHeap, visitAllBlocks: bool; 34 | visitor: VisitHeapCallback; arg: pointer): bool {.importc, cdecl.} 35 | proc mi_heap_contains_block(heap: ptr MiHeap; p: pointer): bool {.importc, cdecl.} 36 | 37 | proc mi_heap_get_default(): ptr MiHeap {.importc, cdecl.} 38 | 39 | type 40 | RefHeader = object 41 | epoch: int 42 | typ: PNimType 43 | 44 | Finalizer {.compilerproc.} = proc (self: pointer) {.nimcall, tags: [], raises: [], gcsafe.} 45 | 46 | template `+!`(p: pointer, s: int): pointer = 47 | cast[pointer](cast[int](p) +% s) 48 | 49 | template `-!`(p: pointer, s: int): pointer = 50 | cast[pointer](cast[int](p) -% s) 51 | 52 | template head(p: pointer): ptr RefHeader = 53 | cast[ptr RefHeader](cast[int](p) -% sizeof(RefHeader)) 54 | 55 | 56 | var 57 | usedMem*, threshold: int 58 | epoch: int 59 | 60 | proc newObjImpl(typ: PNimType, size: int): pointer {.nimcall, tags: [], raises: [], gcsafe.} = 61 | let s = size + sizeof(RefHeader) 62 | result = mi_zalloc_small(s) 63 | var p = cast[ptr RefHeader](result) 64 | p.typ = typ 65 | #p.epoch = epoch 66 | atomicInc usedMem, s 67 | result = result +! sizeof(RefHeader) 68 | 69 | proc rawDispose(p: pointer) = 70 | assert p != nil 71 | let h = head(p) 72 | assert h != nil 73 | assert h.typ != nil 74 | if h.typ.finalizer != nil: 75 | (cast[Finalizer](h.typ.finalizer))(p) 76 | atomicDec usedMem, h.typ.base.size + sizeof(RefHeader) 77 | mi_free(p -! sizeof(RefHeader)) 78 | 79 | proc shouldCollect*(): bool {.inline.} = (usedMem >= threshold) 80 | 81 | proc visitBlock(heap: ptr MiHeap, area: pointer, p: pointer, 82 | blockSize: int, arg: pointer): bool {.cdecl.} = 83 | if p != nil: 84 | let h = cast[ptr RefHeader](p) 85 | if h.epoch < epoch: 86 | # was only traced in an earlier epoch, which means it's garbage: 87 | let toFree = cast[ptr seq[pointer]](arg) 88 | assert p != nil 89 | toFree[].add(p +! sizeof(RefHeader)) 90 | # do not return early: 91 | result = true 92 | 93 | proc sweep() = 94 | var toFree = newSeqOfCap[pointer](1_000_000) 95 | discard mi_heap_visit_blocks(mi_heap_get_default(), true, visitBlock, addr toFree) 96 | for p in toFree: rawDispose(p) 97 | 98 | proc collect*(steps: int) = 99 | inc epoch 100 | traverseGlobals() 101 | traverseThreadLocals() 102 | sweep() 103 | 104 | proc dispose*[T](p: ref T) {.inline.} = 105 | # free a single object. Also calls ``T``'s destructor before. 106 | rawDispose(cast[pointer](p)) 107 | 108 | proc rawDeepDispose(p: pointer) = 109 | let h = head(p) 110 | let m = h.typ.marker 111 | if m != nil: 112 | m(p, 1) 113 | rawDispose(p) 114 | 115 | proc trace(p: pointer) = 116 | let h = head(p) 117 | if h.epoch != epoch: 118 | assert h.typ != nil 119 | h.epoch = epoch 120 | let m = h.typ.marker 121 | if m != nil: m(p, 0) 122 | 123 | proc traverseObjImpl*(p: pointer, op: int) {.nimcall, tags: [], raises: [], gcsafe.} = 124 | if p != nil: 125 | if op != 1: 126 | trace(p) 127 | else: 128 | # free stuff 129 | rawDeepDispose(p) 130 | 131 | proc deepDispose*[T](p: ref T) {.inline.} = 132 | rawDeepDispose(cast[pointer](p)) 133 | 134 | system.newObjHook = newObjImpl 135 | system.traverseObjHook = traverseObjImpl 136 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Andreas Rumpf 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /mimalloc/alloc-aligned.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | #include "mimalloc.h" 9 | #include "mimalloc-internal.h" 10 | 11 | #include // memset 12 | 13 | // ------------------------------------------------------ 14 | // Aligned Allocation 15 | // ------------------------------------------------------ 16 | 17 | static void* mi_heap_malloc_zero_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { 18 | // note: we don't require `size > offset`, we just guarantee that 19 | // the address at offset is aligned regardless of the allocated size. 20 | mi_assert(alignment > 0 && alignment % sizeof(uintptr_t) == 0); 21 | if (alignment <= sizeof(uintptr_t)) return _mi_heap_malloc_zero(heap,size,zero); 22 | if (size >= (SIZE_MAX - alignment)) return NULL; // overflow 23 | 24 | // try if there is a current small block with just the right alignment 25 | if (size <= MI_SMALL_SIZE_MAX) { 26 | mi_page_t* page = _mi_heap_get_free_small_page(heap,size); 27 | if (page->free != NULL && 28 | (((uintptr_t)page->free + offset) % alignment) == 0) 29 | { 30 | #if MI_STAT>1 31 | mi_heap_stat_increase( heap, malloc, size); 32 | #endif 33 | void* p = _mi_page_malloc(heap,page,size); 34 | mi_assert_internal(p != NULL); 35 | mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); 36 | if (zero) memset(p,0,size); 37 | return p; 38 | } 39 | } 40 | 41 | // otherwise over-allocate 42 | void* p = _mi_heap_malloc_zero(heap, size + alignment - 1, zero); 43 | if (p == NULL) return NULL; 44 | 45 | // .. and align within the allocation 46 | _mi_ptr_page(p)->flags.has_aligned = true; 47 | uintptr_t adjust = alignment - (((uintptr_t)p + offset) % alignment); 48 | mi_assert_internal(adjust % sizeof(uintptr_t) == 0); 49 | void* aligned_p = (adjust == alignment ? p : (void*)((uintptr_t)p + adjust)); 50 | mi_assert_internal(((uintptr_t)aligned_p + offset) % alignment == 0); 51 | mi_assert_internal( p == _mi_page_ptr_unalign(_mi_ptr_segment(aligned_p),_mi_ptr_page(aligned_p),aligned_p) ); 52 | return aligned_p; 53 | } 54 | 55 | 56 | void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 57 | return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, false); 58 | } 59 | 60 | void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { 61 | return mi_heap_malloc_aligned_at(heap, size, alignment, 0); 62 | } 63 | 64 | void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 65 | return mi_heap_malloc_zero_aligned_at(heap, size, alignment, offset, true); 66 | } 67 | 68 | void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept { 69 | return mi_heap_zalloc_aligned_at(heap, size, alignment, 0); 70 | } 71 | 72 | void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 73 | size_t total; 74 | if (mi_mul_overflow(count, size, &total)) return NULL; 75 | return mi_heap_zalloc_aligned_at(heap, total, alignment, offset); 76 | } 77 | 78 | void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept { 79 | return mi_heap_calloc_aligned_at(heap,count,size,alignment,0); 80 | } 81 | 82 | void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 83 | return mi_heap_malloc_aligned_at(mi_get_default_heap(), size, alignment, offset); 84 | } 85 | 86 | void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { 87 | return mi_heap_malloc_aligned(mi_get_default_heap(), size, alignment); 88 | } 89 | 90 | void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 91 | return mi_heap_zalloc_aligned_at(mi_get_default_heap(), size, alignment, offset); 92 | } 93 | 94 | void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept { 95 | return mi_heap_zalloc_aligned(mi_get_default_heap(), size, alignment); 96 | } 97 | 98 | void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { 99 | return mi_heap_calloc_aligned_at(mi_get_default_heap(), count, size, alignment, offset); 100 | } 101 | 102 | void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept { 103 | return mi_heap_calloc_aligned(mi_get_default_heap(), count, size, alignment); 104 | } 105 | 106 | 107 | static void* mi_heap_realloc_zero_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { 108 | mi_assert(alignment > 0); 109 | if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero); 110 | if (p == NULL) return mi_heap_malloc_zero_aligned_at(heap,newsize,alignment,offset,zero); 111 | size_t size = mi_usable_size(p); 112 | if (newsize <= size && newsize >= (size - (size / 2)) 113 | && (((uintptr_t)p + offset) % alignment) == 0) { 114 | return p; // reallocation still fits, is aligned and not more than 50% waste 115 | } 116 | else { 117 | void* newp = mi_heap_malloc_aligned_at(heap,newsize,alignment,offset); 118 | if (newp != NULL) { 119 | if (zero && newsize > size) { 120 | // also set last word in the previous allocation to zero to ensure any padding is zero-initialized 121 | size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); 122 | memset((uint8_t*)newp + start, 0, newsize - start); 123 | } 124 | memcpy(newp, p, (newsize > size ? size : newsize)); 125 | mi_free(p); // only free if successful 126 | } 127 | return newp; 128 | } 129 | } 130 | 131 | static void* mi_heap_realloc_zero_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept { 132 | mi_assert(alignment > 0); 133 | if (alignment <= sizeof(uintptr_t)) return _mi_heap_realloc_zero(heap,p,newsize,zero); 134 | size_t offset = ((uintptr_t)p % alignment); // use offset of previous allocation (p can be NULL) 135 | return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,zero); 136 | } 137 | 138 | void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { 139 | return mi_heap_realloc_zero_aligned_at(heap,p,newsize,alignment,offset,false); 140 | } 141 | 142 | void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept { 143 | return mi_heap_realloc_zero_aligned(heap,p,newsize,alignment,false); 144 | } 145 | 146 | void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept { 147 | return mi_heap_realloc_aligned_at(mi_get_default_heap(), p, newsize, alignment, offset); 148 | } 149 | 150 | void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept { 151 | return mi_heap_realloc_aligned(mi_get_default_heap(), p, newsize, alignment); 152 | } 153 | -------------------------------------------------------------------------------- /mimalloc/alloc-override-osx.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | #include "mimalloc.h" 9 | #include "mimalloc-internal.h" 10 | 11 | #if defined(MI_MALLOC_OVERRIDE) 12 | 13 | #if !defined(__APPLE__) 14 | #error "this file should only be included on macOS" 15 | #endif 16 | 17 | /* ------------------------------------------------------ 18 | Override system malloc on macOS 19 | This is done through the malloc zone interface. 20 | ------------------------------------------------------ */ 21 | 22 | #include 23 | #include 24 | #include // memset 25 | 26 | #if defined(MAC_OS_X_VERSION_10_6) && \ 27 | MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 28 | // only available from OSX 10.6 29 | extern malloc_zone_t* malloc_default_purgeable_zone(void) __attribute__((weak_import)); 30 | #endif 31 | 32 | 33 | /* ------------------------------------------------------ 34 | malloc zone members 35 | ------------------------------------------------------ */ 36 | 37 | static size_t zone_size(malloc_zone_t* zone, const void* p) { 38 | return 0; // as we cannot guarantee that `p` comes from us, just return 0 39 | } 40 | 41 | static void* zone_malloc(malloc_zone_t* zone, size_t size) { 42 | return mi_malloc(size); 43 | } 44 | 45 | static void* zone_calloc(malloc_zone_t* zone, size_t count, size_t size) { 46 | return mi_calloc(count, size); 47 | } 48 | 49 | static void* zone_valloc(malloc_zone_t* zone, size_t size) { 50 | return mi_malloc_aligned(size, _mi_os_page_size()); 51 | } 52 | 53 | static void zone_free(malloc_zone_t* zone, void* p) { 54 | return mi_free(p); 55 | } 56 | 57 | static void* zone_realloc(malloc_zone_t* zone, void* p, size_t newsize) { 58 | return mi_realloc(p, newsize); 59 | } 60 | 61 | static void* zone_memalign(malloc_zone_t* zone, size_t alignment, size_t size) { 62 | return mi_malloc_aligned(size,alignment); 63 | } 64 | 65 | static void zone_destroy(malloc_zone_t* zone) { 66 | // todo: ignore for now? 67 | } 68 | 69 | static unsigned zone_batch_malloc(malloc_zone_t* zone, size_t size, void** ps, unsigned count) { 70 | size_t i; 71 | for (i = 0; i < count; i++) { 72 | ps[i] = zone_malloc(zone, size); 73 | if (ps[i] == NULL) break; 74 | } 75 | return i; 76 | } 77 | 78 | static void zone_batch_free(malloc_zone_t* zone, void** ps, unsigned count) { 79 | for(size_t i = 0; i < count; i++) { 80 | zone_free(zone, ps[i]); 81 | ps[i] = NULL; 82 | } 83 | } 84 | 85 | static size_t zone_pressure_relief(malloc_zone_t* zone, size_t size) { 86 | mi_collect(false); 87 | return 0; 88 | } 89 | 90 | static void zone_free_definite_size(malloc_zone_t* zone, void* p, size_t size) { 91 | zone_free(zone,p); 92 | } 93 | 94 | 95 | /* ------------------------------------------------------ 96 | Introspection members 97 | ------------------------------------------------------ */ 98 | 99 | static kern_return_t intro_enumerator(task_t task, void* p, 100 | unsigned type_mask, vm_address_t zone_address, 101 | memory_reader_t reader, 102 | vm_range_recorder_t recorder) 103 | { 104 | // todo: enumerate all memory 105 | return KERN_SUCCESS; 106 | } 107 | 108 | static size_t intro_good_size(malloc_zone_t* zone, size_t size) { 109 | return mi_good_size(size); 110 | } 111 | 112 | static boolean_t intro_check(malloc_zone_t* zone) { 113 | return true; 114 | } 115 | 116 | static void intro_print(malloc_zone_t* zone, boolean_t verbose) { 117 | mi_stats_print(NULL); 118 | } 119 | 120 | static void intro_log(malloc_zone_t* zone, void* p) { 121 | // todo? 122 | } 123 | 124 | static void intro_force_lock(malloc_zone_t* zone) { 125 | // todo? 126 | } 127 | 128 | static void intro_force_unlock(malloc_zone_t* zone) { 129 | // todo? 130 | } 131 | 132 | static void intro_statistics(malloc_zone_t* zone, malloc_statistics_t* stats) { 133 | // todo... 134 | stats->blocks_in_use = 0; 135 | stats->size_in_use = 0; 136 | stats->max_size_in_use = 0; 137 | stats->size_allocated = 0; 138 | } 139 | 140 | static boolean_t intro_zone_locked(malloc_zone_t* zone) { 141 | return false; 142 | } 143 | 144 | 145 | /* ------------------------------------------------------ 146 | At process start, override the default allocator 147 | ------------------------------------------------------ */ 148 | 149 | static malloc_zone_t* mi_get_default_zone() 150 | { 151 | // The first returned zone is the real default 152 | malloc_zone_t** zones = NULL; 153 | unsigned count = 0; 154 | kern_return_t ret = malloc_get_all_zones(0, NULL, (vm_address_t**)&zones, &count); 155 | if (ret == KERN_SUCCESS && count > 0) { 156 | return zones[0]; 157 | } 158 | else { 159 | // fallback 160 | return malloc_default_zone(); 161 | } 162 | } 163 | 164 | 165 | static void __attribute__((constructor)) _mi_macos_override_malloc() 166 | { 167 | static malloc_introspection_t intro; 168 | memset(&intro, 0, sizeof(intro)); 169 | 170 | intro.enumerator = &intro_enumerator; 171 | intro.good_size = &intro_good_size; 172 | intro.check = &intro_check; 173 | intro.print = &intro_print; 174 | intro.log = &intro_log; 175 | intro.force_lock = &intro_force_lock; 176 | intro.force_unlock = &intro_force_unlock; 177 | 178 | static malloc_zone_t zone; 179 | memset(&zone, 0, sizeof(zone)); 180 | 181 | zone.version = 4; 182 | zone.zone_name = "mimalloc"; 183 | zone.size = &zone_size; 184 | zone.introspect = &intro; 185 | zone.malloc = &zone_malloc; 186 | zone.calloc = &zone_calloc; 187 | zone.valloc = &zone_valloc; 188 | zone.free = &zone_free; 189 | zone.realloc = &zone_realloc; 190 | zone.destroy = &zone_destroy; 191 | zone.batch_malloc = &zone_batch_malloc; 192 | zone.batch_free = &zone_batch_free; 193 | 194 | malloc_zone_t* purgeable_zone = NULL; 195 | 196 | #if defined(MAC_OS_X_VERSION_10_6) && \ 197 | MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6 198 | // switch to version 9 on OSX 10.6 to support memalign. 199 | zone.version = 9; 200 | zone.memalign = &zone_memalign; 201 | zone.free_definite_size = &zone_free_definite_size; 202 | zone.pressure_relief = &zone_pressure_relief; 203 | intro.zone_locked = &intro_zone_locked; 204 | 205 | // force the purgeable zone to exist to avoid strange bugs 206 | if (malloc_default_purgeable_zone) { 207 | purgeable_zone = malloc_default_purgeable_zone(); 208 | } 209 | #endif 210 | 211 | // Register our zone 212 | malloc_zone_register(&zone); 213 | 214 | // Unregister the default zone, this makes our zone the new default 215 | // as that was the last registered. 216 | malloc_zone_t *default_zone = mi_get_default_zone(); 217 | malloc_zone_unregister(default_zone); 218 | 219 | // Reregister the default zone so free and realloc in that zone keep working. 220 | malloc_zone_register(default_zone); 221 | 222 | // Unregister, and re-register the purgeable_zone to avoid bugs if it occurs 223 | // earlier than the default zone. 224 | if (purgeable_zone != NULL) { 225 | malloc_zone_unregister(purgeable_zone); 226 | malloc_zone_register(purgeable_zone); 227 | } 228 | } 229 | 230 | #endif // MI_MALLOC_OVERRIDE 231 | -------------------------------------------------------------------------------- /mimalloc/alloc-override.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | #if !defined(MI_IN_ALLOC_C) 9 | #error "this file should be included from 'alloc.c' (so aliases can work)" 10 | #endif 11 | 12 | #if defined(MI_MALLOC_OVERRIDE) && defined(_WIN32) && !(defined(MI_SHARED_LIB) && defined(_DLL)) 13 | #error "It is only possible to override malloc on Windows when building as a DLL (and linking the C runtime as a DLL)" 14 | #endif 15 | 16 | #if defined(MI_MALLOC_OVERRIDE) && !defined(_WIN32) 17 | 18 | // ------------------------------------------------------ 19 | // Override system malloc 20 | // ------------------------------------------------------ 21 | 22 | #if defined(_MSC_VER) 23 | #pragma warning(disable:4273) // inconsistent dll linking 24 | #endif 25 | 26 | #if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) 27 | // use aliasing to alias the exported function to one of our `mi_` functions 28 | #if (defined(__GNUC__) && __GNUC__ >= 9) 29 | #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"), copy(fun))) 30 | #else 31 | #define MI_FORWARD(fun) __attribute__((alias(#fun), used, visibility("default"))) 32 | #endif 33 | #define MI_FORWARD1(fun,x) MI_FORWARD(fun) 34 | #define MI_FORWARD2(fun,x,y) MI_FORWARD(fun) 35 | #define MI_FORWARD3(fun,x,y,z) MI_FORWARD(fun) 36 | #define MI_FORWARD0(fun,x) MI_FORWARD(fun) 37 | #define MI_FORWARD02(fun,x,y) MI_FORWARD(fun) 38 | #else 39 | // use forwarding by calling our `mi_` function 40 | #define MI_FORWARD1(fun,x) { return fun(x); } 41 | #define MI_FORWARD2(fun,x,y) { return fun(x,y); } 42 | #define MI_FORWARD3(fun,x,y,z) { return fun(x,y,z); } 43 | #define MI_FORWARD0(fun,x) { fun(x); } 44 | #define MI_FORWARD02(fun,x,y) { fun(x,y); } 45 | #endif 46 | 47 | #if defined(__APPLE__) && defined(MI_SHARED_LIB_EXPORT) && defined(MI_INTERPOSE) 48 | // use interposing so `DYLD_INSERT_LIBRARIES` works without `DYLD_FORCE_FLAT_NAMESPACE=1` 49 | // See: 50 | struct mi_interpose_s { 51 | const void* replacement; 52 | const void* target; 53 | }; 54 | #define MI_INTERPOSEX(oldfun,newfun) { (const void*)&newfun, (const void*)&oldfun } 55 | #define MI_INTERPOSE_MI(fun) MI_INTERPOSEX(fun,mi_##fun) 56 | __attribute__((used)) static struct mi_interpose_s _mi_interposes[] __attribute__((section("__DATA, __interpose"))) = 57 | { 58 | MI_INTERPOSE_MI(malloc), 59 | MI_INTERPOSE_MI(calloc), 60 | MI_INTERPOSE_MI(realloc), 61 | MI_INTERPOSE_MI(free), 62 | MI_INTERPOSE_MI(strdup), 63 | MI_INTERPOSE_MI(strndup) 64 | }; 65 | #else 66 | // On all other systems forward to our API 67 | void* malloc(size_t size) mi_attr_noexcept MI_FORWARD1(mi_malloc, size); 68 | void* calloc(size_t size, size_t n) mi_attr_noexcept MI_FORWARD2(mi_calloc, size, n); 69 | void* realloc(void* p, size_t newsize) mi_attr_noexcept MI_FORWARD2(mi_realloc, p, newsize); 70 | void free(void* p) mi_attr_noexcept MI_FORWARD0(mi_free, p); 71 | #endif 72 | 73 | #if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) 74 | #pragma GCC visibility push(default) 75 | #endif 76 | 77 | // ------------------------------------------------------ 78 | // Override new/delete 79 | // This is not really necessary as they usually call 80 | // malloc/free anyway, but it improves performance. 81 | // ------------------------------------------------------ 82 | #ifdef __cplusplus 83 | // ------------------------------------------------------ 84 | // With a C++ compiler we override the new/delete operators. 85 | // see 86 | // ------------------------------------------------------ 87 | #include 88 | void operator delete(void* p) noexcept MI_FORWARD0(mi_free,p); 89 | void operator delete[](void* p) noexcept MI_FORWARD0(mi_free,p); 90 | 91 | void* operator new(std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n); 92 | void* operator new[](std::size_t n) noexcept(false) MI_FORWARD1(mi_new,n); 93 | 94 | void* operator new (std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } 95 | void* operator new[](std::size_t n, const std::nothrow_t& tag) noexcept { UNUSED(tag); return mi_new_nothrow(n); } 96 | 97 | #if (__cplusplus >= 201402L) 98 | void operator delete (void* p, std::size_t n) MI_FORWARD02(mi_free_size,p,n); 99 | void operator delete[](void* p, std::size_t n) MI_FORWARD02(mi_free_size,p,n); 100 | #endif 101 | 102 | #if (__cplusplus > 201402L || defined(__cpp_aligned_new)) 103 | void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } 104 | void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } 105 | void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; 106 | void operator delete[](void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; 107 | 108 | void* operator new( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } 109 | void* operator new[]( std::size_t n, std::align_val_t al) noexcept(false) { return mi_new_aligned(n, static_cast(al)); } 110 | void* operator new (std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } 111 | void* operator new[](std::size_t n, std::align_val_t al, const std::nothrow_t&) noexcept { return mi_new_aligned_nothrow(n, static_cast(al)); } 112 | #endif 113 | 114 | #elif (defined(__GNUC__) || defined(__clang__)) 115 | // ------------------------------------------------------ 116 | // Override by defining the mangled C++ names of the operators (as 117 | // used by GCC and CLang). 118 | // See 119 | // ------------------------------------------------------ 120 | void _ZdlPv(void* p) MI_FORWARD0(mi_free,p); // delete 121 | void _ZdaPv(void* p) MI_FORWARD0(mi_free,p); // delete[] 122 | void _ZdlPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n); 123 | void _ZdaPvm(void* p, size_t n) MI_FORWARD02(mi_free_size,p,n); 124 | void _ZdlPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } 125 | void _ZdaPvSt11align_val_t(void* p, size_t al) { mi_free_aligned(p,al); } 126 | void _ZdlPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } 127 | void _ZdaPvmSt11align_val_t(void* p, size_t n, size_t al) { mi_free_size_aligned(p,n,al); } 128 | 129 | typedef struct mi_nothrow_s { } mi_nothrow_t; 130 | #if (MI_INTPTR_SIZE==8) 131 | void* _Znwm(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit 132 | void* _Znam(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit 133 | void* _ZnwmSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); 134 | void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); 135 | void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } 136 | void* _ZnamRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } 137 | void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } 138 | void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } 139 | #elif (MI_INTPTR_SIZE==4) 140 | void* _Znwj(size_t n) MI_FORWARD1(mi_new,n); // new 64-bit 141 | void* _Znaj(size_t n) MI_FORWARD1(mi_new,n); // new[] 64-bit 142 | void* _ZnwjSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); 143 | void* _ZnajSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al); 144 | void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } 145 | void* _ZnajRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { UNUSED(tag); return mi_new_nothrow(n); } 146 | void* _ZnwjSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } 147 | void* _ZnajSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { UNUSED(tag); return mi_new_aligned_nothrow(n,al); } 148 | #else 149 | #error "define overloads for new/delete for this platform (just for performance, can be skipped)" 150 | #endif 151 | #endif // __cplusplus 152 | 153 | 154 | #ifdef __cplusplus 155 | extern "C" { 156 | #endif 157 | 158 | // ------------------------------------------------------ 159 | // Posix & Unix functions definitions 160 | // ------------------------------------------------------ 161 | 162 | void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize); 163 | size_t malloc_size(void* p) MI_FORWARD1(mi_usable_size,p); 164 | size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p); 165 | void cfree(void* p) MI_FORWARD0(mi_free, p); 166 | 167 | // no forwarding here due to aliasing/name mangling issues 168 | void* valloc(size_t size) { return mi_valloc(size); } 169 | void* pvalloc(size_t size) { return mi_pvalloc(size); } 170 | void* reallocarray(void* p, size_t count, size_t size) { return mi_reallocarray(p, count, size); } 171 | void* memalign(size_t alignment, size_t size) { return mi_memalign(alignment, size); } 172 | void* aligned_alloc(size_t alignment, size_t size) { return mi_aligned_alloc(alignment, size); } 173 | int posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p, alignment, size); } 174 | 175 | #if defined(__GLIBC__) && defined(__linux__) 176 | // forward __libc interface (needed for glibc-based Linux distributions) 177 | void* __libc_malloc(size_t size) MI_FORWARD1(mi_malloc,size); 178 | void* __libc_calloc(size_t count, size_t size) MI_FORWARD2(mi_calloc,count,size); 179 | void* __libc_realloc(void* p, size_t size) MI_FORWARD2(mi_realloc,p,size); 180 | void __libc_free(void* p) MI_FORWARD0(mi_free,p); 181 | void __libc_cfree(void* p) MI_FORWARD0(mi_free,p); 182 | 183 | void* __libc_valloc(size_t size) { return mi_valloc(size); } 184 | void* __libc_pvalloc(size_t size) { return mi_pvalloc(size); } 185 | void* __libc_memalign(size_t alignment, size_t size) { return mi_memalign(alignment,size); } 186 | int __posix_memalign(void** p, size_t alignment, size_t size) { return mi_posix_memalign(p,alignment,size); } 187 | #endif 188 | 189 | #ifdef __cplusplus 190 | } 191 | #endif 192 | 193 | #if (defined(__GNUC__) || defined(__clang__)) && !defined(__MACH__) 194 | #pragma GCC visibility pop 195 | #endif 196 | 197 | #endif // MI_MALLOC_OVERRIDE & !_WIN32 198 | -------------------------------------------------------------------------------- /mimalloc/alloc-posix.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018,2019, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | // ------------------------------------------------------------------------ 9 | // mi prefixed publi definitions of various Posix, Unix, and C++ functions 10 | // for convenience and used when overriding these functions. 11 | // ------------------------------------------------------------------------ 12 | 13 | #include "mimalloc.h" 14 | #include "mimalloc-internal.h" 15 | 16 | // ------------------------------------------------------ 17 | // Posix & Unix functions definitions 18 | // ------------------------------------------------------ 19 | 20 | #include 21 | 22 | #ifndef EINVAL 23 | #define EINVAL 22 24 | #endif 25 | #ifndef ENOMEM 26 | #define ENOMEM 12 27 | #endif 28 | 29 | 30 | size_t mi_malloc_size(const void* p) mi_attr_noexcept { 31 | return mi_usable_size(p); 32 | } 33 | 34 | size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { 35 | return mi_usable_size(p); 36 | } 37 | 38 | void mi_cfree(void* p) mi_attr_noexcept { 39 | mi_free(p); 40 | } 41 | 42 | int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept { 43 | // Note: The spec dictates we should not modify `*p` on an error. (issue#27) 44 | // 45 | if (p == NULL) return EINVAL; 46 | if (alignment % sizeof(void*) != 0) return EINVAL; // natural alignment 47 | if ((alignment & (alignment - 1)) != 0) return EINVAL; // not a power of 2 48 | void* q = mi_malloc_aligned(size, alignment); 49 | if (q==NULL && size != 0) return ENOMEM; 50 | *p = q; 51 | return 0; 52 | } 53 | 54 | int mi__posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept { 55 | return mi_posix_memalign(p, alignment, size); 56 | } 57 | 58 | void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept { 59 | return mi_malloc_aligned(size, alignment); 60 | } 61 | 62 | void* mi_valloc(size_t size) mi_attr_noexcept { 63 | return mi_malloc_aligned(size, _mi_os_page_size()); 64 | } 65 | 66 | void* mi_pvalloc(size_t size) mi_attr_noexcept { 67 | size_t psize = _mi_os_page_size(); 68 | if (size >= SIZE_MAX - psize) return NULL; // overflow 69 | size_t asize = ((size + psize - 1) / psize) * psize; 70 | return mi_malloc_aligned(asize, psize); 71 | } 72 | 73 | void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept { 74 | return mi_malloc_aligned(size, alignment); 75 | } 76 | 77 | void* mi_reallocarray( void* p, size_t count, size_t size ) mi_attr_noexcept { // BSD 78 | void* newp = mi_reallocn(p,count,size); 79 | if (newp==NULL) errno = ENOMEM; 80 | return newp; 81 | } 82 | 83 | -------------------------------------------------------------------------------- /mimalloc/alloc.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #include "mimalloc.h" 8 | #include "mimalloc-internal.h" 9 | #include "mimalloc-atomic.h" 10 | 11 | #include // memset 12 | 13 | #define MI_IN_ALLOC_C 14 | #include "alloc-override.c" 15 | #undef MI_IN_ALLOC_C 16 | 17 | // ------------------------------------------------------ 18 | // Allocation 19 | // ------------------------------------------------------ 20 | 21 | // Fast allocation in a page: just pop from the free list. 22 | // Fall back to generic allocation only if the list is empty. 23 | extern inline void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept { 24 | mi_assert_internal(page->block_size==0||page->block_size >= size); 25 | mi_block_t* block = page->free; 26 | if (mi_unlikely(block == NULL)) { 27 | return _mi_malloc_generic(heap, size); // slow path 28 | } 29 | mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); 30 | // pop from the free list 31 | page->free = mi_block_next(page,block); 32 | page->used++; 33 | mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); 34 | #if (MI_DEBUG) 35 | memset(block, MI_DEBUG_UNINIT, size); 36 | #elif (MI_SECURE) 37 | block->next = 0; 38 | #endif 39 | #if (MI_STAT>1) 40 | if(size <= MI_LARGE_SIZE_MAX) { 41 | size_t bin = _mi_bin(size); 42 | mi_heap_stat_increase(heap,normal[bin], 1); 43 | } 44 | #endif 45 | return block; 46 | } 47 | 48 | // allocate a small block 49 | extern inline void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { 50 | mi_assert(size <= MI_SMALL_SIZE_MAX); 51 | mi_page_t* page = _mi_heap_get_free_small_page(heap,size); 52 | return _mi_page_malloc(heap, page, size); 53 | } 54 | 55 | extern inline void* mi_malloc_small(size_t size) mi_attr_noexcept { 56 | return mi_heap_malloc_small(mi_get_default_heap(), size); 57 | } 58 | 59 | // zero initialized small block 60 | void* mi_zalloc_small(size_t size) mi_attr_noexcept { 61 | void* p = mi_malloc_small(size); 62 | if (p != NULL) { memset(p, 0, size); } 63 | return p; 64 | } 65 | 66 | // The main allocation function 67 | extern inline void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { 68 | mi_assert(heap!=NULL); 69 | mi_assert(heap->thread_id == 0 || heap->thread_id == _mi_thread_id()); // heaps are thread local 70 | void* p; 71 | if (mi_likely(size <= MI_SMALL_SIZE_MAX)) { 72 | p = mi_heap_malloc_small(heap, size); 73 | } 74 | else { 75 | p = _mi_malloc_generic(heap, size); 76 | } 77 | #if MI_STAT>1 78 | if (p != NULL) { 79 | if (!mi_heap_is_initialized(heap)) { heap = mi_get_default_heap(); } 80 | mi_heap_stat_increase( heap, malloc, mi_good_size(size) ); // overestimate for aligned sizes 81 | } 82 | #endif 83 | return p; 84 | } 85 | 86 | extern inline void* mi_malloc(size_t size) mi_attr_noexcept { 87 | return mi_heap_malloc(mi_get_default_heap(), size); 88 | } 89 | 90 | void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero) { 91 | void* p = mi_heap_malloc(heap,size); 92 | if (zero && p != NULL) memset(p,0,size); 93 | return p; 94 | } 95 | 96 | extern inline void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { 97 | return _mi_heap_malloc_zero(heap, size, true); 98 | } 99 | 100 | void* mi_zalloc(size_t size) mi_attr_noexcept { 101 | return mi_heap_zalloc(mi_get_default_heap(),size); 102 | } 103 | 104 | 105 | // ------------------------------------------------------ 106 | // Free 107 | // ------------------------------------------------------ 108 | 109 | // multi-threaded free 110 | static mi_decl_noinline void _mi_free_block_mt(mi_page_t* page, mi_block_t* block) 111 | { 112 | mi_thread_free_t tfree; 113 | mi_thread_free_t tfreex; 114 | bool use_delayed; 115 | 116 | do { 117 | tfree = page->thread_free; 118 | use_delayed = (mi_tf_delayed(tfree) == MI_USE_DELAYED_FREE || 119 | (mi_tf_delayed(tfree) == MI_NO_DELAYED_FREE && page->used == page->thread_freed+1) 120 | ); 121 | if (mi_unlikely(use_delayed)) { 122 | // unlikely: this only happens on the first concurrent free in a page that is in the full list 123 | tfreex = mi_tf_set_delayed(tfree,MI_DELAYED_FREEING); 124 | } 125 | else { 126 | // usual: directly add to page thread_free list 127 | mi_block_set_next(page, block, mi_tf_block(tfree)); 128 | tfreex = mi_tf_set_block(tfree,block); 129 | } 130 | } while (!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex, tfree)); 131 | 132 | if (mi_likely(!use_delayed)) { 133 | // increment the thread free count and return 134 | mi_atomic_increment(&page->thread_freed); 135 | } 136 | else { 137 | // racy read on `heap`, but ok because MI_DELAYED_FREEING is set (see `mi_heap_delete` and `mi_heap_collect_abandon`) 138 | mi_heap_t* heap = page->heap; 139 | mi_assert_internal(heap != NULL); 140 | if (heap != NULL) { 141 | // add to the delayed free list of this heap. (do this atomically as the lock only protects heap memory validity) 142 | mi_block_t* dfree; 143 | do { 144 | dfree = (mi_block_t*)heap->thread_delayed_free; 145 | mi_block_set_nextx(heap->cookie,block,dfree); 146 | } while (!mi_atomic_compare_exchange_ptr((volatile void**)&heap->thread_delayed_free, block, dfree)); 147 | } 148 | 149 | // and reset the MI_DELAYED_FREEING flag 150 | do { 151 | tfreex = tfree = page->thread_free; 152 | mi_assert_internal(mi_tf_delayed(tfree) == MI_NEVER_DELAYED_FREE || mi_tf_delayed(tfree) == MI_DELAYED_FREEING); 153 | if (mi_tf_delayed(tfree) != MI_NEVER_DELAYED_FREE) tfreex = mi_tf_set_delayed(tfree,MI_NO_DELAYED_FREE); 154 | } while (!mi_atomic_compare_exchange((volatile uintptr_t*)&page->thread_free, tfreex, tfree)); 155 | } 156 | } 157 | 158 | 159 | // regular free 160 | static inline void _mi_free_block(mi_page_t* page, bool local, mi_block_t* block) 161 | { 162 | #if (MI_DEBUG) 163 | memset(block, MI_DEBUG_FREED, page->block_size); 164 | #endif 165 | 166 | // and push it on the free list 167 | if (mi_likely(local)) { 168 | // owning thread can free a block directly 169 | mi_block_set_next(page, block, page->local_free); 170 | page->local_free = block; 171 | page->used--; 172 | if (mi_unlikely(mi_page_all_free(page))) { 173 | _mi_page_retire(page); 174 | } 175 | else if (mi_unlikely(page->flags.in_full)) { 176 | _mi_page_unfull(page); 177 | } 178 | } 179 | else { 180 | _mi_free_block_mt(page,block); 181 | } 182 | } 183 | 184 | 185 | // Adjust a block that was allocated aligned, to the actual start of the block in the page. 186 | mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p) { 187 | mi_assert_internal(page!=NULL && p!=NULL); 188 | size_t diff = (uint8_t*)p - _mi_page_start(segment, page, NULL); 189 | size_t adjust = (diff % page->block_size); 190 | return (mi_block_t*)((uintptr_t)p - adjust); 191 | } 192 | 193 | 194 | static void mi_decl_noinline mi_free_generic(const mi_segment_t* segment, mi_page_t* page, bool local, void* p) { 195 | mi_block_t* block = (page->flags.has_aligned ? _mi_page_ptr_unalign(segment, page, p) : (mi_block_t*)p); 196 | _mi_free_block(page, local, block); 197 | } 198 | 199 | // Free a block 200 | void mi_free(void* p) mi_attr_noexcept 201 | { 202 | // optimize: merge null check with the segment masking (below) 203 | //if (p == NULL) return; 204 | 205 | #if (MI_DEBUG>0) 206 | if (mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0)) { 207 | _mi_error_message("trying to free an invalid (unaligned) pointer: %p\n", p); 208 | return; 209 | } 210 | #endif 211 | 212 | const mi_segment_t* const segment = _mi_ptr_segment(p); 213 | if (segment == NULL) return; // checks for (p==NULL) 214 | bool local = (_mi_thread_id() == segment->thread_id); // preload, note: putting the thread_id in the page->flags does not improve performance 215 | 216 | #if (MI_DEBUG>0) 217 | if (mi_unlikely(_mi_ptr_cookie(segment) != segment->cookie)) { 218 | _mi_error_message("trying to mi_free a pointer that does not point to a valid heap space: %p\n", p); 219 | return; 220 | } 221 | #endif 222 | 223 | mi_page_t* page = _mi_segment_page_of(segment, p); 224 | 225 | #if (MI_STAT>1) 226 | mi_heap_t* heap = mi_heap_get_default(); 227 | mi_heap_stat_decrease( heap, malloc, mi_usable_size(p)); 228 | if (page->block_size <= MI_LARGE_SIZE_MAX) { 229 | mi_heap_stat_decrease( heap, normal[_mi_bin(page->block_size)], 1); 230 | } 231 | // huge page stat is accounted for in `_mi_page_retire` 232 | #endif 233 | 234 | // adjust if it might be an un-aligned block 235 | if (mi_likely(page->flags.value==0)) { // note: merging both tests (local | value) does not matter for performance 236 | mi_block_t* block = (mi_block_t*)p; 237 | if (mi_likely(local)) { 238 | // owning thread can free a block directly 239 | mi_block_set_next(page, block, page->local_free); // note: moving this write earlier does not matter for performance 240 | page->local_free = block; 241 | page->used--; 242 | if (mi_unlikely(mi_page_all_free(page))) { _mi_page_retire(page); } 243 | } 244 | else { 245 | // use atomic operations for a multi-threaded free 246 | _mi_free_block_mt(page, block); 247 | } 248 | } 249 | else { 250 | // aligned blocks, or a full page; use the more generic path 251 | mi_free_generic(segment, page, local, p); 252 | } 253 | } 254 | 255 | bool _mi_free_delayed_block(mi_block_t* block) { 256 | // get segment and page 257 | const mi_segment_t* segment = _mi_ptr_segment(block); 258 | mi_assert_internal(_mi_ptr_cookie(segment) == segment->cookie); 259 | mi_assert_internal(_mi_thread_id() == segment->thread_id); 260 | mi_page_t* page = _mi_segment_page_of(segment, block); 261 | if (mi_tf_delayed(page->thread_free) == MI_DELAYED_FREEING) { 262 | // we might already start delayed freeing while another thread has not yet 263 | // reset the delayed_freeing flag; in that case don't free it quite yet if 264 | // this is the last block remaining. 265 | if (page->used - page->thread_freed == 1) return false; 266 | } 267 | _mi_free_block(page,true,block); 268 | return true; 269 | } 270 | 271 | // Bytes available in a block 272 | size_t mi_usable_size(const void* p) mi_attr_noexcept { 273 | if (p==NULL) return 0; 274 | const mi_segment_t* segment = _mi_ptr_segment(p); 275 | const mi_page_t* page = _mi_segment_page_of(segment,p); 276 | size_t size = page->block_size; 277 | if (mi_unlikely(page->flags.has_aligned)) { 278 | ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)_mi_page_ptr_unalign(segment,page,p); 279 | mi_assert_internal(adjust >= 0 && (size_t)adjust <= size); 280 | return (size - adjust); 281 | } 282 | else { 283 | return size; 284 | } 285 | } 286 | 287 | 288 | // ------------------------------------------------------ 289 | // ensure explicit external inline definitions are emitted! 290 | // ------------------------------------------------------ 291 | 292 | #ifdef __cplusplus 293 | void* _mi_externs[] = { 294 | (void*)&_mi_page_malloc, 295 | (void*)&mi_malloc, 296 | (void*)&mi_malloc_small, 297 | (void*)&mi_heap_malloc, 298 | (void*)&mi_heap_zalloc, 299 | (void*)&mi_heap_malloc_small 300 | }; 301 | #endif 302 | 303 | 304 | // ------------------------------------------------------ 305 | // Allocation extensions 306 | // ------------------------------------------------------ 307 | 308 | void mi_free_size(void* p, size_t size) mi_attr_noexcept { 309 | UNUSED_RELEASE(size); 310 | mi_assert(size <= mi_usable_size(p)); 311 | mi_free(p); 312 | } 313 | 314 | void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { 315 | UNUSED_RELEASE(alignment); 316 | mi_assert(((uintptr_t)p % alignment) == 0); 317 | mi_free_size(p,size); 318 | } 319 | 320 | void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { 321 | UNUSED_RELEASE(alignment); 322 | mi_assert(((uintptr_t)p % alignment) == 0); 323 | mi_free(p); 324 | } 325 | 326 | extern inline void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { 327 | size_t total; 328 | if (mi_mul_overflow(count,size,&total)) return NULL; 329 | return mi_heap_zalloc(heap,total); 330 | } 331 | 332 | void* mi_calloc(size_t count, size_t size) mi_attr_noexcept { 333 | return mi_heap_calloc(mi_get_default_heap(),count,size); 334 | } 335 | 336 | // Uninitialized `calloc` 337 | extern void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept { 338 | size_t total; 339 | if (mi_mul_overflow(count,size,&total)) return NULL; 340 | return mi_heap_malloc(heap, total); 341 | } 342 | 343 | void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept { 344 | return mi_heap_mallocn(mi_get_default_heap(),count,size); 345 | } 346 | 347 | // Expand in place or fail 348 | void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { 349 | if (p == NULL) return NULL; 350 | size_t size = mi_usable_size(p); 351 | if (newsize > size) return NULL; 352 | return p; // it fits 353 | } 354 | 355 | void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero) { 356 | if (p == NULL) return _mi_heap_malloc_zero(heap,newsize,zero); 357 | size_t size = mi_usable_size(p); 358 | if (newsize <= size && newsize >= (size / 2)) { 359 | return p; // reallocation still fits and not more than 50% waste 360 | } 361 | void* newp = mi_heap_malloc(heap,newsize); 362 | if (mi_likely(newp != NULL)) { 363 | if (zero && newsize > size) { 364 | // also set last word in the previous allocation to zero to ensure any padding is zero-initialized 365 | size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); 366 | memset((uint8_t*)newp + start, 0, newsize - start); 367 | } 368 | memcpy(newp, p, (newsize > size ? size : newsize)); 369 | mi_free(p); // only free if successful 370 | } 371 | return newp; 372 | } 373 | 374 | void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { 375 | return _mi_heap_realloc_zero(heap, p, newsize, false); 376 | } 377 | 378 | void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept { 379 | size_t total; 380 | if (mi_mul_overflow(count, size, &total)) return NULL; 381 | return mi_heap_realloc(heap, p, total); 382 | } 383 | 384 | 385 | // Reallocate but free `p` on errors 386 | void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept { 387 | void* newp = mi_heap_realloc(heap, p, newsize); 388 | if (newp==NULL && p!=NULL) mi_free(p); 389 | return newp; 390 | } 391 | 392 | void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept { 393 | return mi_heap_realloc(mi_get_default_heap(),p,newsize); 394 | } 395 | 396 | void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept { 397 | size_t total; 398 | if (mi_mul_overflow(count, size, &total)) return NULL; 399 | return _mi_heap_realloc_zero(mi_get_default_heap(),p,total,true); 400 | } 401 | 402 | void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept { 403 | return mi_heap_reallocn(mi_get_default_heap(),p,count,size); 404 | } 405 | 406 | // Reallocate but free `p` on errors 407 | void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept { 408 | return mi_heap_reallocf(mi_get_default_heap(),p,newsize); 409 | } 410 | 411 | // ------------------------------------------------------ 412 | // strdup, strndup, and realpath 413 | // ------------------------------------------------------ 414 | 415 | // `strdup` using mi_malloc 416 | char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept { 417 | if (s == NULL) return NULL; 418 | size_t n = strlen(s); 419 | char* t = (char*)mi_heap_malloc(heap,n+1); 420 | if (t != NULL) memcpy(t, s, n + 1); 421 | return t; 422 | } 423 | 424 | char* mi_strdup(const char* s) mi_attr_noexcept { 425 | return mi_heap_strdup(mi_get_default_heap(), s); 426 | } 427 | 428 | // `strndup` using mi_malloc 429 | char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept { 430 | if (s == NULL) return NULL; 431 | size_t m = strlen(s); 432 | if (n > m) n = m; 433 | char* t = (char*)mi_heap_malloc(heap, n+1); 434 | if (t == NULL) return NULL; 435 | memcpy(t, s, n); 436 | t[n] = 0; 437 | return t; 438 | } 439 | 440 | char* mi_strndup(const char* s, size_t n) mi_attr_noexcept { 441 | return mi_heap_strndup(mi_get_default_heap(),s,n); 442 | } 443 | 444 | #ifndef __wasi__ 445 | // `realpath` using mi_malloc 446 | #ifdef _WIN32 447 | #ifndef PATH_MAX 448 | #define PATH_MAX MAX_PATH 449 | #endif 450 | #include 451 | #include 452 | char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { 453 | // todo: use GetFullPathNameW to allow longer file names 454 | char buf[PATH_MAX]; 455 | DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL); 456 | if (res == 0) { 457 | errno = GetLastError(); return NULL; 458 | } 459 | else if (res > PATH_MAX) { 460 | errno = EINVAL; return NULL; 461 | } 462 | else if (resolved_name != NULL) { 463 | return resolved_name; 464 | } 465 | else { 466 | return mi_heap_strndup(heap, buf, PATH_MAX); 467 | } 468 | } 469 | #else 470 | #include 471 | static size_t mi_path_max() { 472 | static size_t path_max = 0; 473 | if (path_max <= 0) { 474 | long m = pathconf("/",_PC_PATH_MAX); 475 | if (m <= 0) path_max = 4096; // guess 476 | else if (m < 256) path_max = 256; // at least 256 477 | else path_max = m; 478 | } 479 | return path_max; 480 | } 481 | 482 | char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept { 483 | if (resolved_name != NULL) { 484 | return realpath(fname,resolved_name); 485 | } 486 | else { 487 | size_t n = mi_path_max(); 488 | char* buf = (char*)mi_malloc(n+1); 489 | if (buf==NULL) return NULL; 490 | char* rname = realpath(fname,buf); 491 | char* result = mi_heap_strndup(heap,rname,n); // ok if `rname==NULL` 492 | mi_free(buf); 493 | return result; 494 | } 495 | } 496 | #endif 497 | 498 | char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept { 499 | return mi_heap_realpath(mi_get_default_heap(),fname,resolved_name); 500 | } 501 | #endif 502 | 503 | /*------------------------------------------------------- 504 | C++ new and new_aligned 505 | The standard requires calling into `get_new_handler` and 506 | throwing the bad_alloc exception on failure. If we compile 507 | with a C++ compiler we can implement this precisely. If we 508 | use a C compiler we cannot throw a `bad_alloc` exception 509 | but we call `exit` instead (i.e. not returning). 510 | -------------------------------------------------------*/ 511 | 512 | #ifdef __cplusplus 513 | #include 514 | static bool mi_try_new_handler(bool nothrow) { 515 | std::new_handler h = std::get_new_handler(); 516 | if (h==NULL) { 517 | if (!nothrow) throw std::bad_alloc(); 518 | return false; 519 | } 520 | else { 521 | h(); 522 | return true; 523 | } 524 | } 525 | #else 526 | #include 527 | #ifndef ENOMEM 528 | #define ENOMEM 12 529 | #endif 530 | typedef void (*std_new_handler_t)(); 531 | 532 | #if (defined(__GNUC__) || defined(__clang__)) 533 | std_new_handler_t __attribute((weak)) _ZSt15get_new_handlerv() { 534 | return NULL; 535 | } 536 | std_new_handler_t mi_get_new_handler() { 537 | return _ZSt15get_new_handlerv(); 538 | } 539 | #else 540 | std_new_handler_t mi_get_new_handler() { 541 | return NULL; 542 | } 543 | #endif 544 | 545 | static bool mi_try_new_handler(bool nothrow) { 546 | std_new_handler_t h = mi_get_new_handler(); 547 | if (h==NULL) { 548 | if (!nothrow) exit(ENOMEM); 549 | return false; 550 | } 551 | else { 552 | h(); 553 | return true; 554 | } 555 | } 556 | #endif 557 | 558 | static mi_decl_noinline void* mi_try_new(size_t n, bool nothrow ) { 559 | void* p = NULL; 560 | while(p == NULL && mi_try_new_handler(nothrow)) { 561 | p = mi_malloc(n); 562 | } 563 | return p; 564 | } 565 | 566 | void* mi_new(size_t n) { 567 | void* p = mi_malloc(n); 568 | if (mi_unlikely(p == NULL)) return mi_try_new(n,false); 569 | return p; 570 | } 571 | 572 | void* mi_new_aligned(size_t n, size_t alignment) { 573 | void* p; 574 | do { p = mi_malloc_aligned(n, alignment); } 575 | while(p == NULL && mi_try_new_handler(false)); 576 | return p; 577 | } 578 | 579 | void* mi_new_nothrow(size_t n) { 580 | void* p = mi_malloc(n); 581 | if (mi_unlikely(p == NULL)) return mi_try_new(n,true); 582 | return p; 583 | } 584 | 585 | void* mi_new_aligned_nothrow(size_t n, size_t alignment) { 586 | void* p; 587 | do { p = mi_malloc_aligned(n, alignment); } 588 | while (p == NULL && mi_try_new_handler(true)); 589 | return p; 590 | } 591 | -------------------------------------------------------------------------------- /mimalloc/heap.c: -------------------------------------------------------------------------------- 1 | /*---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | #include "mimalloc.h" 9 | #include "mimalloc-internal.h" 10 | #include "mimalloc-atomic.h" 11 | 12 | #include // memset, memcpy 13 | 14 | 15 | /* ----------------------------------------------------------- 16 | Helpers 17 | ----------------------------------------------------------- */ 18 | 19 | // return `true` if ok, `false` to break 20 | typedef bool (heap_page_visitor_fun)(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2); 21 | 22 | // Visit all pages in a heap; returns `false` if break was called. 23 | static bool mi_heap_visit_pages(mi_heap_t* heap, heap_page_visitor_fun* fn, void* arg1, void* arg2) 24 | { 25 | if (heap==NULL || heap->page_count==0) return 0; 26 | 27 | // visit all pages 28 | #if MI_DEBUG>1 29 | size_t total = heap->page_count; 30 | #endif 31 | size_t count = 0; 32 | for (size_t i = 0; i <= MI_BIN_FULL; i++) { 33 | mi_page_queue_t* pq = &heap->pages[i]; 34 | mi_page_t* page = pq->first; 35 | while(page != NULL) { 36 | mi_page_t* next = page->next; // save next in case the page gets removed from the queue 37 | mi_assert_internal(page->heap == heap); 38 | count++; 39 | if (!fn(heap, pq, page, arg1, arg2)) return false; 40 | page = next; // and continue 41 | } 42 | } 43 | mi_assert_internal(count == total); 44 | return true; 45 | } 46 | 47 | 48 | #if MI_DEBUG>1 49 | static bool _mi_heap_page_is_valid(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { 50 | UNUSED(arg1); 51 | UNUSED(arg2); 52 | UNUSED(pq); 53 | mi_assert_internal(page->heap == heap); 54 | mi_segment_t* segment = _mi_page_segment(page); 55 | mi_assert_internal(segment->thread_id == heap->thread_id); 56 | mi_assert_expensive(_mi_page_is_valid(page)); 57 | return true; 58 | } 59 | 60 | static bool mi_heap_is_valid(mi_heap_t* heap) { 61 | mi_assert_internal(heap!=NULL); 62 | mi_heap_visit_pages(heap, &_mi_heap_page_is_valid, NULL, NULL); 63 | return true; 64 | } 65 | #endif 66 | 67 | 68 | 69 | 70 | /* ----------------------------------------------------------- 71 | "Collect" pages by migrating `local_free` and `thread_free` 72 | lists and freeing empty pages. This is done when a thread 73 | stops (and in that case abandons pages if there are still 74 | blocks alive) 75 | ----------------------------------------------------------- */ 76 | 77 | typedef enum mi_collect_e { 78 | NORMAL, 79 | FORCE, 80 | ABANDON 81 | } mi_collect_t; 82 | 83 | 84 | static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg_collect, void* arg2 ) { 85 | UNUSED(arg2); 86 | UNUSED(heap); 87 | mi_collect_t collect = *((mi_collect_t*)arg_collect); 88 | _mi_page_free_collect(page); 89 | if (mi_page_all_free(page)) { 90 | // no more used blocks, free the page. TODO: should we retire here and be less aggressive? 91 | _mi_page_free(page, pq, collect != NORMAL); 92 | } 93 | else if (collect == ABANDON) { 94 | // still used blocks but the thread is done; abandon the page 95 | _mi_page_abandon(page, pq); 96 | } 97 | return true; // don't break 98 | } 99 | 100 | static bool mi_heap_page_never_delayed_free(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { 101 | UNUSED(arg1); 102 | UNUSED(arg2); 103 | UNUSED(heap); 104 | UNUSED(pq); 105 | _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE); 106 | return true; // don't break 107 | } 108 | 109 | static void mi_heap_collect_ex(mi_heap_t* heap, mi_collect_t collect) 110 | { 111 | _mi_deferred_free(heap,collect > NORMAL); 112 | if (!mi_heap_is_initialized(heap)) return; 113 | 114 | 115 | // collect (some) abandoned pages 116 | if (collect >= NORMAL && !heap->no_reclaim) { 117 | if (collect == NORMAL) { 118 | // this may free some segments (but also take ownership of abandoned pages) 119 | _mi_segment_try_reclaim_abandoned(heap, false, &heap->tld->segments); 120 | } 121 | #if MI_DEBUG 122 | else if (collect == ABANDON && _mi_is_main_thread() && mi_heap_is_backing(heap)) { 123 | // the main thread is abandoned, try to free all abandoned segments. 124 | // if all memory is freed by now, all segments should be freed. 125 | _mi_segment_try_reclaim_abandoned(heap, true, &heap->tld->segments); 126 | } 127 | #endif 128 | } 129 | 130 | // if abandoning, mark all pages to no longer add to delayed_free 131 | if (collect == ABANDON) { 132 | //for (mi_page_t* page = heap->pages[MI_BIN_FULL].first; page != NULL; page = page->next) { 133 | // _mi_page_use_delayed_free(page, false); // set thread_free.delayed to MI_NO_DELAYED_FREE 134 | //} 135 | mi_heap_visit_pages(heap, &mi_heap_page_never_delayed_free, NULL, NULL); 136 | } 137 | 138 | // free thread delayed blocks. 139 | // (if abandoning, after this there are no more local references into the pages.) 140 | _mi_heap_delayed_free(heap); 141 | 142 | // collect all pages owned by this thread 143 | mi_heap_visit_pages(heap, &mi_heap_page_collect, &collect, NULL); 144 | mi_assert_internal( collect != ABANDON || heap->thread_delayed_free == NULL ); 145 | 146 | // collect segment caches 147 | if (collect >= FORCE) { 148 | _mi_segment_thread_collect(&heap->tld->segments); 149 | } 150 | 151 | // collect regions 152 | if (collect >= FORCE && _mi_is_main_thread()) { 153 | _mi_mem_collect(&heap->tld->stats); 154 | } 155 | } 156 | 157 | void _mi_heap_collect_abandon(mi_heap_t* heap) { 158 | mi_heap_collect_ex(heap, ABANDON); 159 | } 160 | 161 | void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept { 162 | mi_heap_collect_ex(heap, (force ? FORCE : NORMAL)); 163 | } 164 | 165 | void mi_collect(bool force) mi_attr_noexcept { 166 | mi_heap_collect(mi_get_default_heap(), force); 167 | } 168 | 169 | 170 | /* ----------------------------------------------------------- 171 | Heap new 172 | ----------------------------------------------------------- */ 173 | 174 | mi_heap_t* mi_heap_get_default(void) { 175 | mi_thread_init(); 176 | return mi_get_default_heap(); 177 | } 178 | 179 | mi_heap_t* mi_heap_get_backing(void) { 180 | mi_heap_t* heap = mi_heap_get_default(); 181 | mi_assert_internal(heap!=NULL); 182 | mi_heap_t* bheap = heap->tld->heap_backing; 183 | mi_assert_internal(bheap!=NULL); 184 | mi_assert_internal(bheap->thread_id == _mi_thread_id()); 185 | return bheap; 186 | } 187 | 188 | uintptr_t _mi_heap_random(mi_heap_t* heap) { 189 | uintptr_t r = heap->random; 190 | heap->random = _mi_random_shuffle(r); 191 | return r; 192 | } 193 | 194 | mi_heap_t* mi_heap_new(void) { 195 | mi_heap_t* bheap = mi_heap_get_backing(); 196 | mi_heap_t* heap = mi_heap_malloc_tp(bheap, mi_heap_t); 197 | if (heap==NULL) return NULL; 198 | memcpy(heap, &_mi_heap_empty, sizeof(mi_heap_t)); 199 | heap->tld = bheap->tld; 200 | heap->thread_id = _mi_thread_id(); 201 | heap->cookie = ((uintptr_t)heap ^ _mi_heap_random(bheap)) | 1; 202 | heap->random = _mi_heap_random(bheap); 203 | heap->no_reclaim = true; // don't reclaim abandoned pages or otherwise destroy is unsafe 204 | return heap; 205 | } 206 | 207 | // zero out the page queues 208 | static void mi_heap_reset_pages(mi_heap_t* heap) { 209 | mi_assert_internal(mi_heap_is_initialized(heap)); 210 | // TODO: copy full empty heap instead? 211 | memset(&heap->pages_free_direct, 0, sizeof(heap->pages_free_direct)); 212 | #ifdef MI_MEDIUM_DIRECT 213 | memset(&heap->pages_free_medium, 0, sizeof(heap->pages_free_medium)); 214 | #endif 215 | memcpy(&heap->pages, &_mi_heap_empty.pages, sizeof(heap->pages)); 216 | heap->thread_delayed_free = NULL; 217 | heap->page_count = 0; 218 | } 219 | 220 | // called from `mi_heap_destroy` and `mi_heap_delete` to free the internal heap resources. 221 | static void mi_heap_free(mi_heap_t* heap) { 222 | mi_assert_internal(mi_heap_is_initialized(heap)); 223 | if (mi_heap_is_backing(heap)) return; // dont free the backing heap 224 | 225 | // reset default 226 | if (mi_heap_is_default(heap)) { 227 | _mi_heap_default = heap->tld->heap_backing; 228 | } 229 | // and free the used memory 230 | mi_free(heap); 231 | } 232 | 233 | 234 | /* ----------------------------------------------------------- 235 | Heap destroy 236 | ----------------------------------------------------------- */ 237 | 238 | static bool _mi_heap_page_destroy(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { 239 | UNUSED(arg1); 240 | UNUSED(arg2); 241 | UNUSED(heap); 242 | UNUSED(pq); 243 | 244 | // ensure no more thread_delayed_free will be added 245 | _mi_page_use_delayed_free(page, MI_NEVER_DELAYED_FREE); 246 | 247 | // stats 248 | if (page->block_size > MI_LARGE_SIZE_MAX) { 249 | mi_heap_stat_decrease(heap,huge,page->block_size); 250 | } 251 | #if (MI_STAT>1) 252 | size_t inuse = page->used - page->thread_freed; 253 | if (page->block_size <= MI_LARGE_SIZE_MAX) { 254 | mi_heap_stat_decrease(heap,normal[_mi_bin(page->block_size)], inuse); 255 | } 256 | mi_heap_stat_decrease(heap,malloc, page->block_size * inuse); // todo: off for aligned blocks... 257 | #endif 258 | 259 | // pretend it is all free now 260 | mi_assert_internal(page->thread_freed<=0xFFFF); 261 | page->used = (uint16_t)page->thread_freed; 262 | 263 | // and free the page 264 | _mi_segment_page_free(page,false /* no force? */, &heap->tld->segments); 265 | 266 | return true; // keep going 267 | } 268 | 269 | void _mi_heap_destroy_pages(mi_heap_t* heap) { 270 | mi_heap_visit_pages(heap, &_mi_heap_page_destroy, NULL, NULL); 271 | mi_heap_reset_pages(heap); 272 | } 273 | 274 | void mi_heap_destroy(mi_heap_t* heap) { 275 | mi_assert(mi_heap_is_initialized(heap)); 276 | mi_assert(heap->no_reclaim); 277 | mi_assert_expensive(mi_heap_is_valid(heap)); 278 | if (!mi_heap_is_initialized(heap)) return; 279 | if (!heap->no_reclaim) { 280 | // don't free in case it may contain reclaimed pages 281 | mi_heap_delete(heap); 282 | } 283 | else { 284 | // free all pages 285 | _mi_heap_destroy_pages(heap); 286 | mi_heap_free(heap); 287 | } 288 | } 289 | 290 | 291 | 292 | /* ----------------------------------------------------------- 293 | Safe Heap delete 294 | ----------------------------------------------------------- */ 295 | 296 | // Tranfer the pages from one heap to the other 297 | static void mi_heap_absorb(mi_heap_t* heap, mi_heap_t* from) { 298 | mi_assert_internal(heap!=NULL); 299 | if (from==NULL || from->page_count == 0) return; 300 | 301 | // unfull all full pages 302 | mi_page_t* page = heap->pages[MI_BIN_FULL].first; 303 | while (page != NULL) { 304 | mi_page_t* next = page->next; 305 | _mi_page_unfull(page); 306 | page = next; 307 | } 308 | mi_assert_internal(heap->pages[MI_BIN_FULL].first == NULL); 309 | 310 | // free outstanding thread delayed free blocks 311 | _mi_heap_delayed_free(from); 312 | 313 | // transfer all pages by appending the queues; this will set 314 | // a new heap field which is ok as all pages are unfull'd and thus 315 | // other threads won't access this field anymore (see `mi_free_block_mt`) 316 | for (size_t i = 0; i < MI_BIN_FULL; i++) { 317 | mi_page_queue_t* pq = &heap->pages[i]; 318 | mi_page_queue_t* append = &from->pages[i]; 319 | size_t pcount = _mi_page_queue_append(heap, pq, append); 320 | heap->page_count += pcount; 321 | from->page_count -= pcount; 322 | } 323 | mi_assert_internal(from->thread_delayed_free == NULL); 324 | mi_assert_internal(from->page_count == 0); 325 | 326 | // and reset the `from` heap 327 | mi_heap_reset_pages(from); 328 | } 329 | 330 | // Safe delete a heap without freeing any still allocated blocks in that heap. 331 | void mi_heap_delete(mi_heap_t* heap) 332 | { 333 | mi_assert(mi_heap_is_initialized(heap)); 334 | mi_assert_expensive(mi_heap_is_valid(heap)); 335 | if (!mi_heap_is_initialized(heap)) return; 336 | 337 | if (!mi_heap_is_backing(heap)) { 338 | // tranfer still used pages to the backing heap 339 | mi_heap_absorb(heap->tld->heap_backing, heap); 340 | } 341 | else { 342 | // the backing heap abandons its pages 343 | _mi_heap_collect_abandon(heap); 344 | } 345 | mi_assert_internal(heap->page_count==0); 346 | mi_heap_free(heap); 347 | } 348 | 349 | mi_heap_t* mi_heap_set_default(mi_heap_t* heap) { 350 | mi_assert(mi_heap_is_initialized(heap)); 351 | if (!mi_heap_is_initialized(heap)) return NULL; 352 | mi_assert_expensive(mi_heap_is_valid(heap)); 353 | mi_heap_t* old = _mi_heap_default; 354 | _mi_heap_default = heap; 355 | return old; 356 | } 357 | 358 | 359 | 360 | 361 | /* ----------------------------------------------------------- 362 | Analysis 363 | ----------------------------------------------------------- */ 364 | 365 | // static since it is not thread safe to access heaps from other threads. 366 | static mi_heap_t* mi_heap_of_block(const void* p) { 367 | if (p == NULL) return NULL; 368 | mi_segment_t* segment = _mi_ptr_segment(p); 369 | bool valid = (_mi_ptr_cookie(segment) == segment->cookie); 370 | mi_assert_internal(valid); 371 | if (mi_unlikely(!valid)) return NULL; 372 | return _mi_segment_page_of(segment,p)->heap; 373 | } 374 | 375 | bool mi_heap_contains_block(mi_heap_t* heap, const void* p) { 376 | mi_assert(heap != NULL); 377 | if (!mi_heap_is_initialized(heap)) return false; 378 | return (heap == mi_heap_of_block(p)); 379 | } 380 | 381 | 382 | static bool mi_heap_page_check_owned(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* p, void* vfound) { 383 | UNUSED(heap); 384 | UNUSED(pq); 385 | bool* found = (bool*)vfound; 386 | mi_segment_t* segment = _mi_page_segment(page); 387 | void* start = _mi_page_start(segment, page, NULL); 388 | void* end = (uint8_t*)start + (page->capacity * page->block_size); 389 | *found = (p >= start && p < end); 390 | return (!*found); // continue if not found 391 | } 392 | 393 | bool mi_heap_check_owned(mi_heap_t* heap, const void* p) { 394 | mi_assert(heap != NULL); 395 | if (!mi_heap_is_initialized(heap)) return false; 396 | if (((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0) return false; // only aligned pointers 397 | bool found = false; 398 | mi_heap_visit_pages(heap, &mi_heap_page_check_owned, (void*)p, &found); 399 | return found; 400 | } 401 | 402 | bool mi_check_owned(const void* p) { 403 | return mi_heap_check_owned(mi_get_default_heap(), p); 404 | } 405 | 406 | /* ----------------------------------------------------------- 407 | Visit all heap blocks and areas 408 | Todo: enable visiting abandoned pages, and 409 | enable visiting all blocks of all heaps across threads 410 | ----------------------------------------------------------- */ 411 | 412 | // Separate struct to keep `mi_page_t` out of the public interface 413 | typedef struct mi_heap_area_ex_s { 414 | mi_heap_area_t area; 415 | mi_page_t* page; 416 | } mi_heap_area_ex_t; 417 | 418 | static bool mi_heap_area_visit_blocks(const mi_heap_area_ex_t* xarea, mi_block_visit_fun* visitor, void* arg) { 419 | mi_assert(xarea != NULL); 420 | if (xarea==NULL) return true; 421 | const mi_heap_area_t* area = &xarea->area; 422 | mi_page_t* page = xarea->page; 423 | mi_assert(page != NULL); 424 | if (page == NULL) return true; 425 | 426 | _mi_page_free_collect(page); 427 | mi_assert_internal(page->local_free == NULL); 428 | if (page->used == 0) return true; 429 | 430 | size_t psize; 431 | uint8_t* pstart = _mi_page_start(_mi_page_segment(page), page, &psize); 432 | 433 | if (page->capacity == 1) { 434 | // optimize page with one block 435 | mi_assert_internal(page->used == 1 && page->free == NULL); 436 | return visitor(page->heap, area, pstart, page->block_size, arg); 437 | } 438 | 439 | // create a bitmap of free blocks. 440 | #define MI_MAX_BLOCKS (MI_SMALL_PAGE_SIZE / sizeof(void*)) 441 | uintptr_t free_map[MI_MAX_BLOCKS / sizeof(uintptr_t)]; 442 | memset(free_map, 0, sizeof(free_map)); 443 | 444 | size_t free_count = 0; 445 | for (mi_block_t* block = page->free; block != NULL; block = mi_block_next(page,block)) { 446 | free_count++; 447 | mi_assert_internal((uint8_t*)block >= pstart && (uint8_t*)block < (pstart + psize)); 448 | size_t offset = (uint8_t*)block - pstart; 449 | mi_assert_internal(offset % page->block_size == 0); 450 | size_t blockidx = offset / page->block_size; // Todo: avoid division? 451 | mi_assert_internal( blockidx < MI_MAX_BLOCKS); 452 | size_t bitidx = (blockidx / sizeof(uintptr_t)); 453 | size_t bit = blockidx - (bitidx * sizeof(uintptr_t)); 454 | free_map[bitidx] |= ((uintptr_t)1 << bit); 455 | } 456 | mi_assert_internal(page->capacity == (free_count + page->used)); 457 | 458 | // walk through all blocks skipping the free ones 459 | size_t used_count = 0; 460 | for (size_t i = 0; i < page->capacity; i++) { 461 | size_t bitidx = (i / sizeof(uintptr_t)); 462 | size_t bit = i - (bitidx * sizeof(uintptr_t)); 463 | uintptr_t m = free_map[bitidx]; 464 | if (bit == 0 && m == UINTPTR_MAX) { 465 | i += (sizeof(uintptr_t) - 1); // skip a run of free blocks 466 | } 467 | else if ((m & ((uintptr_t)1 << bit)) == 0) { 468 | used_count++; 469 | uint8_t* block = pstart + (i * page->block_size); 470 | if (!visitor(page->heap, area, block, page->block_size, arg)) return false; 471 | } 472 | } 473 | mi_assert_internal(page->used == used_count); 474 | return true; 475 | } 476 | 477 | typedef bool (mi_heap_area_visit_fun)(const mi_heap_t* heap, const mi_heap_area_ex_t* area, void* arg); 478 | 479 | 480 | static bool mi_heap_visit_areas_page(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t* page, void* vfun, void* arg) { 481 | UNUSED(heap); 482 | UNUSED(pq); 483 | mi_heap_area_visit_fun* fun = (mi_heap_area_visit_fun*)vfun; 484 | mi_heap_area_ex_t xarea; 485 | xarea.page = page; 486 | xarea.area.reserved = page->reserved * page->block_size; 487 | xarea.area.committed = page->capacity * page->block_size; 488 | xarea.area.blocks = _mi_page_start(_mi_page_segment(page), page, NULL); 489 | xarea.area.used = page->used - page->thread_freed; // race is ok 490 | xarea.area.block_size = page->block_size; 491 | return fun(heap, &xarea, arg); 492 | } 493 | 494 | // Visit all heap pages as areas 495 | static bool mi_heap_visit_areas(const mi_heap_t* heap, mi_heap_area_visit_fun* visitor, void* arg) { 496 | if (visitor == NULL) return false; 497 | return mi_heap_visit_pages((mi_heap_t*)heap, &mi_heap_visit_areas_page, (void*)(visitor), arg); // note: function pointer to void* :-{ 498 | } 499 | 500 | // Just to pass arguments 501 | typedef struct mi_visit_blocks_args_s { 502 | bool visit_blocks; 503 | mi_block_visit_fun* visitor; 504 | void* arg; 505 | } mi_visit_blocks_args_t; 506 | 507 | static bool mi_heap_area_visitor(const mi_heap_t* heap, const mi_heap_area_ex_t* xarea, void* arg) { 508 | mi_visit_blocks_args_t* args = (mi_visit_blocks_args_t*)arg; 509 | if (!args->visitor(heap, &xarea->area, NULL, xarea->area.block_size, args->arg)) return false; 510 | if (args->visit_blocks) { 511 | return mi_heap_area_visit_blocks(xarea, args->visitor, args->arg); 512 | } 513 | else { 514 | return true; 515 | } 516 | } 517 | 518 | // Visit all blocks in a heap 519 | bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg) { 520 | mi_visit_blocks_args_t args = { visit_blocks, visitor, arg }; 521 | return mi_heap_visit_areas(heap, &mi_heap_area_visitor, &args); 522 | } 523 | 524 | -------------------------------------------------------------------------------- /mimalloc/init.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #include "mimalloc.h" 8 | #include "mimalloc-internal.h" 9 | 10 | #include // memcpy 11 | 12 | // Empty page used to initialize the small free pages array 13 | const mi_page_t _mi_page_empty = { 14 | 0, false, false, false, {0}, 15 | 0, 0, 16 | NULL, 0, 0, // free, used, cookie 17 | NULL, 0, 0, 18 | 0, NULL, NULL, NULL 19 | #if (MI_INTPTR_SIZE==4) 20 | , { NULL } 21 | #endif 22 | }; 23 | 24 | #define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) 25 | #define MI_SMALL_PAGES_EMPTY \ 26 | { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } 27 | 28 | 29 | // Empty page queues for every bin 30 | #define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) } 31 | #define MI_PAGE_QUEUES_EMPTY \ 32 | { QNULL(1), \ 33 | QNULL(1), QNULL(2), QNULL(3), QNULL(4), QNULL(5), QNULL(6), QNULL(7), QNULL(8), \ 34 | QNULL(10), QNULL(12), QNULL(14), QNULL(16), QNULL(20), QNULL(24), QNULL(28), QNULL(32), \ 35 | QNULL(40), QNULL(48), QNULL(56), QNULL(64), QNULL(80), QNULL(96), QNULL(112), QNULL(128), \ 36 | QNULL(160), QNULL(192), QNULL(224), QNULL(256), QNULL(320), QNULL(384), QNULL(448), QNULL(512), \ 37 | QNULL(640), QNULL(768), QNULL(896), QNULL(1024), QNULL(1280), QNULL(1536), QNULL(1792), QNULL(2048), \ 38 | QNULL(2560), QNULL(3072), QNULL(3584), QNULL(4096), QNULL(5120), QNULL(6144), QNULL(7168), QNULL(8192), \ 39 | QNULL(10240), QNULL(12288), QNULL(14336), QNULL(16384), QNULL(20480), QNULL(24576), QNULL(28672), QNULL(32768), \ 40 | QNULL(40960), QNULL(49152), QNULL(57344), QNULL(65536), QNULL(81920), QNULL(98304), QNULL(114688), \ 41 | QNULL(MI_LARGE_WSIZE_MAX + 1 /*131072, Huge queue */), \ 42 | QNULL(MI_LARGE_WSIZE_MAX + 2) /* Full queue */ } 43 | 44 | #define MI_STAT_COUNT_NULL() {0,0,0,0} 45 | 46 | // Empty statistics 47 | #if MI_STAT>1 48 | #define MI_STAT_COUNT_END_NULL() , { MI_STAT_COUNT_NULL(), MI_INIT64(MI_STAT_COUNT_NULL) } 49 | #else 50 | #define MI_STAT_COUNT_END_NULL() 51 | #endif 52 | 53 | #define MI_STATS_NULL \ 54 | MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 55 | MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 56 | MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 57 | MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 58 | MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 59 | MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 60 | MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 61 | MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 62 | { 0, 0 } \ 63 | MI_STAT_COUNT_END_NULL() 64 | 65 | // -------------------------------------------------------- 66 | // Statically allocate an empty heap as the initial 67 | // thread local value for the default heap, 68 | // and statically allocate the backing heap for the main 69 | // thread so it can function without doing any allocation 70 | // itself (as accessing a thread local for the first time 71 | // may lead to allocation itself on some platforms) 72 | // -------------------------------------------------------- 73 | 74 | const mi_heap_t _mi_heap_empty = { 75 | NULL, 76 | MI_SMALL_PAGES_EMPTY, 77 | MI_PAGE_QUEUES_EMPTY, 78 | NULL, 79 | 0, 80 | 0, 81 | 0, 82 | 0, 83 | false 84 | }; 85 | 86 | mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty; 87 | 88 | 89 | #define tld_main_stats ((mi_stats_t*)((uint8_t*)&tld_main + offsetof(mi_tld_t,stats))) 90 | 91 | static mi_tld_t tld_main = { 92 | 0, 93 | &_mi_heap_main, 94 | { { NULL, NULL }, {NULL ,NULL}, 0, 0, 0, 0, 0, 0, NULL, tld_main_stats }, // segments 95 | { 0, NULL, NULL, 0, tld_main_stats }, // os 96 | { MI_STATS_NULL } // stats 97 | }; 98 | 99 | mi_heap_t _mi_heap_main = { 100 | &tld_main, 101 | MI_SMALL_PAGES_EMPTY, 102 | MI_PAGE_QUEUES_EMPTY, 103 | NULL, 104 | 0, 105 | 0, 106 | #if MI_INTPTR_SIZE==8 // the cookie of the main heap can be fixed (unlike page cookies that need to be secure!) 107 | 0xCDCDCDCDCDCDCDCDUL, 108 | #else 109 | 0xCDCDCDCDUL, 110 | #endif 111 | 0, 112 | false // can reclaim 113 | }; 114 | 115 | bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. 116 | 117 | mi_stats_t _mi_stats_main = { MI_STATS_NULL }; 118 | 119 | /* ----------------------------------------------------------- 120 | Initialization of random numbers 121 | ----------------------------------------------------------- */ 122 | 123 | #if defined(_WIN32) 124 | #include 125 | #elif defined(__APPLE__) 126 | #include 127 | #else 128 | #include 129 | #endif 130 | 131 | uintptr_t _mi_random_shuffle(uintptr_t x) { 132 | #if (MI_INTPTR_SIZE==8) 133 | // by Sebastiano Vigna, see: 134 | x ^= x >> 30; 135 | x *= 0xbf58476d1ce4e5b9UL; 136 | x ^= x >> 27; 137 | x *= 0x94d049bb133111ebUL; 138 | x ^= x >> 31; 139 | #elif (MI_INTPTR_SIZE==4) 140 | // by Chris Wellons, see: 141 | x ^= x >> 16; 142 | x *= 0x7feb352dUL; 143 | x ^= x >> 15; 144 | x *= 0x846ca68bUL; 145 | x ^= x >> 16; 146 | #endif 147 | return x; 148 | } 149 | 150 | uintptr_t _mi_random_init(uintptr_t seed /* can be zero */) { 151 | #ifdef __wasi__ // no ASLR when using WebAssembly, and time granularity may be coarse 152 | uintptr_t x; 153 | arc4random_buf(&x, sizeof x); 154 | #else 155 | // Hopefully, ASLR makes our function address random 156 | uintptr_t x = (uintptr_t)((void*)&_mi_random_init); 157 | x ^= seed; 158 | // xor with high res time 159 | #if defined(_WIN32) 160 | LARGE_INTEGER pcount; 161 | QueryPerformanceCounter(&pcount); 162 | x ^= (uintptr_t)(pcount.QuadPart); 163 | #elif defined(__APPLE__) 164 | x ^= (uintptr_t)mach_absolute_time(); 165 | #else 166 | struct timespec time; 167 | clock_gettime(CLOCK_MONOTONIC, &time); 168 | x ^= (uintptr_t)time.tv_sec; 169 | x ^= (uintptr_t)time.tv_nsec; 170 | #endif 171 | // and do a few randomization steps 172 | uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1; 173 | for (uintptr_t i = 0; i < max; i++) { 174 | x = _mi_random_shuffle(x); 175 | } 176 | #endif 177 | return x; 178 | } 179 | 180 | uintptr_t _mi_ptr_cookie(const void* p) { 181 | return ((uintptr_t)p ^ _mi_heap_main.cookie); 182 | } 183 | 184 | /* ----------------------------------------------------------- 185 | Initialization and freeing of the thread local heaps 186 | ----------------------------------------------------------- */ 187 | 188 | typedef struct mi_thread_data_s { 189 | mi_heap_t heap; // must come first due to cast in `_mi_heap_done` 190 | mi_tld_t tld; 191 | } mi_thread_data_t; 192 | 193 | // Initialize the thread local default heap, called from `mi_thread_init` 194 | static bool _mi_heap_init(void) { 195 | if (mi_heap_is_initialized(_mi_heap_default)) return true; 196 | if (_mi_is_main_thread()) { 197 | // the main heap is statically allocated 198 | _mi_heap_default = &_mi_heap_main; 199 | mi_assert_internal(_mi_heap_default->tld->heap_backing == _mi_heap_default); 200 | } 201 | else { 202 | // use `_mi_os_alloc` to allocate directly from the OS 203 | mi_thread_data_t* td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t),&_mi_stats_main); // Todo: more efficient allocation? 204 | if (td == NULL) { 205 | _mi_error_message("failed to allocate thread local heap memory\n"); 206 | return false; 207 | } 208 | mi_tld_t* tld = &td->tld; 209 | mi_heap_t* heap = &td->heap; 210 | memcpy(heap, &_mi_heap_empty, sizeof(*heap)); 211 | heap->thread_id = _mi_thread_id(); 212 | heap->random = _mi_random_init(heap->thread_id); 213 | heap->cookie = ((uintptr_t)heap ^ _mi_heap_random(heap)) | 1; 214 | heap->tld = tld; 215 | memset(tld, 0, sizeof(*tld)); 216 | tld->heap_backing = heap; 217 | tld->segments.stats = &tld->stats; 218 | tld->os.stats = &tld->stats; 219 | _mi_heap_default = heap; 220 | } 221 | return false; 222 | } 223 | 224 | // Free the thread local default heap (called from `mi_thread_done`) 225 | static bool _mi_heap_done(void) { 226 | mi_heap_t* heap = _mi_heap_default; 227 | if (!mi_heap_is_initialized(heap)) return true; 228 | 229 | // reset default heap 230 | _mi_heap_default = (_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty); 231 | 232 | // todo: delete all non-backing heaps? 233 | 234 | // switch to backing heap and free it 235 | heap = heap->tld->heap_backing; 236 | if (!mi_heap_is_initialized(heap)) return false; 237 | 238 | // collect if not the main thread 239 | if (heap != &_mi_heap_main) { 240 | _mi_heap_collect_abandon(heap); 241 | } 242 | 243 | // merge stats 244 | _mi_stats_done(&heap->tld->stats); 245 | 246 | // free if not the main thread 247 | if (heap != &_mi_heap_main) { 248 | _mi_os_free(heap, sizeof(mi_thread_data_t), &_mi_stats_main); 249 | } 250 | #if (MI_DEBUG > 0) 251 | else { 252 | _mi_heap_destroy_pages(heap); 253 | mi_assert_internal(heap->tld->heap_backing == &_mi_heap_main); 254 | } 255 | #endif 256 | return false; 257 | } 258 | 259 | 260 | 261 | // -------------------------------------------------------- 262 | // Try to run `mi_thread_done()` automatically so any memory 263 | // owned by the thread but not yet released can be abandoned 264 | // and re-owned by another thread. 265 | // 266 | // 1. windows dynamic library: 267 | // call from DllMain on DLL_THREAD_DETACH 268 | // 2. windows static library: 269 | // use `FlsAlloc` to call a destructor when the thread is done 270 | // 3. unix, pthreads: 271 | // use a pthread key to call a destructor when a pthread is done 272 | // 273 | // In the last two cases we also need to call `mi_process_init` 274 | // to set up the thread local keys. 275 | // -------------------------------------------------------- 276 | 277 | #ifdef __wasi__ 278 | // no pthreads in the WebAssembly Standard Interface 279 | #elif !defined(_WIN32) 280 | #define MI_USE_PTHREADS 281 | #endif 282 | 283 | #if defined(_WIN32) && defined(MI_SHARED_LIB) 284 | // nothing to do as it is done in DllMain 285 | #elif defined(_WIN32) && !defined(MI_SHARED_LIB) 286 | // use thread local storage keys to detect thread ending 287 | #include 288 | #include 289 | static DWORD mi_fls_key; 290 | static void NTAPI mi_fls_done(PVOID value) { 291 | if (value!=NULL) mi_thread_done(); 292 | } 293 | #elif defined(MI_USE_PTHREADS) 294 | // use pthread locol storage keys to detect thread ending 295 | #include 296 | static pthread_key_t mi_pthread_key; 297 | static void mi_pthread_done(void* value) { 298 | if (value!=NULL) mi_thread_done(); 299 | } 300 | #elif defined(__wasi__) 301 | // no pthreads in the WebAssembly Standard Interface 302 | #else 303 | #pragma message("define a way to call mi_thread_done when a thread is done") 304 | #endif 305 | 306 | // Set up handlers so `mi_thread_done` is called automatically 307 | static void mi_process_setup_auto_thread_done(void) { 308 | static bool tls_initialized = false; // fine if it races 309 | if (tls_initialized) return; 310 | tls_initialized = true; 311 | #if defined(_WIN32) && defined(MI_SHARED_LIB) 312 | // nothing to do as it is done in DllMain 313 | #elif defined(_WIN32) && !defined(MI_SHARED_LIB) 314 | mi_fls_key = FlsAlloc(&mi_fls_done); 315 | #elif defined(MI_USE_PTHREADS) 316 | pthread_key_create(&mi_pthread_key, &mi_pthread_done); 317 | #endif 318 | } 319 | 320 | 321 | bool _mi_is_main_thread(void) { 322 | return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id()); 323 | } 324 | 325 | // This is called from the `mi_malloc_generic` 326 | void mi_thread_init(void) mi_attr_noexcept 327 | { 328 | // ensure our process has started already 329 | mi_process_init(); 330 | 331 | // initialize the thread local default heap 332 | if (_mi_heap_init()) return; // returns true if already initialized 333 | 334 | // don't further initialize for the main thread 335 | if (_mi_is_main_thread()) return; 336 | 337 | _mi_stat_increase(&mi_get_default_heap()->tld->stats.threads, 1); 338 | 339 | // set hooks so our mi_thread_done() will be called 340 | #if defined(_WIN32) && defined(MI_SHARED_LIB) 341 | // nothing to do as it is done in DllMain 342 | #elif defined(_WIN32) && !defined(MI_SHARED_LIB) 343 | FlsSetValue(mi_fls_key, (void*)(_mi_thread_id()|1)); // set to a dummy value so that `mi_fls_done` is called 344 | #elif defined(MI_USE_PTHREADS) 345 | pthread_setspecific(mi_pthread_key, (void*)(_mi_thread_id()|1)); // set to a dummy value so that `mi_pthread_done` is called 346 | #endif 347 | 348 | #if (MI_DEBUG>0) // not in release mode as that leads to crashes on Windows dynamic override 349 | _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); 350 | #endif 351 | } 352 | 353 | void mi_thread_done(void) mi_attr_noexcept { 354 | // stats 355 | mi_heap_t* heap = mi_get_default_heap(); 356 | if (!_mi_is_main_thread() && mi_heap_is_initialized(heap)) { 357 | _mi_stat_decrease(&heap->tld->stats.threads, 1); 358 | } 359 | 360 | // abandon the thread local heap 361 | if (_mi_heap_done()) return; // returns true if already ran 362 | 363 | #if (MI_DEBUG>0) 364 | if (!_mi_is_main_thread()) { 365 | _mi_verbose_message("thread done: 0x%zx\n", _mi_thread_id()); 366 | } 367 | #endif 368 | } 369 | 370 | 371 | // -------------------------------------------------------- 372 | // Run functions on process init/done, and thread init/done 373 | // -------------------------------------------------------- 374 | static void mi_process_done(void); 375 | 376 | void mi_process_init(void) mi_attr_noexcept { 377 | // ensure we are called once 378 | if (_mi_process_is_initialized) return; 379 | // access _mi_heap_default before setting _mi_process_is_initialized to ensure 380 | // that the TLS slot is allocated without getting into recursion on macOS 381 | // when using dynamic linking with interpose. 382 | mi_heap_t* h = _mi_heap_default; 383 | _mi_process_is_initialized = true; 384 | 385 | _mi_heap_main.thread_id = _mi_thread_id(); 386 | _mi_verbose_message("process init: 0x%zx\n", _mi_heap_main.thread_id); 387 | uintptr_t random = _mi_random_init(_mi_heap_main.thread_id) ^ (uintptr_t)h; 388 | #ifndef __APPLE__ 389 | _mi_heap_main.cookie = (uintptr_t)&_mi_heap_main ^ random; 390 | #endif 391 | _mi_heap_main.random = _mi_random_shuffle(random); 392 | #if (MI_DEBUG) 393 | _mi_verbose_message("debug level : %d\n", MI_DEBUG); 394 | #endif 395 | atexit(&mi_process_done); 396 | mi_process_setup_auto_thread_done(); 397 | mi_stats_reset(); 398 | _mi_os_init(); 399 | } 400 | 401 | static void mi_process_done(void) { 402 | // only shutdown if we were initialized 403 | if (!_mi_process_is_initialized) return; 404 | // ensure we are called once 405 | static bool process_done = false; 406 | if (process_done) return; 407 | process_done = true; 408 | 409 | #ifndef NDEBUG 410 | mi_collect(true); 411 | #endif 412 | if (mi_option_is_enabled(mi_option_show_stats) || 413 | mi_option_is_enabled(mi_option_verbose)) { 414 | mi_stats_print(NULL); 415 | } 416 | _mi_verbose_message("process done: 0x%zx\n", _mi_heap_main.thread_id); 417 | } 418 | 419 | 420 | 421 | #if defined(_WIN32) && defined(MI_SHARED_LIB) 422 | // Windows DLL: easy to hook into process_init and thread_done 423 | #include 424 | 425 | __declspec(dllexport) BOOL WINAPI DllMain(HINSTANCE inst, DWORD reason, LPVOID reserved) { 426 | UNUSED(reserved); 427 | UNUSED(inst); 428 | if (reason==DLL_PROCESS_ATTACH) { 429 | mi_process_init(); 430 | } 431 | else if (reason==DLL_THREAD_DETACH) { 432 | mi_thread_done(); 433 | } 434 | return TRUE; 435 | } 436 | 437 | #elif defined(__cplusplus) 438 | // C++: use static initialization to detect process start 439 | static bool _mi_process_init(void) { 440 | mi_process_init(); 441 | return (_mi_heap_main.thread_id != 0); 442 | } 443 | static bool mi_initialized = _mi_process_init(); 444 | 445 | #elif defined(__GNUC__) || defined(__clang__) 446 | // GCC,Clang: use the constructor attribute 447 | static void __attribute__((constructor)) _mi_process_init(void) { 448 | mi_process_init(); 449 | } 450 | 451 | #elif defined(_MSC_VER) 452 | // MSVC: use data section magic for static libraries 453 | // See 454 | static int _mi_process_init(void) { 455 | mi_process_init(); 456 | return 0; 457 | } 458 | typedef int(*_crt_cb)(void); 459 | #ifdef _M_X64 460 | __pragma(comment(linker, "/include:" "_mi_msvc_initu")) 461 | #pragma section(".CRT$XIU", long, read) 462 | #else 463 | __pragma(comment(linker, "/include:" "__mi_msvc_initu")) 464 | #endif 465 | #pragma data_seg(".CRT$XIU") 466 | _crt_cb _mi_msvc_initu[] = { &_mi_process_init }; 467 | #pragma data_seg() 468 | 469 | #else 470 | #pragma message("define a way to call mi_process_init/done on your platform") 471 | #endif 472 | -------------------------------------------------------------------------------- /mimalloc/memory.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2019, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | /* ---------------------------------------------------------------------------- 9 | This implements a layer between the raw OS memory (VirtualAlloc/mmap/sbrk/..) 10 | and the segment and huge object allocation by mimalloc. There may be multiple 11 | implementations of this (one could be the identity going directly to the OS, 12 | another could be a simple cache etc), but the current one uses large "regions". 13 | In contrast to the rest of mimalloc, the "regions" are shared between threads and 14 | need to be accessed using atomic operations. 15 | We need this memory layer between the raw OS calls because of: 16 | 1. on `sbrk` like systems (like WebAssembly) we need our own memory maps in order 17 | to reuse memory effectively. 18 | 2. It turns out that for large objects, between 1MiB and 32MiB (?), the cost of 19 | an OS allocation/free is still (much) too expensive relative to the accesses in that 20 | object :-( (`mallloc-large` tests this). This means we need a cheaper way to 21 | reuse memory. 22 | 3. This layer can help with a NUMA aware allocation in the future. 23 | 24 | Possible issues: 25 | - (2) can potentially be addressed too with a small cache per thread which is much 26 | simpler. Generally though that requires shrinking of huge pages, and may overuse 27 | memory per thread. (and is not compatible with `sbrk`). 28 | - Since the current regions are per-process, we need atomic operations to 29 | claim blocks which may be contended 30 | - In the worst case, we need to search the whole region map (16KiB for 256GiB) 31 | linearly. At what point will direct OS calls be faster? Is there a way to 32 | do this better without adding too much complexity? 33 | -----------------------------------------------------------------------------*/ 34 | #include "mimalloc.h" 35 | #include "mimalloc-internal.h" 36 | #include "mimalloc-atomic.h" 37 | 38 | #include // memset 39 | 40 | // Internal raw OS interface 41 | size_t _mi_os_large_page_size(); 42 | bool _mi_os_protect(void* addr, size_t size); 43 | bool _mi_os_unprotect(void* addr, size_t size); 44 | bool _mi_os_commit(void* p, size_t size, mi_stats_t* stats); 45 | bool _mi_os_decommit(void* p, size_t size, mi_stats_t* stats); 46 | bool _mi_os_reset(void* p, size_t size, mi_stats_t* stats); 47 | bool _mi_os_unreset(void* p, size_t size, mi_stats_t* stats); 48 | void* _mi_os_alloc_aligned(size_t size, size_t alignment, bool commit, mi_os_tld_t* tld); 49 | 50 | 51 | // Constants 52 | #if (MI_INTPTR_SIZE==8) 53 | #define MI_HEAP_REGION_MAX_SIZE (256 * (1ULL << 30)) // 256GiB => 16KiB for the region map 54 | #elif (MI_INTPTR_SIZE==4) 55 | #define MI_HEAP_REGION_MAX_SIZE (3 * (1UL << 30)) // 3GiB => 196 bytes for the region map 56 | #else 57 | #error "define the maximum heap space allowed for regions on this platform" 58 | #endif 59 | 60 | #define MI_SEGMENT_ALIGN MI_SEGMENT_SIZE 61 | 62 | #define MI_REGION_MAP_BITS (MI_INTPTR_SIZE * 8) 63 | #define MI_REGION_SIZE (MI_SEGMENT_SIZE * MI_REGION_MAP_BITS) 64 | #define MI_REGION_MAX_ALLOC_SIZE ((MI_REGION_MAP_BITS/4)*MI_SEGMENT_SIZE) // 64MiB 65 | #define MI_REGION_MAX (MI_HEAP_REGION_MAX_SIZE / MI_REGION_SIZE) 66 | #define MI_REGION_MAP_FULL UINTPTR_MAX 67 | 68 | 69 | // A region owns a chunk of REGION_SIZE (256MiB) (virtual) memory with 70 | // a bit map with one bit per MI_SEGMENT_SIZE (4MiB) block. 71 | typedef struct mem_region_s { 72 | volatile uintptr_t map; // in-use bit per MI_SEGMENT_SIZE block 73 | volatile void* start; // start of virtual memory area 74 | } mem_region_t; 75 | 76 | 77 | // The region map; 16KiB for a 256GiB HEAP_REGION_MAX 78 | // TODO: in the future, maintain a map per NUMA node for numa aware allocation 79 | static mem_region_t regions[MI_REGION_MAX]; 80 | 81 | static volatile size_t regions_count = 0; // allocated regions 82 | static volatile uintptr_t region_next_idx = 0; // good place to start searching 83 | 84 | 85 | /* ---------------------------------------------------------------------------- 86 | Utility functions 87 | -----------------------------------------------------------------------------*/ 88 | 89 | // Blocks (of 4MiB) needed for the given size. 90 | static size_t mi_region_block_count(size_t size) { 91 | mi_assert_internal(size <= MI_REGION_MAX_ALLOC_SIZE); 92 | return (size + MI_SEGMENT_SIZE - 1) / MI_SEGMENT_SIZE; 93 | } 94 | 95 | // The bit mask for a given number of blocks at a specified bit index. 96 | static uintptr_t mi_region_block_mask(size_t blocks, size_t bitidx) { 97 | mi_assert_internal(blocks + bitidx <= MI_REGION_MAP_BITS); 98 | return ((((uintptr_t)1 << blocks) - 1) << bitidx); 99 | } 100 | 101 | // Return a rounded commit/reset size such that we don't fragment large OS pages into small ones. 102 | static size_t mi_good_commit_size(size_t size) { 103 | if (size > (SIZE_MAX - _mi_os_large_page_size())) return size; 104 | return _mi_align_up(size, _mi_os_large_page_size()); 105 | } 106 | 107 | // Return if a pointer points into a region reserved by us. 108 | bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { 109 | size_t count = mi_atomic_read(®ions_count); 110 | for (size_t i = 0; i < count; i++) { 111 | uint8_t* start = (uint8_t*)mi_atomic_read_ptr(®ions[i].start); 112 | if (start != NULL && (uint8_t*)p >= start && (uint8_t*)p < start + MI_REGION_SIZE) return true; 113 | } 114 | return false; 115 | } 116 | 117 | /* ---------------------------------------------------------------------------- 118 | Commit from a region 119 | -----------------------------------------------------------------------------*/ 120 | 121 | #define ALLOCATING ((void*)1) 122 | 123 | // Commit the `blocks` in `region` at `idx` and `bitidx` of a given `size`. 124 | // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written 125 | // if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call. 126 | // (not being able to claim is not considered an error so check for `p != NULL` afterwards). 127 | static bool mi_region_commit_blocks(mem_region_t* region, size_t idx, size_t bitidx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) 128 | { 129 | size_t mask = mi_region_block_mask(blocks,bitidx); 130 | mi_assert_internal(mask != 0); 131 | mi_assert_internal((mask & mi_atomic_read(®ion->map)) == mask); 132 | 133 | // ensure the region is reserved 134 | void* start; 135 | do { 136 | start = mi_atomic_read_ptr(®ion->start); 137 | if (start == NULL) { 138 | start = ALLOCATING; // try to start allocating 139 | } 140 | else if (start == ALLOCATING) { 141 | mi_atomic_yield(); // another thead is already allocating.. wait it out 142 | continue; 143 | } 144 | } while( start == ALLOCATING && !mi_atomic_compare_exchange_ptr(®ion->start, ALLOCATING, NULL) ); 145 | mi_assert_internal(start != NULL); 146 | 147 | // allocate the region if needed 148 | if (start == ALLOCATING) { 149 | start = _mi_os_alloc_aligned(MI_REGION_SIZE, MI_SEGMENT_ALIGN, mi_option_is_enabled(mi_option_eager_region_commit), tld); 150 | // set the new allocation (or NULL on failure) -- this releases any waiting threads. 151 | mi_atomic_write_ptr(®ion->start, start); 152 | 153 | if (start == NULL) { 154 | // failure to allocate from the OS! unclaim the blocks and fail 155 | size_t map; 156 | do { 157 | map = mi_atomic_read(®ion->map); 158 | } while (!mi_atomic_compare_exchange(®ion->map, map & ~mask, map)); 159 | return false; 160 | } 161 | 162 | // update the region count if this is a new max idx. 163 | mi_atomic_compare_exchange(®ions_count, idx+1, idx); 164 | } 165 | mi_assert_internal(start != NULL && start != ALLOCATING); 166 | mi_assert_internal(start == mi_atomic_read_ptr(®ion->start)); 167 | 168 | // Commit the blocks to memory 169 | void* blocks_start = (uint8_t*)start + (bitidx * MI_SEGMENT_SIZE); 170 | if (commit && !mi_option_is_enabled(mi_option_eager_region_commit)) { 171 | _mi_os_commit(blocks_start, mi_good_commit_size(size), tld->stats); // only commit needed size (unless using large OS pages) 172 | } 173 | 174 | // and return the allocation 175 | mi_atomic_write(®ion_next_idx,idx); // next search from here 176 | *p = blocks_start; 177 | *id = (idx*MI_REGION_MAP_BITS) + bitidx; 178 | return true; 179 | } 180 | 181 | // Allocate `blocks` in a `region` at `idx` of a given `size`. 182 | // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written 183 | // if the blocks were successfully claimed so ensure they are initialized to NULL/SIZE_MAX before the call. 184 | // (not being able to claim is not considered an error so check for `p != NULL` afterwards). 185 | static bool mi_region_alloc_blocks(mem_region_t* region, size_t idx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) 186 | { 187 | mi_assert_internal(p != NULL && id != NULL); 188 | mi_assert_internal(blocks < MI_REGION_MAP_BITS); 189 | 190 | const uintptr_t mask = mi_region_block_mask(blocks,0); 191 | const size_t bitidx_max = MI_REGION_MAP_BITS - blocks; 192 | 193 | // scan linearly for a free range of zero bits 194 | uintptr_t map = mi_atomic_read(®ion->map); 195 | uintptr_t m = mask; // the mask shifted by bitidx 196 | for(size_t bitidx = 0; bitidx <= bitidx_max; bitidx++, m <<= 1) { 197 | if ((map & m) == 0) { // are the mask bits free at bitidx? 198 | mi_assert_internal((m >> bitidx) == mask); // no overflow? 199 | uintptr_t newmap = map | m; 200 | mi_assert_internal((newmap^map) >> bitidx == mask); 201 | if (!mi_atomic_compare_exchange(®ion->map, newmap, map)) { 202 | // no success, another thread claimed concurrently.. keep going 203 | map = mi_atomic_read(®ion->map); 204 | } 205 | else { 206 | // success, we claimed the bits 207 | // now commit the block memory -- this can still fail 208 | return mi_region_commit_blocks(region, idx, bitidx, blocks, size, commit, p, id, tld); 209 | } 210 | } 211 | } 212 | // no error, but also no bits found 213 | return true; 214 | } 215 | 216 | // Try to allocate `blocks` in a `region` at `idx` of a given `size`. Does a quick check before trying to claim. 217 | // Returns `false` on an error (OOM); `true` otherwise. `p` and `id` are only written 218 | // if the blocks were successfully claimed so ensure they are initialized to NULL/0 before the call. 219 | // (not being able to claim is not considered an error so check for `p != NULL` afterwards). 220 | static bool mi_region_try_alloc_blocks(size_t idx, size_t blocks, size_t size, bool commit, void** p, size_t* id, mi_os_tld_t* tld) 221 | { 222 | // check if there are available blocks in the region.. 223 | mi_assert_internal(idx < MI_REGION_MAX); 224 | mem_region_t* region = ®ions[idx]; 225 | uintptr_t m = mi_atomic_read(®ion->map); 226 | if (m != MI_REGION_MAP_FULL) { // some bits are zero 227 | return mi_region_alloc_blocks(region, idx, blocks, size, commit, p, id, tld); 228 | } 229 | else { 230 | return true; // no error, but no success either 231 | } 232 | } 233 | 234 | /* ---------------------------------------------------------------------------- 235 | Allocation 236 | -----------------------------------------------------------------------------*/ 237 | 238 | // Allocate `size` memory aligned at `alignment`. Return non NULL on success, with a given memory `id`. 239 | // (`id` is abstract, but `id = idx*MI_REGION_MAP_BITS + bitidx`) 240 | void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t* id, mi_os_tld_t* tld) 241 | { 242 | mi_assert_internal(id != NULL && tld != NULL); 243 | mi_assert_internal(size > 0); 244 | *id = SIZE_MAX; 245 | 246 | // use direct OS allocation for huge blocks or alignment (with `id = SIZE_MAX`) 247 | if (size > MI_REGION_MAX_ALLOC_SIZE || alignment > MI_SEGMENT_ALIGN) { 248 | return _mi_os_alloc_aligned(mi_good_commit_size(size), alignment, true, tld); // round up size 249 | } 250 | 251 | // always round size to OS page size multiple (so commit/decommit go over the entire range) 252 | // TODO: use large OS page size here? 253 | size = _mi_align_up(size, _mi_os_page_size()); 254 | 255 | // calculate the number of needed blocks 256 | size_t blocks = mi_region_block_count(size); 257 | mi_assert_internal(blocks > 0 && blocks <= 8*MI_INTPTR_SIZE); 258 | 259 | // find a range of free blocks 260 | void* p = NULL; 261 | size_t count = mi_atomic_read(®ions_count); 262 | size_t idx = mi_atomic_read(®ion_next_idx); 263 | for (size_t visited = 0; visited < count; visited++, idx++) { 264 | if (idx >= count) idx = 0; // wrap around 265 | if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, &p, id, tld)) return NULL; // error 266 | if (p != NULL) break; 267 | } 268 | 269 | if (p == NULL) { 270 | // no free range in existing regions -- try to extend beyond the count.. but at most 4 regions 271 | for (idx = count; idx < count + 4 && idx < MI_REGION_MAX; idx++) { 272 | if (!mi_region_try_alloc_blocks(idx, blocks, size, commit, &p, id, tld)) return NULL; // error 273 | if (p != NULL) break; 274 | } 275 | } 276 | 277 | if (p == NULL) { 278 | // we could not find a place to allocate, fall back to the os directly 279 | p = _mi_os_alloc_aligned(size, alignment, commit, tld); 280 | } 281 | 282 | mi_assert_internal( p == NULL || (uintptr_t)p % alignment == 0); 283 | return p; 284 | } 285 | 286 | 287 | // Allocate `size` memory. Return non NULL on success, with a given memory `id`. 288 | void* _mi_mem_alloc(size_t size, bool commit, size_t* id, mi_os_tld_t* tld) { 289 | return _mi_mem_alloc_aligned(size,0,commit,id,tld); 290 | } 291 | 292 | /* ---------------------------------------------------------------------------- 293 | Free 294 | -----------------------------------------------------------------------------*/ 295 | 296 | // Free previously allocated memory with a given id. 297 | void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats) { 298 | mi_assert_internal(size > 0 && stats != NULL); 299 | if (p==NULL) return; 300 | if (size==0) return; 301 | if (id == SIZE_MAX) { 302 | // was a direct OS allocation, pass through 303 | _mi_os_free(p, size, stats); 304 | } 305 | else { 306 | // allocated in a region 307 | mi_assert_internal(size <= MI_REGION_MAX_ALLOC_SIZE); if (size > MI_REGION_MAX_ALLOC_SIZE) return; 308 | // we can align the size up to page size (as we allocate that way too) 309 | // this ensures we fully commit/decommit/reset 310 | size = _mi_align_up(size, _mi_os_page_size()); 311 | size_t idx = (id / MI_REGION_MAP_BITS); 312 | size_t bitidx = (id % MI_REGION_MAP_BITS); 313 | size_t blocks = mi_region_block_count(size); 314 | size_t mask = mi_region_block_mask(blocks, bitidx); 315 | mi_assert_internal(idx < MI_REGION_MAX); if (idx >= MI_REGION_MAX) return; // or `abort`? 316 | mem_region_t* region = ®ions[idx]; 317 | mi_assert_internal((mi_atomic_read(®ion->map) & mask) == mask ); // claimed? 318 | void* start = mi_atomic_read_ptr(®ion->start); 319 | mi_assert_internal(start != NULL); 320 | void* blocks_start = (uint8_t*)start + (bitidx * MI_SEGMENT_SIZE); 321 | mi_assert_internal(blocks_start == p); // not a pointer in our area? 322 | mi_assert_internal(bitidx + blocks <= MI_REGION_MAP_BITS); 323 | if (blocks_start != p || bitidx + blocks > MI_REGION_MAP_BITS) return; // or `abort`? 324 | 325 | // decommit (or reset) the blocks to reduce the working set. 326 | // TODO: implement delayed decommit/reset as these calls are too expensive 327 | // if the memory is reused soon. 328 | // reset: 10x slowdown on malloc-large, decommit: 17x slowdown on malloc-large 329 | if (!mi_option_is_enabled(mi_option_large_os_pages)) { 330 | if (mi_option_is_enabled(mi_option_eager_region_commit)) { 331 | //_mi_os_reset(p, size, stats); 332 | } 333 | else { 334 | //_mi_os_decommit(p, size, stats); 335 | } 336 | } 337 | 338 | // TODO: should we free empty regions? currently only done _mi_mem_collect. 339 | // this frees up virtual address space which 340 | // might be useful on 32-bit systems? 341 | 342 | // and unclaim 343 | uintptr_t map; 344 | uintptr_t newmap; 345 | do { 346 | map = mi_atomic_read(®ion->map); 347 | newmap = map & ~mask; 348 | } while (!mi_atomic_compare_exchange(®ion->map, newmap, map)); 349 | } 350 | } 351 | 352 | 353 | /* ---------------------------------------------------------------------------- 354 | collection 355 | -----------------------------------------------------------------------------*/ 356 | void _mi_mem_collect(mi_stats_t* stats) { 357 | // free every region that has no segments in use. 358 | for (size_t i = 0; i < regions_count; i++) { 359 | mem_region_t* region = ®ions[i]; 360 | if (mi_atomic_read(®ion->map) == 0 && region->start != NULL) { 361 | // if no segments used, try to claim the whole region 362 | uintptr_t m; 363 | do { 364 | m = mi_atomic_read(®ion->map); 365 | } while(m == 0 && !mi_atomic_compare_exchange(®ion->map, ~((uintptr_t)0), 0 )); 366 | if (m == 0) { 367 | // on success, free the whole region 368 | if (region->start != NULL) _mi_os_free((void*)region->start, MI_REGION_SIZE, stats); 369 | // and release 370 | region->start = 0; 371 | mi_atomic_write(®ion->map,0); 372 | } 373 | } 374 | } 375 | } 376 | 377 | /* ---------------------------------------------------------------------------- 378 | Other 379 | -----------------------------------------------------------------------------*/ 380 | 381 | bool _mi_mem_commit(void* p, size_t size, mi_stats_t* stats) { 382 | return _mi_os_commit(p, size, stats); 383 | } 384 | 385 | bool _mi_mem_decommit(void* p, size_t size, mi_stats_t* stats) { 386 | return _mi_os_decommit(p, size, stats); 387 | } 388 | 389 | bool _mi_mem_reset(void* p, size_t size, mi_stats_t* stats) { 390 | return _mi_os_reset(p, size, stats); 391 | } 392 | 393 | bool _mi_mem_unreset(void* p, size_t size, mi_stats_t* stats) { 394 | return _mi_os_unreset(p, size, stats); 395 | } 396 | 397 | bool _mi_mem_protect(void* p, size_t size) { 398 | return _mi_os_protect(p, size); 399 | } 400 | 401 | bool _mi_mem_unprotect(void* p, size_t size) { 402 | return _mi_os_unprotect(p, size); 403 | } 404 | -------------------------------------------------------------------------------- /mimalloc/mimalloc-atomic.h: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #pragma once 8 | #ifndef MIMALLOC_ATOMIC_H 9 | #define MIMALLOC_ATOMIC_H 10 | 11 | // ------------------------------------------------------ 12 | // Atomics 13 | // ------------------------------------------------------ 14 | 15 | // Atomically increment a value; returns the incremented result. 16 | static inline uintptr_t mi_atomic_increment(volatile uintptr_t* p); 17 | 18 | // Atomically increment a value; returns the incremented result. 19 | static inline uint32_t mi_atomic_increment32(volatile uint32_t* p); 20 | 21 | // Atomically decrement a value; returns the decremented result. 22 | static inline uintptr_t mi_atomic_decrement(volatile uintptr_t* p); 23 | 24 | // Atomically add a 64-bit value; returns the added result. 25 | static inline int64_t mi_atomic_add(volatile int64_t* p, int64_t add); 26 | 27 | // Atomically subtract a value; returns the subtracted result. 28 | static inline uintptr_t mi_atomic_subtract(volatile uintptr_t* p, uintptr_t sub); 29 | 30 | // Atomically subtract a value; returns the subtracted result. 31 | static inline uint32_t mi_atomic_subtract32(volatile uint32_t* p, uint32_t sub); 32 | 33 | // Atomically compare and exchange a value; returns `true` if successful. 34 | static inline bool mi_atomic_compare_exchange32(volatile uint32_t* p, uint32_t exchange, uint32_t compare); 35 | 36 | // Atomically compare and exchange a value; returns `true` if successful. 37 | static inline bool mi_atomic_compare_exchange(volatile uintptr_t* p, uintptr_t exchange, uintptr_t compare); 38 | 39 | // Atomically exchange a value. 40 | static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exchange); 41 | 42 | // Atomically read a value 43 | static inline uintptr_t mi_atomic_read(volatile uintptr_t* p); 44 | 45 | // Atomically write a value 46 | static inline void mi_atomic_write(volatile uintptr_t* p, uintptr_t x); 47 | 48 | // Atomically read a pointer 49 | static inline void* mi_atomic_read_ptr(volatile void** p) { 50 | return (void*)mi_atomic_read( (volatile uintptr_t*)p ); 51 | } 52 | 53 | static inline void mi_atomic_yield(void); 54 | 55 | 56 | // Atomically write a pointer 57 | static inline void mi_atomic_write_ptr(volatile void** p, void* x) { 58 | mi_atomic_write((volatile uintptr_t*)p, (uintptr_t)x ); 59 | } 60 | 61 | // Atomically compare and exchange a pointer; returns `true` if successful. 62 | static inline bool mi_atomic_compare_exchange_ptr(volatile void** p, void* newp, void* compare) { 63 | return mi_atomic_compare_exchange((volatile uintptr_t*)p, (uintptr_t)newp, (uintptr_t)compare); 64 | } 65 | 66 | // Atomically exchange a pointer value. 67 | static inline void* mi_atomic_exchange_ptr(volatile void** p, void* exchange) { 68 | return (void*)mi_atomic_exchange((volatile uintptr_t*)p, (uintptr_t)exchange); 69 | } 70 | 71 | 72 | #ifdef _MSC_VER 73 | #define WIN32_LEAN_AND_MEAN 74 | #include 75 | #include 76 | #if (MI_INTPTR_SIZE==8) 77 | typedef LONG64 msc_intptr_t; 78 | #define RC64(f) f##64 79 | #else 80 | typedef LONG msc_intptr_t; 81 | #define RC64(f) f 82 | #endif 83 | static inline uintptr_t mi_atomic_increment(volatile uintptr_t* p) { 84 | return (uintptr_t)RC64(_InterlockedIncrement)((volatile msc_intptr_t*)p); 85 | } 86 | static inline uint32_t mi_atomic_increment32(volatile uint32_t* p) { 87 | return (uint32_t)_InterlockedIncrement((volatile LONG*)p); 88 | } 89 | static inline uintptr_t mi_atomic_decrement(volatile uintptr_t* p) { 90 | return (uintptr_t)RC64(_InterlockedDecrement)((volatile msc_intptr_t*)p); 91 | } 92 | static inline uintptr_t mi_atomic_subtract(volatile uintptr_t* p, uintptr_t sub) { 93 | return (uintptr_t)RC64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub)) - sub; 94 | } 95 | static inline uint32_t mi_atomic_subtract32(volatile uint32_t* p, uint32_t sub) { 96 | return (uint32_t)_InterlockedExchangeAdd((volatile LONG*)p, -((LONG)sub)) - sub; 97 | } 98 | static inline bool mi_atomic_compare_exchange32(volatile uint32_t* p, uint32_t exchange, uint32_t compare) { 99 | return ((int32_t)compare == _InterlockedCompareExchange((volatile LONG*)p, (LONG)exchange, (LONG)compare)); 100 | } 101 | static inline bool mi_atomic_compare_exchange(volatile uintptr_t* p, uintptr_t exchange, uintptr_t compare) { 102 | return (compare == RC64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange, (msc_intptr_t)compare)); 103 | } 104 | static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exchange) { 105 | return (uintptr_t)RC64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange); 106 | } 107 | static inline uintptr_t mi_atomic_read(volatile uintptr_t* p) { 108 | return *p; 109 | } 110 | static inline void mi_atomic_write(volatile uintptr_t* p, uintptr_t x) { 111 | *p = x; 112 | } 113 | static inline void mi_atomic_yield(void) { 114 | YieldProcessor(); 115 | } 116 | static inline int64_t mi_atomic_add(volatile int64_t* p, int64_t add) { 117 | #if (MI_INTPTR_SIZE==8) 118 | return _InterlockedExchangeAdd64(p, add) + add; 119 | #else 120 | int64_t current; 121 | int64_t sum; 122 | do { 123 | current = *p; 124 | sum = current + add; 125 | } while (_InterlockedCompareExchange64(p, sum, current) != current); 126 | return sum; 127 | #endif 128 | } 129 | 130 | #else 131 | #ifdef __cplusplus 132 | #include 133 | #define MI_USING_STD using namespace std; 134 | #define _Atomic(tp) atomic 135 | #else 136 | #include 137 | #define MI_USING_STD 138 | #endif 139 | static inline uintptr_t mi_atomic_increment(volatile uintptr_t* p) { 140 | MI_USING_STD 141 | return atomic_fetch_add_explicit((volatile atomic_uintptr_t*)p, (uintptr_t)1, memory_order_relaxed) + 1; 142 | } 143 | static inline uint32_t mi_atomic_increment32(volatile uint32_t* p) { 144 | MI_USING_STD 145 | return atomic_fetch_add_explicit((volatile _Atomic(uint32_t)*)p, (uint32_t)1, memory_order_relaxed) + 1; 146 | } 147 | static inline uintptr_t mi_atomic_decrement(volatile uintptr_t* p) { 148 | MI_USING_STD 149 | return atomic_fetch_sub_explicit((volatile atomic_uintptr_t*)p, (uintptr_t)1, memory_order_relaxed) - 1; 150 | } 151 | static inline int64_t mi_atomic_add(volatile int64_t* p, int64_t add) { 152 | MI_USING_STD 153 | return atomic_fetch_add_explicit((volatile _Atomic(int64_t)*)p, add, memory_order_relaxed) + add; 154 | } 155 | static inline uintptr_t mi_atomic_subtract(volatile uintptr_t* p, uintptr_t sub) { 156 | MI_USING_STD 157 | return atomic_fetch_sub_explicit((volatile atomic_uintptr_t*)p, sub, memory_order_relaxed) - sub; 158 | } 159 | static inline uint32_t mi_atomic_subtract32(volatile uint32_t* p, uint32_t sub) { 160 | MI_USING_STD 161 | return atomic_fetch_sub_explicit((volatile _Atomic(uint32_t)*)p, sub, memory_order_relaxed) - sub; 162 | } 163 | static inline bool mi_atomic_compare_exchange32(volatile uint32_t* p, uint32_t exchange, uint32_t compare) { 164 | MI_USING_STD 165 | return atomic_compare_exchange_weak_explicit((volatile _Atomic(uint32_t)*)p, &compare, exchange, memory_order_release, memory_order_relaxed); 166 | } 167 | static inline bool mi_atomic_compare_exchange(volatile uintptr_t* p, uintptr_t exchange, uintptr_t compare) { 168 | MI_USING_STD 169 | return atomic_compare_exchange_weak_explicit((volatile atomic_uintptr_t*)p, &compare, exchange, memory_order_release, memory_order_relaxed); 170 | } 171 | static inline uintptr_t mi_atomic_exchange(volatile uintptr_t* p, uintptr_t exchange) { 172 | MI_USING_STD 173 | return atomic_exchange_explicit((volatile atomic_uintptr_t*)p, exchange, memory_order_acquire); 174 | } 175 | static inline uintptr_t mi_atomic_read(volatile uintptr_t* p) { 176 | MI_USING_STD 177 | return atomic_load_explicit((volatile atomic_uintptr_t*)p, memory_order_relaxed); 178 | } 179 | static inline void mi_atomic_write(volatile uintptr_t* p, uintptr_t x) { 180 | MI_USING_STD 181 | return atomic_store_explicit((volatile atomic_uintptr_t*)p, x, memory_order_relaxed); 182 | } 183 | 184 | #if defined(__cplusplus) 185 | #include 186 | static inline void mi_atomic_yield(void) { 187 | std::this_thread::yield(); 188 | } 189 | #elif (defined(__GNUC__) || defined(__clang__)) && \ 190 | (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__)) 191 | #if defined(__x86_64__) || defined(__i386__) 192 | static inline void mi_atomic_yield(void) { 193 | asm volatile ("pause" ::: "memory"); 194 | } 195 | #elif defined(__arm__) || defined(__aarch64__) 196 | static inline void mi_atomic_yield(void) { 197 | asm volatile("yield"); 198 | } 199 | #endif 200 | #elif defined(__wasi__) 201 | #include 202 | static inline void mi_atomic_yield() { 203 | sched_yield(); 204 | } 205 | #else 206 | #include 207 | static inline void mi_atomic_yield(void) { 208 | sleep(0); 209 | } 210 | #endif 211 | 212 | #endif 213 | 214 | #endif // __MIMALLOC_ATOMIC_H 215 | -------------------------------------------------------------------------------- /mimalloc/mimalloc-internal.h: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #pragma once 8 | #ifndef MIMALLOC_INTERNAL_H 9 | #define MIMALLOC_INTERNAL_H 10 | 11 | #include "mimalloc-types.h" 12 | 13 | #if defined(MI_MALLOC_OVERRIDE) && (defined(__APPLE__) || defined(__OpenBSD__)) 14 | #define MI_TLS_RECURSE_GUARD 15 | #endif 16 | 17 | #if (MI_DEBUG>0) 18 | #define mi_trace_message(...) _mi_trace_message(__VA_ARGS__) 19 | #else 20 | #define mi_trace_message(...) 21 | #endif 22 | 23 | 24 | // "options.c" 25 | void _mi_fprintf(FILE* out, const char* fmt, ...); 26 | void _mi_error_message(const char* fmt, ...); 27 | void _mi_warning_message(const char* fmt, ...); 28 | void _mi_verbose_message(const char* fmt, ...); 29 | void _mi_trace_message(const char* fmt, ...); 30 | 31 | // "init.c" 32 | extern mi_stats_t _mi_stats_main; 33 | extern const mi_page_t _mi_page_empty; 34 | bool _mi_is_main_thread(void); 35 | uintptr_t _mi_ptr_cookie(const void* p); 36 | uintptr_t _mi_random_shuffle(uintptr_t x); 37 | uintptr_t _mi_random_init(uintptr_t seed /* can be zero */); 38 | 39 | // os.c 40 | size_t _mi_os_page_size(void); 41 | uintptr_t _mi_align_up(uintptr_t sz, size_t alignment); 42 | void _mi_os_init(void); // called from process init 43 | void* _mi_os_alloc(size_t size, mi_stats_t* stats); // to allocate thread local data 44 | void _mi_os_free(void* p, size_t size, mi_stats_t* stats); // to free thread local data 45 | 46 | // memory.c 47 | void* _mi_mem_alloc_aligned(size_t size, size_t alignment, bool commit, size_t* id, mi_os_tld_t* tld); 48 | void* _mi_mem_alloc(size_t size, bool commit, size_t* id, mi_os_tld_t* tld); 49 | void _mi_mem_free(void* p, size_t size, size_t id, mi_stats_t* stats); 50 | 51 | bool _mi_mem_reset(void* p, size_t size, mi_stats_t* stats); 52 | bool _mi_mem_unreset(void* p, size_t size, mi_stats_t* stats); 53 | bool _mi_mem_commit(void* p, size_t size, mi_stats_t* stats); 54 | bool _mi_mem_protect(void* addr, size_t size); 55 | bool _mi_mem_unprotect(void* addr, size_t size); 56 | 57 | void _mi_mem_collect(mi_stats_t* stats); 58 | 59 | // "segment.c" 60 | mi_page_t* _mi_segment_page_alloc(size_t block_wsize, mi_segments_tld_t* tld, mi_os_tld_t* os_tld); 61 | void _mi_segment_page_free(mi_page_t* page, bool force, mi_segments_tld_t* tld); 62 | void _mi_segment_page_abandon(mi_page_t* page, mi_segments_tld_t* tld); 63 | bool _mi_segment_try_reclaim_abandoned( mi_heap_t* heap, bool try_all, mi_segments_tld_t* tld); 64 | void _mi_segment_thread_collect(mi_segments_tld_t* tld); 65 | uint8_t* _mi_segment_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t block_size, size_t* page_size); // page start for any page 66 | 67 | // "page.c" 68 | void* _mi_malloc_generic(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc; 69 | 70 | void _mi_page_retire(mi_page_t* page); // free the page if there are no other pages with many free blocks 71 | void _mi_page_unfull(mi_page_t* page); 72 | void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq, bool force); // free the page 73 | void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... 74 | void _mi_heap_delayed_free(mi_heap_t* heap); 75 | 76 | void _mi_page_use_delayed_free(mi_page_t* page, mi_delayed_t delay); 77 | size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append); 78 | void _mi_deferred_free(mi_heap_t* heap, bool force); 79 | 80 | void _mi_page_free_collect(mi_page_t* page); 81 | void _mi_page_reclaim(mi_heap_t* heap, mi_page_t* page); // callback from segments 82 | 83 | size_t _mi_bin_size(uint8_t bin); // for stats 84 | uint8_t _mi_bin(size_t size); // for stats 85 | uint8_t _mi_bsr(uintptr_t x); // bit-scan-right, used on BSD in "os.c" 86 | 87 | // "heap.c" 88 | void _mi_heap_destroy_pages(mi_heap_t* heap); 89 | void _mi_heap_collect_abandon(mi_heap_t* heap); 90 | uintptr_t _mi_heap_random(mi_heap_t* heap); 91 | 92 | // "stats.c" 93 | void _mi_stats_done(mi_stats_t* stats); 94 | 95 | // "alloc.c" 96 | void* _mi_page_malloc(mi_heap_t* heap, mi_page_t* page, size_t size) mi_attr_noexcept; // called from `_mi_malloc_generic` 97 | void* _mi_heap_malloc_zero(mi_heap_t* heap, size_t size, bool zero); 98 | void* _mi_heap_realloc_zero(mi_heap_t* heap, void* p, size_t newsize, bool zero); 99 | mi_block_t* _mi_page_ptr_unalign(const mi_segment_t* segment, const mi_page_t* page, const void* p); 100 | bool _mi_free_delayed_block(mi_block_t* block); 101 | 102 | #if MI_DEBUG>1 103 | bool _mi_page_is_valid(mi_page_t* page); 104 | #endif 105 | 106 | 107 | // ------------------------------------------------------ 108 | // Branches 109 | // ------------------------------------------------------ 110 | 111 | #if defined(__GNUC__) || defined(__clang__) 112 | #define mi_unlikely(x) __builtin_expect((x),0) 113 | #define mi_likely(x) __builtin_expect((x),1) 114 | #else 115 | #define mi_unlikely(x) (x) 116 | #define mi_likely(x) (x) 117 | #endif 118 | 119 | #ifndef __has_builtin 120 | #define __has_builtin(x) 0 121 | #endif 122 | 123 | #if defined(_MSC_VER) 124 | #define mi_decl_noinline __declspec(noinline) 125 | #elif defined(__GNUC__) || defined(__clang__) 126 | #define mi_decl_noinline __attribute__((noinline)) 127 | #else 128 | #define mi_decl_noinline 129 | #endif 130 | 131 | 132 | /* ----------------------------------------------------------- 133 | Inlined definitions 134 | ----------------------------------------------------------- */ 135 | #define UNUSED(x) (void)(x) 136 | #if (MI_DEBUG>0) 137 | #define UNUSED_RELEASE(x) 138 | #else 139 | #define UNUSED_RELEASE(x) UNUSED(x) 140 | #endif 141 | 142 | #define MI_INIT4(x) x(),x(),x(),x() 143 | #define MI_INIT8(x) MI_INIT4(x),MI_INIT4(x) 144 | #define MI_INIT16(x) MI_INIT8(x),MI_INIT8(x) 145 | #define MI_INIT32(x) MI_INIT16(x),MI_INIT16(x) 146 | #define MI_INIT64(x) MI_INIT32(x),MI_INIT32(x) 147 | #define MI_INIT128(x) MI_INIT64(x),MI_INIT64(x) 148 | #define MI_INIT256(x) MI_INIT128(x),MI_INIT128(x) 149 | 150 | 151 | // Overflow detecting multiply 152 | #define MI_MUL_NO_OVERFLOW ((size_t)1 << (4*sizeof(size_t))) // sqrt(SIZE_MAX) 153 | static inline bool mi_mul_overflow(size_t size, size_t count, size_t* total) { 154 | #if __has_builtin(__builtin_umul_overflow) || __GNUC__ >= 5 155 | #if (MI_INTPTR_SIZE == 4) 156 | return __builtin_umul_overflow(size, count, total); 157 | #else 158 | return __builtin_umull_overflow(size, count, total); 159 | #endif 160 | #else /* __builtin_umul_overflow is unavailable */ 161 | *total = size * count; 162 | return ((size >= MI_MUL_NO_OVERFLOW || count >= MI_MUL_NO_OVERFLOW) 163 | && size > 0 && (SIZE_MAX / size) < count); 164 | #endif 165 | } 166 | 167 | // Align a byte size to a size in _machine words_, 168 | // i.e. byte size == `wsize*sizeof(void*)`. 169 | static inline size_t _mi_wsize_from_size(size_t size) { 170 | mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t)); 171 | return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t); 172 | } 173 | 174 | extern const mi_heap_t _mi_heap_empty; // read-only empty heap, initial value of the thread local default heap 175 | extern mi_heap_t _mi_heap_main; // statically allocated main backing heap 176 | extern bool _mi_process_is_initialized; 177 | 178 | extern mi_decl_thread mi_heap_t* _mi_heap_default; // default heap to allocate from 179 | 180 | static inline mi_heap_t* mi_get_default_heap(void) { 181 | #ifdef MI_TLS_RECURSE_GUARD 182 | // on some platforms, like macOS, the dynamic loader calls `malloc` 183 | // to initialize thread local data. To avoid recursion, we need to avoid 184 | // accessing the thread local `_mi_default_heap` until our module is loaded 185 | // and use the statically allocated main heap until that time. 186 | // TODO: patch ourselves dynamically to avoid this check every time? 187 | if (!_mi_process_is_initialized) return &_mi_heap_main; 188 | #endif 189 | return _mi_heap_default; 190 | } 191 | 192 | static inline bool mi_heap_is_default(const mi_heap_t* heap) { 193 | return (heap == mi_get_default_heap()); 194 | } 195 | 196 | static inline bool mi_heap_is_backing(const mi_heap_t* heap) { 197 | return (heap->tld->heap_backing == heap); 198 | } 199 | 200 | static inline bool mi_heap_is_initialized(mi_heap_t* heap) { 201 | mi_assert_internal(heap != NULL); 202 | return (heap != &_mi_heap_empty); 203 | } 204 | 205 | static inline mi_page_t* _mi_heap_get_free_small_page(mi_heap_t* heap, size_t size) { 206 | mi_assert_internal(size <= MI_SMALL_SIZE_MAX); 207 | return heap->pages_free_direct[_mi_wsize_from_size(size)]; 208 | } 209 | 210 | // Get the page belonging to a certain size class 211 | static inline mi_page_t* _mi_get_free_small_page(size_t size) { 212 | return _mi_heap_get_free_small_page(mi_get_default_heap(), size); 213 | } 214 | 215 | 216 | // Segment that contains the pointer 217 | static inline mi_segment_t* _mi_ptr_segment(const void* p) { 218 | // mi_assert_internal(p != NULL); 219 | return (mi_segment_t*)((uintptr_t)p & ~MI_SEGMENT_MASK); 220 | } 221 | 222 | // Segment belonging to a page 223 | static inline mi_segment_t* _mi_page_segment(const mi_page_t* page) { 224 | mi_segment_t* segment = _mi_ptr_segment(page); 225 | mi_assert_internal(segment == NULL || page == &segment->pages[page->segment_idx]); 226 | return segment; 227 | } 228 | 229 | // Get the page containing the pointer 230 | static inline mi_page_t* _mi_segment_page_of(const mi_segment_t* segment, const void* p) { 231 | // if (segment->page_size > MI_SEGMENT_SIZE) return &segment->pages[0]; // huge pages 232 | ptrdiff_t diff = (uint8_t*)p - (uint8_t*)segment; 233 | mi_assert_internal(diff >= 0 && diff < MI_SEGMENT_SIZE); 234 | uintptr_t idx = (uintptr_t)diff >> segment->page_shift; 235 | mi_assert_internal(idx < segment->capacity); 236 | mi_assert_internal(segment->page_kind <= MI_PAGE_MEDIUM || idx == 0); 237 | return &((mi_segment_t*)segment)->pages[idx]; 238 | } 239 | 240 | // Quick page start for initialized pages 241 | static inline uint8_t* _mi_page_start(const mi_segment_t* segment, const mi_page_t* page, size_t* page_size) { 242 | return _mi_segment_page_start(segment, page, page->block_size, page_size); 243 | } 244 | 245 | // Get the page containing the pointer 246 | static inline mi_page_t* _mi_ptr_page(void* p) { 247 | return _mi_segment_page_of(_mi_ptr_segment(p), p); 248 | } 249 | 250 | // Thread free access 251 | static inline mi_block_t* mi_tf_block(mi_thread_free_t tf) { 252 | return (mi_block_t*)(tf & ~0x03); 253 | } 254 | static inline mi_delayed_t mi_tf_delayed(mi_thread_free_t tf) { 255 | return (mi_delayed_t)(tf & 0x03); 256 | } 257 | static inline mi_thread_free_t mi_tf_make(mi_block_t* block, mi_delayed_t delayed) { 258 | return (mi_thread_free_t)((uintptr_t)block | (uintptr_t)delayed); 259 | } 260 | static inline mi_thread_free_t mi_tf_set_delayed(mi_thread_free_t tf, mi_delayed_t delayed) { 261 | return mi_tf_make(mi_tf_block(tf),delayed); 262 | } 263 | static inline mi_thread_free_t mi_tf_set_block(mi_thread_free_t tf, mi_block_t* block) { 264 | return mi_tf_make(block, mi_tf_delayed(tf)); 265 | } 266 | 267 | // are all blocks in a page freed? 268 | static inline bool mi_page_all_free(const mi_page_t* page) { 269 | mi_assert_internal(page != NULL); 270 | return (page->used - page->thread_freed == 0); 271 | } 272 | 273 | // are there immediately available blocks 274 | static inline bool mi_page_immediate_available(const mi_page_t* page) { 275 | mi_assert_internal(page != NULL); 276 | return (page->free != NULL); 277 | } 278 | // are there free blocks in this page? 279 | static inline bool mi_page_has_free(mi_page_t* page) { 280 | mi_assert_internal(page != NULL); 281 | bool hasfree = (mi_page_immediate_available(page) || page->local_free != NULL || (mi_tf_block(page->thread_free) != NULL)); 282 | mi_assert_internal(hasfree || page->used - page->thread_freed == page->capacity); 283 | return hasfree; 284 | } 285 | 286 | // are all blocks in use? 287 | static inline bool mi_page_all_used(mi_page_t* page) { 288 | mi_assert_internal(page != NULL); 289 | return !mi_page_has_free(page); 290 | } 291 | 292 | // is more than 7/8th of a page in use? 293 | static inline bool mi_page_mostly_used(const mi_page_t* page) { 294 | if (page==NULL) return true; 295 | uint16_t frac = page->reserved / 8U; 296 | return (page->reserved - page->used + page->thread_freed < frac); 297 | } 298 | 299 | static inline mi_page_queue_t* mi_page_queue(const mi_heap_t* heap, size_t size) { 300 | return &((mi_heap_t*)heap)->pages[_mi_bin(size)]; 301 | } 302 | 303 | // ------------------------------------------------------------------- 304 | // Encoding/Decoding the free list next pointers 305 | // ------------------------------------------------------------------- 306 | 307 | static inline mi_block_t* mi_block_nextx( uintptr_t cookie, mi_block_t* block ) { 308 | #if MI_SECURE 309 | return (mi_block_t*)(block->next ^ cookie); 310 | #else 311 | UNUSED(cookie); 312 | return (mi_block_t*)block->next; 313 | #endif 314 | } 315 | 316 | static inline void mi_block_set_nextx(uintptr_t cookie, mi_block_t* block, mi_block_t* next) { 317 | #if MI_SECURE 318 | block->next = (mi_encoded_t)next ^ cookie; 319 | #else 320 | UNUSED(cookie); 321 | block->next = (mi_encoded_t)next; 322 | #endif 323 | } 324 | 325 | static inline mi_block_t* mi_block_next(mi_page_t* page, mi_block_t* block) { 326 | return mi_block_nextx(page->cookie,block); 327 | } 328 | 329 | static inline void mi_block_set_next(mi_page_t* page, mi_block_t* block, mi_block_t* next) { 330 | mi_block_set_nextx(page->cookie,block,next); 331 | } 332 | // ------------------------------------------------------------------- 333 | // Getting the thread id should be performant 334 | // as it is called in the fast path of `_mi_free`, 335 | // so we specialize for various platforms. 336 | // ------------------------------------------------------------------- 337 | #if defined(_WIN32) 338 | #define WIN32_LEAN_AND_MEAN 339 | #include 340 | static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { 341 | // Windows: works on Intel and ARM in both 32- and 64-bit 342 | return (uintptr_t)NtCurrentTeb(); 343 | } 344 | #elif (defined(__GNUC__) || defined(__clang__)) && \ 345 | (defined(__x86_64__) || defined(__i386__) || defined(__arm__) || defined(__aarch64__)) 346 | // TLS register on x86 is in the FS or GS register 347 | // see: https://akkadia.org/drepper/tls.pdf 348 | static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { 349 | uintptr_t tid; 350 | #if defined(__i386__) 351 | __asm__("movl %%gs:0, %0" : "=r" (tid) : : ); // 32-bit always uses GS 352 | #elif defined(__MACH__) 353 | __asm__("movq %%gs:0, %0" : "=r" (tid) : : ); // x86_64 macOS uses GS 354 | #elif defined(__x86_64__) 355 | __asm__("movq %%fs:0, %0" : "=r" (tid) : : ); // x86_64 Linux, BSD uses FS 356 | #elif defined(__arm__) 357 | asm volatile ("mrc p15, 0, %0, c13, c0, 3" : "=r" (tid)); 358 | #elif defined(__aarch64__) 359 | asm volatile ("mrs %0, tpidr_el0" : "=r" (tid)); 360 | #endif 361 | return tid; 362 | } 363 | #else 364 | // otherwise use standard C 365 | static inline uintptr_t _mi_thread_id(void) mi_attr_noexcept { 366 | return (uintptr_t)&_mi_heap_default; 367 | } 368 | #endif 369 | 370 | 371 | #endif 372 | -------------------------------------------------------------------------------- /mimalloc/mimalloc-types.h: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #pragma once 8 | #ifndef MIMALLOC_TYPES_H 9 | #define MIMALLOC_TYPES_H 10 | 11 | #include // size_t etc. 12 | #include // ptrdiff_t 13 | #include // uintptr_t, uint16_t, etc 14 | 15 | // ------------------------------------------------------ 16 | // Variants 17 | // ------------------------------------------------------ 18 | 19 | // Define NDEBUG in the release version to disable assertions. 20 | // #define NDEBUG 21 | 22 | // Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). 23 | // #define MI_STAT 1 24 | 25 | // Define MI_SECURE as 1 to encode free lists 26 | // #define MI_SECURE 1 27 | 28 | #if !defined(MI_SECURE) 29 | #define MI_SECURE 0 30 | #endif 31 | 32 | // Define MI_DEBUG as 1 for basic assert checks and statistics 33 | // set it to 2 to do internal asserts, 34 | // and to 3 to do extensive invariant checking. 35 | #if !defined(MI_DEBUG) 36 | #if !defined(NDEBUG) || defined(_DEBUG) 37 | #define MI_DEBUG 1 38 | #else 39 | #define MI_DEBUG 0 40 | #endif 41 | #endif 42 | 43 | 44 | // ------------------------------------------------------ 45 | // Platform specific values 46 | // ------------------------------------------------------ 47 | 48 | 49 | // ------------------------------------------------------ 50 | // Size of a pointer. 51 | // We assume that `sizeof(void*)==sizeof(intptr_t)` 52 | // and it holds for all platforms we know of. 53 | // 54 | // However, the C standard only requires that: 55 | // p == (void*)((intptr_t)p)) 56 | // but we also need: 57 | // i == (intptr_t)((void*)i) 58 | // or otherwise one might define an intptr_t type that is larger than a pointer... 59 | // ------------------------------------------------------ 60 | 61 | #if INTPTR_MAX == 9223372036854775807LL 62 | # define MI_INTPTR_SHIFT (3) 63 | #elif INTPTR_MAX == 2147483647LL 64 | # define MI_INTPTR_SHIFT (2) 65 | #else 66 | #error platform must be 32 or 64 bits 67 | #endif 68 | 69 | #define MI_INTPTR_SIZE (1<>MI_INTPTR_SHIFT) 98 | 99 | 100 | // Maximum number of size classes. (spaced exponentially in 16.7% increments) 101 | #define MI_BIN_HUGE (64U) 102 | 103 | // Minimal alignment necessary. On most platforms 16 bytes are needed 104 | // due to SSE registers for example. This must be at least `MI_INTPTR_SIZE` 105 | #define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) 106 | 107 | #if (MI_LARGE_WSIZE_MAX > 131072) 108 | #error "define more bins" 109 | #endif 110 | 111 | typedef uintptr_t mi_encoded_t; 112 | 113 | // free lists contain blocks 114 | typedef struct mi_block_s { 115 | mi_encoded_t next; 116 | } mi_block_t; 117 | 118 | 119 | typedef enum mi_delayed_e { 120 | MI_NO_DELAYED_FREE = 0, 121 | MI_USE_DELAYED_FREE = 1, 122 | MI_DELAYED_FREEING = 2, 123 | MI_NEVER_DELAYED_FREE = 3 124 | } mi_delayed_t; 125 | 126 | 127 | typedef union mi_page_flags_u { 128 | uint16_t value; 129 | struct { 130 | bool has_aligned; 131 | bool in_full; 132 | }; 133 | } mi_page_flags_t; 134 | 135 | // Thread free list. 136 | // We use bottom 2 bits of the pointer for mi_delayed_t flags 137 | typedef uintptr_t mi_thread_free_t; 138 | 139 | 140 | // A page contains blocks of one specific size (`block_size`). 141 | // Each page has three list of free blocks: 142 | // `free` for blocks that can be allocated, 143 | // `local_free` for freed blocks that are not yet available to `mi_malloc` 144 | // `thread_free` for freed blocks by other threads 145 | // The `local_free` and `thread_free` lists are migrated to the `free` list 146 | // when it is exhausted. The separate `local_free` list is necessary to 147 | // implement a monotonic heartbeat. The `thread_free` list is needed for 148 | // avoiding atomic operations in the common case. 149 | // 150 | // `used - thread_freed` == actual blocks that are in use (alive) 151 | // `used - thread_freed + |free| + |local_free| == capacity` 152 | // 153 | // note: we don't count `freed` (as |free|) instead of `used` to reduce 154 | // the number of memory accesses in the `mi_page_all_free` function(s). 155 | // note: the funny layout here is due to: 156 | // - access is optimized for `mi_free` and `mi_page_alloc` 157 | // - using `uint16_t` does not seem to slow things down 158 | typedef struct mi_page_s { 159 | // "owned" by the segment 160 | uint8_t segment_idx; // index in the segment `pages` array, `page == &segment->pages[page->segment_idx]` 161 | bool segment_in_use:1; // `true` if the segment allocated this page 162 | bool is_reset:1; // `true` if the page memory was reset 163 | bool is_committed:1; // `true` if the page virtual memory is committed 164 | 165 | // layout like this to optimize access in `mi_malloc` and `mi_free` 166 | mi_page_flags_t flags; 167 | uint16_t capacity; // number of blocks committed 168 | uint16_t reserved; // number of blocks reserved in memory 169 | 170 | mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) 171 | uintptr_t cookie; // random cookie to encode the free lists 172 | size_t used; // number of blocks in use (including blocks in `local_free` and `thread_free`) 173 | 174 | mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) 175 | volatile uintptr_t thread_freed; // at least this number of blocks are in `thread_free` 176 | volatile mi_thread_free_t thread_free; // list of deferred free blocks freed by other threads 177 | 178 | // less accessed info 179 | size_t block_size; // size available in each block (always `>0`) 180 | mi_heap_t* heap; // the owning heap 181 | struct mi_page_s* next; // next page owned by this thread with the same `block_size` 182 | struct mi_page_s* prev; // previous page owned by this thread with the same `block_size` 183 | 184 | // improve page index calculation 185 | #if MI_INTPTR_SIZE==8 186 | //void* padding[1]; // 10 words on 64-bit 187 | #elif MI_INTPTR_SIZE==4 188 | void* padding[1]; // 12 words on 32-bit 189 | #endif 190 | } mi_page_t; 191 | 192 | 193 | 194 | typedef enum mi_page_kind_e { 195 | MI_PAGE_SMALL, // small blocks go into 64kb pages inside a segment 196 | MI_PAGE_MEDIUM, // medium blocks go into 512kb pages inside a segment 197 | MI_PAGE_LARGE, // larger blocks go into a single page spanning a whole segment 198 | MI_PAGE_HUGE // huge blocks (>512kb) are put into a single page in a segment of the exact size (but still 2mb aligned) 199 | } mi_page_kind_t; 200 | 201 | // Segments are large allocated memory blocks (2mb on 64 bit) from 202 | // the OS. Inside segments we allocated fixed size _pages_ that 203 | // contain blocks. 204 | typedef struct mi_segment_s { 205 | struct mi_segment_s* next; 206 | struct mi_segment_s* prev; 207 | struct mi_segment_s* abandoned_next; 208 | size_t abandoned; // abandoned pages (i.e. the original owning thread stopped) (`abandoned <= used`) 209 | size_t used; // count of pages in use (`used <= capacity`) 210 | size_t capacity; // count of available pages (`#free + used`) 211 | size_t segment_size;// for huge pages this may be different from `MI_SEGMENT_SIZE` 212 | size_t segment_info_size; // space we are using from the first page for segment meta-data and possible guard pages. 213 | uintptr_t cookie; // verify addresses in debug mode: `mi_ptr_cookie(segment) == segment->cookie` 214 | size_t memid; // id for the os-level memory manager 215 | 216 | // layout like this to optimize access in `mi_free` 217 | size_t page_shift; // `1 << page_shift` == the page sizes == `page->block_size * page->reserved` (unless the first page, then `-segment_info_size`). 218 | uintptr_t thread_id; // unique id of the thread owning this segment 219 | mi_page_kind_t page_kind; // kind of pages: small, large, or huge 220 | mi_page_t pages[1]; // up to `MI_SMALL_PAGES_PER_SEGMENT` pages 221 | } mi_segment_t; 222 | 223 | 224 | // ------------------------------------------------------ 225 | // Heaps 226 | // Provide first-class heaps to allocate from. 227 | // A heap just owns a set of pages for allocation and 228 | // can only be allocate/reallocate from the thread that created it. 229 | // Freeing blocks can be done from any thread though. 230 | // Per thread, the segments are shared among its heaps. 231 | // Per thread, there is always a default heap that is 232 | // used for allocation; it is initialized to statically 233 | // point to an empty heap to avoid initialization checks 234 | // in the fast path. 235 | // ------------------------------------------------------ 236 | 237 | // Thread local data 238 | typedef struct mi_tld_s mi_tld_t; 239 | 240 | // Pages of a certain block size are held in a queue. 241 | typedef struct mi_page_queue_s { 242 | mi_page_t* first; 243 | mi_page_t* last; 244 | size_t block_size; 245 | } mi_page_queue_t; 246 | 247 | #define MI_BIN_FULL (MI_BIN_HUGE+1) 248 | 249 | // A heap owns a set of pages. 250 | struct mi_heap_s { 251 | mi_tld_t* tld; 252 | mi_page_t* pages_free_direct[MI_SMALL_WSIZE_MAX + 2]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. 253 | mi_page_queue_t pages[MI_BIN_FULL + 1]; // queue of pages for each size class (or "bin") 254 | volatile mi_block_t* thread_delayed_free; 255 | uintptr_t thread_id; // thread this heap belongs too 256 | uintptr_t cookie; 257 | uintptr_t random; // random number used for secure allocation 258 | size_t page_count; // total number of pages in the `pages` queues. 259 | bool no_reclaim; // `true` if this heap should not reclaim abandoned pages 260 | }; 261 | 262 | 263 | 264 | // ------------------------------------------------------ 265 | // Debug 266 | // ------------------------------------------------------ 267 | 268 | #define MI_DEBUG_UNINIT (0xD0) 269 | #define MI_DEBUG_FREED (0xDF) 270 | 271 | 272 | #if (MI_DEBUG) 273 | // use our own assertion to print without memory allocation 274 | void _mi_assert_fail(const char* assertion, const char* fname, unsigned int line, const char* func ); 275 | #define mi_assert(expr) ((expr) ? (void)0 : _mi_assert_fail(#expr,__FILE__,__LINE__,__func__)) 276 | #else 277 | #define mi_assert(x) 278 | #endif 279 | 280 | #if (MI_DEBUG>1) 281 | #define mi_assert_internal mi_assert 282 | #else 283 | #define mi_assert_internal(x) 284 | #endif 285 | 286 | #if (MI_DEBUG>2) 287 | #define mi_assert_expensive mi_assert 288 | #else 289 | #define mi_assert_expensive(x) 290 | #endif 291 | 292 | // ------------------------------------------------------ 293 | // Statistics 294 | // ------------------------------------------------------ 295 | 296 | #ifndef MI_STAT 297 | #if (MI_DEBUG>0) 298 | #define MI_STAT 2 299 | #else 300 | #define MI_STAT 0 301 | #endif 302 | #endif 303 | 304 | typedef struct mi_stat_count_s { 305 | int64_t allocated; 306 | int64_t freed; 307 | int64_t peak; 308 | int64_t current; 309 | } mi_stat_count_t; 310 | 311 | typedef struct mi_stat_counter_s { 312 | int64_t total; 313 | int64_t count; 314 | } mi_stat_counter_t; 315 | 316 | typedef struct mi_stats_s { 317 | mi_stat_count_t segments; 318 | mi_stat_count_t pages; 319 | mi_stat_count_t reserved; 320 | mi_stat_count_t committed; 321 | mi_stat_count_t reset; 322 | mi_stat_count_t page_committed; 323 | mi_stat_count_t segments_abandoned; 324 | mi_stat_count_t pages_abandoned; 325 | mi_stat_count_t pages_extended; 326 | mi_stat_count_t mmap_calls; 327 | mi_stat_count_t mmap_right_align; 328 | mi_stat_count_t mmap_ensure_aligned; 329 | mi_stat_count_t commit_calls; 330 | mi_stat_count_t threads; 331 | mi_stat_count_t huge; 332 | mi_stat_count_t malloc; 333 | mi_stat_counter_t searches; 334 | #if MI_STAT>1 335 | mi_stat_count_t normal[MI_BIN_HUGE+1]; 336 | #endif 337 | } mi_stats_t; 338 | 339 | 340 | void _mi_stat_increase(mi_stat_count_t* stat, size_t amount); 341 | void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount); 342 | void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); 343 | 344 | #if (MI_STAT) 345 | #define mi_stat_increase(stat,amount) _mi_stat_increase( &(stat), amount) 346 | #define mi_stat_decrease(stat,amount) _mi_stat_decrease( &(stat), amount) 347 | #define mi_stat_counter_increase(stat,amount) _mi_stat_counter_increase( &(stat), amount) 348 | #else 349 | #define mi_stat_increase(stat,amount) (void)0 350 | #define mi_stat_decrease(stat,amount) (void)0 351 | #define mi_stat_counter_increase(stat,amount) (void)0 352 | #endif 353 | 354 | #define mi_heap_stat_increase(heap,stat,amount) mi_stat_increase( (heap)->tld->stats.stat, amount) 355 | #define mi_heap_stat_decrease(heap,stat,amount) mi_stat_decrease( (heap)->tld->stats.stat, amount) 356 | 357 | 358 | // ------------------------------------------------------ 359 | // Thread Local data 360 | // ------------------------------------------------------ 361 | 362 | // Queue of segments 363 | typedef struct mi_segment_queue_s { 364 | mi_segment_t* first; 365 | mi_segment_t* last; 366 | } mi_segment_queue_t; 367 | 368 | 369 | // Segments thread local data 370 | typedef struct mi_segments_tld_s { 371 | mi_segment_queue_t small_free; // queue of segments with free small pages 372 | mi_segment_queue_t medium_free; // queue of segments with free medium pages 373 | size_t count; // current number of segments; 374 | size_t peak_count; // peak number of segments 375 | size_t current_size; // current size of all segments 376 | size_t peak_size; // peak size of all segments 377 | size_t cache_count; // number of segments in the cache 378 | size_t cache_size; // total size of all segments in the cache 379 | mi_segment_t* cache; // (small) cache of segments 380 | mi_stats_t* stats; // points to tld stats 381 | } mi_segments_tld_t; 382 | 383 | // OS thread local data 384 | typedef struct mi_os_tld_s { 385 | uintptr_t mmap_next_probable; // probable next address start allocated by mmap (to guess which path to take on alignment) 386 | void* mmap_previous; // previous address returned by mmap 387 | uint8_t* pool; // pool of segments to reduce mmap calls on some platforms 388 | size_t pool_available; // bytes available in the pool 389 | mi_stats_t* stats; // points to tld stats 390 | } mi_os_tld_t; 391 | 392 | // Thread local data 393 | struct mi_tld_s { 394 | unsigned long long heartbeat; // monotonic heartbeat count 395 | mi_heap_t* heap_backing; // backing heap of this thread (cannot be deleted) 396 | mi_segments_tld_t segments; // segment tld 397 | mi_os_tld_t os; // os tld 398 | mi_stats_t stats; // statistics 399 | }; 400 | 401 | #endif 402 | -------------------------------------------------------------------------------- /mimalloc/mimalloc.h: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #pragma once 8 | #ifndef MIMALLOC_H 9 | #define MIMALLOC_H 10 | 11 | #define MI_MALLOC_VERSION 100 // major + 2 digits minor 12 | 13 | // ------------------------------------------------------ 14 | // Compiler specific attributes 15 | // ------------------------------------------------------ 16 | 17 | #ifdef __cplusplus 18 | #if (__GNUC__ <= 5) || (_MSC_VER <= 1900) 19 | #define mi_attr_noexcept throw() 20 | #else 21 | #define mi_attr_noexcept noexcept 22 | #endif 23 | #else 24 | #define mi_attr_noexcept 25 | #endif 26 | 27 | #ifdef _MSC_VER 28 | #if !defined(MI_SHARED_LIB) 29 | #define mi_decl_export 30 | #elif defined(MI_SHARED_LIB_EXPORT) 31 | #define mi_decl_export __declspec(dllexport) 32 | #else 33 | #define mi_decl_export __declspec(dllimport) 34 | #endif 35 | #if (_MSC_VER >= 1900) && !defined(__EDG__) 36 | #define mi_decl_allocator __declspec(allocator) __declspec(restrict) 37 | #else 38 | #define mi_decl_allocator __declspec(restrict) 39 | #endif 40 | #define mi_decl_thread __declspec(thread) 41 | #define mi_attr_malloc 42 | #define mi_attr_alloc_size(s) 43 | #define mi_attr_alloc_size2(s1,s2) 44 | #define mi_cdecl __cdecl 45 | #elif defined(__GNUC__) || defined(__clang__) 46 | #define mi_decl_thread __thread 47 | #define mi_decl_export __attribute__((visibility("default"))) 48 | #define mi_decl_allocator 49 | #define mi_attr_malloc __attribute__((malloc)) 50 | #if defined(__clang_major__) && (__clang_major__ < 4) 51 | #define mi_attr_alloc_size(s) 52 | #define mi_attr_alloc_size2(s1,s2) 53 | #else 54 | #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) 55 | #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) 56 | #define mi_cdecl // leads to warnings... __attribute__((cdecl)) 57 | #endif 58 | #else 59 | #define mi_decl_thread __thread 60 | #define mi_decl_export 61 | #define mi_decl_allocator 62 | #define mi_attr_malloc 63 | #define mi_attr_alloc_size(s) 64 | #define mi_attr_alloc_size2(s1,s2) 65 | #define mi_cdecl 66 | #endif 67 | 68 | // ------------------------------------------------------ 69 | // Includes 70 | // ------------------------------------------------------ 71 | 72 | #include // size_t, malloc etc. 73 | #include // bool 74 | #include // FILE 75 | 76 | #ifdef __cplusplus 77 | extern "C" { 78 | #endif 79 | 80 | // ------------------------------------------------------ 81 | // Standard malloc interface 82 | // ------------------------------------------------------ 83 | 84 | mi_decl_export mi_decl_allocator void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 85 | mi_decl_export mi_decl_allocator void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); 86 | mi_decl_export mi_decl_allocator void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 87 | mi_decl_export mi_decl_allocator void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 88 | 89 | mi_decl_export void mi_free(void* p) mi_attr_noexcept; 90 | mi_decl_export char* mi_strdup(const char* s) mi_attr_noexcept; 91 | mi_decl_export char* mi_strndup(const char* s, size_t n) mi_attr_noexcept; 92 | mi_decl_export char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept; 93 | 94 | // ------------------------------------------------------ 95 | // Extended functionality 96 | // ------------------------------------------------------ 97 | #define MI_SMALL_WSIZE_MAX (128) 98 | #define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) 99 | 100 | mi_decl_export mi_decl_allocator void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 101 | mi_decl_export mi_decl_allocator void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 102 | mi_decl_export mi_decl_allocator void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 103 | 104 | mi_decl_export mi_decl_allocator void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept; 105 | mi_decl_export mi_decl_allocator void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept; 106 | mi_decl_export mi_decl_allocator void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 107 | 108 | mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; 109 | mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; 110 | 111 | mi_decl_export void mi_collect(bool force) mi_attr_noexcept; 112 | mi_decl_export void mi_stats_print(FILE* out) mi_attr_noexcept; 113 | mi_decl_export void mi_stats_reset(void) mi_attr_noexcept; 114 | mi_decl_export int mi_version(void) mi_attr_noexcept; 115 | 116 | mi_decl_export void mi_process_init(void) mi_attr_noexcept; 117 | mi_decl_export void mi_thread_init(void) mi_attr_noexcept; 118 | mi_decl_export void mi_thread_done(void) mi_attr_noexcept; 119 | mi_decl_export void mi_thread_stats_print(FILE* out) mi_attr_noexcept; 120 | 121 | typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat); 122 | mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free) mi_attr_noexcept; 123 | 124 | // ------------------------------------------------------ 125 | // Aligned allocation 126 | // ------------------------------------------------------ 127 | 128 | mi_decl_export mi_decl_allocator void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 129 | mi_decl_export mi_decl_allocator void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 130 | mi_decl_export mi_decl_allocator void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 131 | mi_decl_export mi_decl_allocator void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 132 | mi_decl_export mi_decl_allocator void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); 133 | mi_decl_export mi_decl_allocator void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); 134 | mi_decl_export mi_decl_allocator void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 135 | mi_decl_export mi_decl_allocator void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 136 | 137 | 138 | // ------------------------------------------------------ 139 | // Heaps 140 | // ------------------------------------------------------ 141 | struct mi_heap_s; 142 | typedef struct mi_heap_s mi_heap_t; 143 | 144 | mi_decl_export mi_heap_t* mi_heap_new(void); 145 | mi_decl_export void mi_heap_delete(mi_heap_t* heap); 146 | mi_decl_export void mi_heap_destroy(mi_heap_t* heap); 147 | mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap); 148 | mi_decl_export mi_heap_t* mi_heap_get_default(void); 149 | mi_decl_export mi_heap_t* mi_heap_get_backing(void); 150 | mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept; 151 | 152 | mi_decl_export mi_decl_allocator void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 153 | mi_decl_export mi_decl_allocator void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 154 | mi_decl_export mi_decl_allocator void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); 155 | mi_decl_export mi_decl_allocator void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); 156 | mi_decl_export mi_decl_allocator void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 157 | 158 | mi_decl_export mi_decl_allocator void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3); 159 | mi_decl_export mi_decl_allocator void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept; 160 | mi_decl_export mi_decl_allocator void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3); 161 | 162 | mi_decl_export char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept; 163 | mi_decl_export char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept; 164 | mi_decl_export char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept; 165 | 166 | mi_decl_export mi_decl_allocator void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 167 | mi_decl_export mi_decl_allocator void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 168 | mi_decl_export mi_decl_allocator void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 169 | mi_decl_export mi_decl_allocator void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 170 | mi_decl_export mi_decl_allocator void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); 171 | mi_decl_export mi_decl_allocator void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); 172 | mi_decl_export mi_decl_allocator void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3); 173 | mi_decl_export mi_decl_allocator void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(3); 174 | 175 | 176 | // ------------------------------------------------------ 177 | // Analysis 178 | // ------------------------------------------------------ 179 | 180 | mi_decl_export bool mi_heap_contains_block(mi_heap_t* heap, const void* p); 181 | 182 | mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p); 183 | mi_decl_export bool mi_check_owned(const void* p); 184 | 185 | // An area of heap space contains blocks of a single size. 186 | typedef struct mi_heap_area_s { 187 | void* blocks; // start of the area containing heap blocks 188 | size_t reserved; // bytes reserved for this area (virtual) 189 | size_t committed; // current available bytes for this area 190 | size_t used; // bytes in use by allocated blocks 191 | size_t block_size; // size in bytes of each block 192 | } mi_heap_area_t; 193 | 194 | typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); 195 | 196 | mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_all_blocks, mi_block_visit_fun* visitor, void* arg); 197 | 198 | mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; 199 | 200 | // ------------------------------------------------------ 201 | // Convenience 202 | // ------------------------------------------------------ 203 | 204 | #define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp))) 205 | #define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp))) 206 | #define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp))) 207 | #define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp))) 208 | #define mi_reallocn_tp(p,tp,n) ((tp*)mi_reallocn(p,n,sizeof(tp))) 209 | 210 | #define mi_heap_malloc_tp(hp,tp) ((tp*)mi_heap_malloc(hp,sizeof(tp))) 211 | #define mi_heap_zalloc_tp(hp,tp) ((tp*)mi_heap_zalloc(hp,sizeof(tp))) 212 | #define mi_heap_calloc_tp(hp,tp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp))) 213 | #define mi_heap_mallocn_tp(hp,tp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp))) 214 | #define mi_heap_reallocn_tp(hp,tp,n) ((tp*)mi_heap_reallocn(hp,n,sizeof(tp))) 215 | 216 | 217 | // ------------------------------------------------------ 218 | // Options, all `false` by default 219 | // ------------------------------------------------------ 220 | 221 | typedef enum mi_option_e { 222 | // stable options 223 | mi_option_show_stats, 224 | mi_option_show_errors, 225 | mi_option_verbose, 226 | // the following options are experimental 227 | mi_option_secure, 228 | mi_option_eager_commit, 229 | mi_option_eager_region_commit, 230 | mi_option_large_os_pages, // implies eager commit 231 | mi_option_page_reset, 232 | mi_option_cache_reset, 233 | mi_option_reset_decommits, 234 | mi_option_reset_discards, 235 | _mi_option_last 236 | } mi_option_t; 237 | 238 | 239 | mi_decl_export bool mi_option_is_enabled(mi_option_t option); 240 | mi_decl_export void mi_option_enable(mi_option_t option, bool enable); 241 | mi_decl_export void mi_option_enable_default(mi_option_t option, bool enable); 242 | 243 | mi_decl_export long mi_option_get(mi_option_t option); 244 | mi_decl_export void mi_option_set(mi_option_t option, long value); 245 | mi_decl_export void mi_option_set_default(mi_option_t option, long value); 246 | 247 | 248 | // ---------------------------------------------------------------------------------- 249 | // mi prefixed implementations of various posix, unix, and C++ allocation functions. 250 | // ----------------------------------------------------------------------------------- 251 | 252 | mi_decl_export void* mi_recalloc(void* p, size_t count, size_t size) mi_attr_noexcept; 253 | mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; 254 | mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; 255 | mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; 256 | 257 | mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; 258 | mi_decl_export int mi__posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; 259 | mi_decl_export mi_decl_allocator void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 260 | mi_decl_export mi_decl_allocator void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 261 | 262 | mi_decl_export mi_decl_allocator void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 263 | mi_decl_export mi_decl_allocator void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 264 | mi_decl_export mi_decl_allocator void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2,3); 265 | 266 | mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; 267 | mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; 268 | mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; 269 | 270 | mi_decl_export void* mi_new(size_t n) mi_attr_malloc mi_attr_alloc_size(1); 271 | mi_decl_export void* mi_new_aligned(size_t n, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1); 272 | mi_decl_export void* mi_new_nothrow(size_t n) mi_attr_malloc mi_attr_alloc_size(1); 273 | mi_decl_export void* mi_new_aligned_nothrow(size_t n, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1); 274 | 275 | #ifdef __cplusplus 276 | } 277 | #endif 278 | 279 | 280 | #endif 281 | -------------------------------------------------------------------------------- /mimalloc/options.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #include "mimalloc.h" 8 | #include "mimalloc-internal.h" 9 | 10 | #include 11 | #include // strcmp 12 | #include // toupper 13 | #include 14 | 15 | int mi_version(void) mi_attr_noexcept { 16 | return MI_MALLOC_VERSION; 17 | } 18 | 19 | // -------------------------------------------------------- 20 | // Options 21 | // These can be accessed by multiple threads and may be 22 | // concurrently initialized, but an initializing data race 23 | // is ok since they resolve to the same value. 24 | // -------------------------------------------------------- 25 | typedef enum mi_init_e { 26 | UNINIT, // not yet initialized 27 | DEFAULTED, // not found in the environment, use default value 28 | INITIALIZED // found in environment or set explicitly 29 | } mi_init_t; 30 | 31 | typedef struct mi_option_desc_s { 32 | long value; // the value 33 | mi_init_t init; // is it initialized yet? (from the environment) 34 | const char* name; // option name without `mimalloc_` prefix 35 | } mi_option_desc_t; 36 | 37 | static mi_option_desc_t options[_mi_option_last] = 38 | { 39 | // stable options 40 | { 0, UNINIT, "show_stats" }, 41 | { MI_DEBUG, UNINIT, "show_errors" }, 42 | { 0, UNINIT, "verbose" }, 43 | 44 | #if MI_SECURE 45 | { MI_SECURE, INITIALIZED, "secure" }, // in a secure build the environment setting is ignored 46 | #else 47 | { 0, UNINIT, "secure" }, 48 | #endif 49 | 50 | // the following options are experimental and not all combinations make sense. 51 | { 1, UNINIT, "eager_commit" }, // note: if eager_region_commit is on, this should be on too. 52 | #ifdef _WIN32 // and BSD? 53 | { 0, UNINIT, "eager_region_commit" }, // don't commit too eagerly on windows (just for looks...) 54 | #else 55 | { 1, UNINIT, "eager_region_commit" }, 56 | #endif 57 | { 0, UNINIT, "large_os_pages" }, // use large OS pages, use only with eager commit to prevent fragmentation of VMA's 58 | { 0, UNINIT, "page_reset" }, 59 | { 0, UNINIT, "cache_reset" }, 60 | { 0, UNINIT, "reset_decommits" }, // note: cannot enable this if secure is on 61 | { 0, UNINIT, "reset_discards" } // note: cannot enable this if secure is on 62 | }; 63 | 64 | static void mi_option_init(mi_option_desc_t* desc); 65 | 66 | long mi_option_get(mi_option_t option) { 67 | mi_assert(option >= 0 && option < _mi_option_last); 68 | mi_option_desc_t* desc = &options[option]; 69 | if (mi_unlikely(desc->init == UNINIT)) { 70 | mi_option_init(desc); 71 | if (option != mi_option_verbose) { 72 | _mi_verbose_message("option '%s': %ld\n", desc->name, desc->value); 73 | } 74 | } 75 | return desc->value; 76 | } 77 | 78 | void mi_option_set(mi_option_t option, long value) { 79 | mi_assert(option >= 0 && option < _mi_option_last); 80 | mi_option_desc_t* desc = &options[option]; 81 | desc->value = value; 82 | desc->init = INITIALIZED; 83 | } 84 | 85 | void mi_option_set_default(mi_option_t option, long value) { 86 | mi_assert(option >= 0 && option < _mi_option_last); 87 | mi_option_desc_t* desc = &options[option]; 88 | if (desc->init != INITIALIZED) { 89 | desc->value = value; 90 | } 91 | } 92 | 93 | bool mi_option_is_enabled(mi_option_t option) { 94 | return (mi_option_get(option) != 0); 95 | } 96 | 97 | void mi_option_enable(mi_option_t option, bool enable) { 98 | mi_option_set(option, (enable ? 1 : 0)); 99 | } 100 | 101 | void mi_option_enable_default(mi_option_t option, bool enable) { 102 | mi_option_set_default(option, (enable ? 1 : 0)); 103 | } 104 | 105 | // -------------------------------------------------------- 106 | // Messages 107 | // -------------------------------------------------------- 108 | 109 | // Define our own limited `fprintf` that avoids memory allocation. 110 | // We do this using `snprintf` with a limited buffer. 111 | static void mi_vfprintf( FILE* out, const char* prefix, const char* fmt, va_list args ) { 112 | char buf[256]; 113 | if (fmt==NULL) return; 114 | if (out==NULL) out = stdout; 115 | vsnprintf(buf,sizeof(buf)-1,fmt,args); 116 | if (prefix != NULL) fputs(prefix,out); 117 | fputs(buf,out); 118 | } 119 | 120 | void _mi_fprintf( FILE* out, const char* fmt, ... ) { 121 | va_list args; 122 | va_start(args,fmt); 123 | mi_vfprintf(out,NULL,fmt,args); 124 | va_end(args); 125 | } 126 | 127 | void _mi_trace_message(const char* fmt, ...) { 128 | if (mi_option_get(mi_option_verbose) <= 1) return; // only with verbose level 2 or higher 129 | va_list args; 130 | va_start(args, fmt); 131 | mi_vfprintf(stderr, "mimalloc: ", fmt, args); 132 | va_end(args); 133 | } 134 | 135 | void _mi_verbose_message(const char* fmt, ...) { 136 | if (!mi_option_is_enabled(mi_option_verbose)) return; 137 | va_list args; 138 | va_start(args,fmt); 139 | mi_vfprintf(stderr, "mimalloc: ", fmt, args); 140 | va_end(args); 141 | } 142 | 143 | void _mi_error_message(const char* fmt, ...) { 144 | if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return; 145 | va_list args; 146 | va_start(args,fmt); 147 | mi_vfprintf(stderr, "mimalloc: error: ", fmt, args); 148 | va_end(args); 149 | mi_assert(false); 150 | } 151 | 152 | void _mi_warning_message(const char* fmt, ...) { 153 | if (!mi_option_is_enabled(mi_option_show_errors) && !mi_option_is_enabled(mi_option_verbose)) return; 154 | va_list args; 155 | va_start(args,fmt); 156 | mi_vfprintf(stderr, "mimalloc: warning: ", fmt, args); 157 | va_end(args); 158 | } 159 | 160 | 161 | #if MI_DEBUG 162 | void _mi_assert_fail(const char* assertion, const char* fname, unsigned line, const char* func ) { 163 | _mi_fprintf(stderr,"mimalloc: assertion failed: at \"%s\":%u, %s\n assertion: \"%s\"\n", fname, line, (func==NULL?"":func), assertion); 164 | abort(); 165 | } 166 | #endif 167 | 168 | // -------------------------------------------------------- 169 | // Initialize options by checking the environment 170 | // -------------------------------------------------------- 171 | 172 | static void mi_strlcpy(char* dest, const char* src, size_t dest_size) { 173 | dest[0] = 0; 174 | #pragma warning(suppress:4996) 175 | strncpy(dest, src, dest_size - 1); 176 | dest[dest_size - 1] = 0; 177 | } 178 | 179 | static void mi_strlcat(char* dest, const char* src, size_t dest_size) { 180 | #pragma warning(suppress:4996) 181 | strncat(dest, src, dest_size - 1); 182 | dest[dest_size - 1] = 0; 183 | } 184 | 185 | static void mi_option_init(mi_option_desc_t* desc) { 186 | // Read option value from the environment 187 | char buf[32]; 188 | mi_strlcpy(buf, "mimalloc_", sizeof(buf)); 189 | mi_strlcat(buf, desc->name, sizeof(buf)); 190 | #pragma warning(suppress:4996) 191 | char* s = getenv(buf); 192 | if (s == NULL) { 193 | size_t buf_size = strlen(buf); 194 | for (size_t i = 0; i < buf_size; i++) { 195 | buf[i] = toupper(buf[i]); 196 | } 197 | #pragma warning(suppress:4996) 198 | s = getenv(buf); 199 | } 200 | if (s != NULL) { 201 | mi_strlcpy(buf, s, sizeof(buf)); 202 | size_t buf_size = strlen(buf); // TODO: use strnlen? 203 | for (size_t i = 0; i < buf_size; i++) { 204 | buf[i] = toupper(buf[i]); 205 | } 206 | if (buf[0]==0 || strstr("1;TRUE;YES;ON", buf) != NULL) { 207 | desc->value = 1; 208 | desc->init = INITIALIZED; 209 | } 210 | else if (strstr("0;FALSE;NO;OFF", buf) != NULL) { 211 | desc->value = 0; 212 | desc->init = INITIALIZED; 213 | } 214 | else { 215 | char* end = buf; 216 | long value = strtol(buf, &end, 10); 217 | if (*end == 0) { 218 | desc->value = value; 219 | desc->init = INITIALIZED; 220 | } 221 | else { 222 | _mi_warning_message("environment option mimalloc_%s has an invalid value: %s\n", desc->name, buf); 223 | desc->init = DEFAULTED; 224 | } 225 | } 226 | } 227 | else { 228 | desc->init = DEFAULTED; 229 | } 230 | mi_assert_internal(desc->init != UNINIT); 231 | } 232 | -------------------------------------------------------------------------------- /mimalloc/page-queue.c: -------------------------------------------------------------------------------- 1 | /*---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | 8 | /* ----------------------------------------------------------- 9 | Definition of page queues for each block size 10 | ----------------------------------------------------------- */ 11 | 12 | #ifndef MI_IN_PAGE_C 13 | #error "this file should be included from 'page.c'" 14 | #endif 15 | 16 | /* ----------------------------------------------------------- 17 | Minimal alignment in machine words (i.e. `sizeof(void*)`) 18 | ----------------------------------------------------------- */ 19 | 20 | #if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE) 21 | #error "define alignment for more than 4x word size for this platform" 22 | #elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE) 23 | #define MI_ALIGN4W // 4 machine words minimal alignment 24 | #elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE) 25 | #define MI_ALIGN2W // 2 machine words minimal alignment 26 | #else 27 | // ok, default alignment is 1 word 28 | #endif 29 | 30 | 31 | /* ----------------------------------------------------------- 32 | Queue query 33 | ----------------------------------------------------------- */ 34 | 35 | 36 | static inline bool mi_page_queue_is_huge(const mi_page_queue_t* pq) { 37 | return (pq->block_size == (MI_LARGE_SIZE_MAX+sizeof(uintptr_t))); 38 | } 39 | 40 | static inline bool mi_page_queue_is_full(const mi_page_queue_t* pq) { 41 | return (pq->block_size == (MI_LARGE_SIZE_MAX+(2*sizeof(uintptr_t)))); 42 | } 43 | 44 | static inline bool mi_page_queue_is_special(const mi_page_queue_t* pq) { 45 | return (pq->block_size > MI_LARGE_SIZE_MAX); 46 | } 47 | 48 | /* ----------------------------------------------------------- 49 | Bins 50 | ----------------------------------------------------------- */ 51 | 52 | // Bit scan reverse: return the index of the highest bit. 53 | static inline uint8_t mi_bsr32(uint32_t x); 54 | 55 | #if defined(_MSC_VER) 56 | #include 57 | static inline uint8_t mi_bsr32(uint32_t x) { 58 | uint32_t idx; 59 | _BitScanReverse((DWORD*)&idx, x); 60 | return idx; 61 | } 62 | #elif defined(__GNUC__) || defined(__clang__) 63 | static inline uint8_t mi_bsr32(uint32_t x) { 64 | return (31 - __builtin_clz(x)); 65 | } 66 | #else 67 | static inline uint8_t mi_bsr32(uint32_t x) { 68 | // de Bruijn multiplication, see 69 | static const uint8_t debruijn[32] = { 70 | 31, 0, 22, 1, 28, 23, 18, 2, 29, 26, 24, 10, 19, 7, 3, 12, 71 | 30, 21, 27, 17, 25, 9, 6, 11, 20, 16, 8, 5, 15, 4, 14, 13, 72 | }; 73 | x |= x >> 1; 74 | x |= x >> 2; 75 | x |= x >> 4; 76 | x |= x >> 8; 77 | x |= x >> 16; 78 | x++; 79 | return debruijn[(x*0x076be629) >> 27]; 80 | } 81 | #endif 82 | 83 | // Bit scan reverse: return the index of the highest bit. 84 | uint8_t _mi_bsr(uintptr_t x) { 85 | if (x == 0) return 0; 86 | #if MI_INTPTR_SIZE==8 87 | uint32_t hi = (x >> 32); 88 | return (hi == 0 ? mi_bsr32((uint32_t)x) : 32 + mi_bsr32(hi)); 89 | #elif MI_INTPTR_SIZE==4 90 | return mi_bsr32(x); 91 | #else 92 | # error "define bsr for non-32 or 64-bit platforms" 93 | #endif 94 | } 95 | 96 | // Return the bin for a given field size. 97 | // Returns MI_BIN_HUGE if the size is too large. 98 | // We use `wsize` for the size in "machine word sizes", 99 | // i.e. byte size == `wsize*sizeof(void*)`. 100 | inline uint8_t _mi_bin(size_t size) { 101 | size_t wsize = _mi_wsize_from_size(size); 102 | uint8_t bin; 103 | if (wsize <= 1) { 104 | bin = 1; 105 | } 106 | #if defined(MI_ALIGN4W) 107 | else if (wsize <= 4) { 108 | bin = (uint8_t)((wsize+1)&~1); // round to double word sizes 109 | } 110 | #elif defined(MI_ALIGN2W) 111 | else if (wsize <= 8) { 112 | bin = (uint8_t)((wsize+1)&~1); // round to double word sizes 113 | } 114 | #else 115 | else if (wsize <= 8) { 116 | bin = (uint8_t)wsize; 117 | } 118 | #endif 119 | else if (wsize > MI_LARGE_WSIZE_MAX) { 120 | bin = MI_BIN_HUGE; 121 | } 122 | else { 123 | #if defined(MI_ALIGN4W) 124 | if (wsize <= 16) { wsize = (wsize+3)&~3; } // round to 4x word sizes 125 | #endif 126 | wsize--; 127 | // find the highest bit 128 | uint8_t b = mi_bsr32((uint32_t)wsize); 129 | // and use the top 3 bits to determine the bin (~16% worst internal fragmentation). 130 | // - adjust with 3 because we use do not round the first 8 sizes 131 | // which each get an exact bin 132 | bin = ((b << 2) + (uint8_t)((wsize >> (b - 2)) & 0x03)) - 3; 133 | } 134 | mi_assert_internal(bin > 0 && bin <= MI_BIN_HUGE); 135 | return bin; 136 | } 137 | 138 | 139 | 140 | /* ----------------------------------------------------------- 141 | Queue of pages with free blocks 142 | ----------------------------------------------------------- */ 143 | 144 | size_t _mi_bin_size(uint8_t bin) { 145 | return _mi_heap_empty.pages[bin].block_size; 146 | } 147 | 148 | // Good size for allocation 149 | size_t mi_good_size(size_t size) mi_attr_noexcept { 150 | if (size <= MI_LARGE_SIZE_MAX) { 151 | return _mi_bin_size(_mi_bin(size)); 152 | } 153 | else { 154 | return _mi_align_up(size,_mi_os_page_size()); 155 | } 156 | } 157 | 158 | #if (MI_DEBUG>1) 159 | static bool mi_page_queue_contains(mi_page_queue_t* queue, const mi_page_t* page) { 160 | mi_assert_internal(page != NULL); 161 | mi_page_t* list = queue->first; 162 | while (list != NULL) { 163 | mi_assert_internal(list->next == NULL || list->next->prev == list); 164 | mi_assert_internal(list->prev == NULL || list->prev->next == list); 165 | if (list == page) break; 166 | list = list->next; 167 | } 168 | return (list == page); 169 | } 170 | 171 | #endif 172 | 173 | #if (MI_DEBUG>1) 174 | static bool mi_heap_contains_queue(const mi_heap_t* heap, const mi_page_queue_t* pq) { 175 | return (pq >= &heap->pages[0] && pq <= &heap->pages[MI_BIN_FULL]); 176 | } 177 | #endif 178 | 179 | static mi_page_queue_t* mi_page_queue_of(const mi_page_t* page) { 180 | uint8_t bin = (page->flags.in_full ? MI_BIN_FULL : _mi_bin(page->block_size)); 181 | mi_heap_t* heap = page->heap; 182 | mi_assert_internal(heap != NULL && bin <= MI_BIN_FULL); 183 | mi_page_queue_t* pq = &heap->pages[bin]; 184 | mi_assert_internal(bin >= MI_BIN_HUGE || page->block_size == pq->block_size); 185 | mi_assert_expensive(mi_page_queue_contains(pq, page)); 186 | return pq; 187 | } 188 | 189 | static mi_page_queue_t* mi_heap_page_queue_of(mi_heap_t* heap, const mi_page_t* page) { 190 | uint8_t bin = (page->flags.in_full ? MI_BIN_FULL : _mi_bin(page->block_size)); 191 | mi_assert_internal(bin <= MI_BIN_FULL); 192 | mi_page_queue_t* pq = &heap->pages[bin]; 193 | mi_assert_internal(page->flags.in_full || page->block_size == pq->block_size); 194 | return pq; 195 | } 196 | 197 | // The current small page array is for efficiency and for each 198 | // small size (up to 256) it points directly to the page for that 199 | // size without having to compute the bin. This means when the 200 | // current free page queue is updated for a small bin, we need to update a 201 | // range of entries in `_mi_page_small_free`. 202 | static inline void mi_heap_queue_first_update(mi_heap_t* heap, const mi_page_queue_t* pq) { 203 | mi_assert_internal(mi_heap_contains_queue(heap,pq)); 204 | size_t size = pq->block_size; 205 | if (size > MI_SMALL_SIZE_MAX) return; 206 | 207 | mi_page_t* page = pq->first; 208 | if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty; 209 | 210 | // find index in the right direct page array 211 | size_t start; 212 | size_t idx = _mi_wsize_from_size(size); 213 | mi_page_t** pages_free = heap->pages_free_direct; 214 | 215 | if (pages_free[idx] == page) return; // already set 216 | 217 | // find start slot 218 | if (idx<=1) { 219 | start = 0; 220 | } 221 | else { 222 | // find previous size; due to minimal alignment upto 3 previous bins may need to be skipped 223 | uint8_t bin = _mi_bin(size); 224 | const mi_page_queue_t* prev = pq - 1; 225 | while( bin == _mi_bin(prev->block_size) && prev > &heap->pages[0]) { 226 | prev--; 227 | } 228 | start = 1 + _mi_wsize_from_size(prev->block_size); 229 | if (start > idx) start = idx; 230 | } 231 | 232 | // set size range to the right page 233 | mi_assert(start <= idx); 234 | for (size_t sz = start; sz <= idx; sz++) { 235 | pages_free[sz] = page; 236 | } 237 | } 238 | 239 | /* 240 | static bool mi_page_queue_is_empty(mi_page_queue_t* queue) { 241 | return (queue->first == NULL); 242 | } 243 | */ 244 | 245 | static void mi_page_queue_remove(mi_page_queue_t* queue, mi_page_t* page) { 246 | mi_assert_internal(page != NULL); 247 | mi_assert_expensive(mi_page_queue_contains(queue, page)); 248 | mi_assert_internal(page->block_size == queue->block_size || (page->block_size > MI_LARGE_SIZE_MAX && mi_page_queue_is_huge(queue)) || (page->flags.in_full && mi_page_queue_is_full(queue))); 249 | if (page->prev != NULL) page->prev->next = page->next; 250 | if (page->next != NULL) page->next->prev = page->prev; 251 | if (page == queue->last) queue->last = page->prev; 252 | if (page == queue->first) { 253 | queue->first = page->next; 254 | // update first 255 | mi_heap_t* heap = page->heap; 256 | mi_assert_internal(mi_heap_contains_queue(heap, queue)); 257 | mi_heap_queue_first_update(heap,queue); 258 | } 259 | page->heap->page_count--; 260 | page->next = NULL; 261 | page->prev = NULL; 262 | page->heap = NULL; 263 | page->flags.in_full = false; 264 | } 265 | 266 | 267 | static void mi_page_queue_push(mi_heap_t* heap, mi_page_queue_t* queue, mi_page_t* page) { 268 | mi_assert_internal(page->heap == NULL); 269 | mi_assert_internal(!mi_page_queue_contains(queue, page)); 270 | mi_assert_internal(page->block_size == queue->block_size || 271 | (page->block_size > MI_LARGE_SIZE_MAX && mi_page_queue_is_huge(queue)) || 272 | (page->flags.in_full && mi_page_queue_is_full(queue))); 273 | 274 | page->flags.in_full = mi_page_queue_is_full(queue); 275 | page->heap = heap; 276 | page->next = queue->first; 277 | page->prev = NULL; 278 | if (queue->first != NULL) { 279 | mi_assert_internal(queue->first->prev == NULL); 280 | queue->first->prev = page; 281 | queue->first = page; 282 | } 283 | else { 284 | queue->first = queue->last = page; 285 | } 286 | 287 | // update direct 288 | mi_heap_queue_first_update(heap, queue); 289 | heap->page_count++; 290 | } 291 | 292 | 293 | static void mi_page_queue_enqueue_from(mi_page_queue_t* to, mi_page_queue_t* from, mi_page_t* page) { 294 | mi_assert_internal(page != NULL); 295 | mi_assert_expensive(mi_page_queue_contains(from, page)); 296 | mi_assert_expensive(!mi_page_queue_contains(to, page)); 297 | mi_assert_internal((page->block_size == to->block_size && page->block_size == from->block_size) || 298 | (page->block_size == to->block_size && mi_page_queue_is_full(from)) || 299 | (page->block_size == from->block_size && mi_page_queue_is_full(to)) || 300 | (page->block_size > MI_LARGE_SIZE_MAX && mi_page_queue_is_huge(to)) || 301 | (page->block_size > MI_LARGE_SIZE_MAX && mi_page_queue_is_full(to))); 302 | 303 | if (page->prev != NULL) page->prev->next = page->next; 304 | if (page->next != NULL) page->next->prev = page->prev; 305 | if (page == from->last) from->last = page->prev; 306 | if (page == from->first) { 307 | from->first = page->next; 308 | // update first 309 | mi_heap_t* heap = page->heap; 310 | mi_assert_internal(mi_heap_contains_queue(heap, from)); 311 | mi_heap_queue_first_update(heap, from); 312 | } 313 | 314 | page->prev = to->last; 315 | page->next = NULL; 316 | if (to->last != NULL) { 317 | mi_assert_internal(page->heap == to->last->heap); 318 | to->last->next = page; 319 | to->last = page; 320 | } 321 | else { 322 | to->first = page; 323 | to->last = page; 324 | mi_heap_queue_first_update(page->heap, to); 325 | } 326 | 327 | page->flags.in_full = mi_page_queue_is_full(to); 328 | } 329 | 330 | size_t _mi_page_queue_append(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_queue_t* append) { 331 | mi_assert_internal(mi_heap_contains_queue(heap,pq)); 332 | mi_assert_internal(pq->block_size == append->block_size); 333 | 334 | if (append->first==NULL) return 0; 335 | 336 | // set append pages to new heap and count 337 | size_t count = 0; 338 | for (mi_page_t* page = append->first; page != NULL; page = page->next) { 339 | page->heap = heap; 340 | count++; 341 | } 342 | 343 | if (pq->last==NULL) { 344 | // take over afresh 345 | mi_assert_internal(pq->first==NULL); 346 | pq->first = append->first; 347 | pq->last = append->last; 348 | mi_heap_queue_first_update(heap, pq); 349 | } 350 | else { 351 | // append to end 352 | mi_assert_internal(pq->last!=NULL); 353 | mi_assert_internal(append->first!=NULL); 354 | pq->last->next = append->first; 355 | append->first->prev = pq->last; 356 | pq->last = append->last; 357 | } 358 | return count; 359 | } 360 | -------------------------------------------------------------------------------- /mimalloc/static.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #define _DEFAULT_SOURCE 8 | 9 | #include "mimalloc.h" 10 | #include "mimalloc-internal.h" 11 | 12 | // For a static override we create a single object file 13 | // containing the whole library. If it is linked first 14 | // it will override all the standard library allocation 15 | // functions (on Unix's). 16 | #include "stats.c" 17 | #include "os.c" 18 | #include "memory.c" 19 | #include "segment.c" 20 | #include "page.c" 21 | #include "heap.c" 22 | #include "alloc.c" 23 | #include "alloc-aligned.c" 24 | #include "alloc-posix.c" 25 | #include "init.c" 26 | #include "options.c" 27 | -------------------------------------------------------------------------------- /mimalloc/stats.c: -------------------------------------------------------------------------------- 1 | /* ---------------------------------------------------------------------------- 2 | Copyright (c) 2018, Microsoft Research, Daan Leijen 3 | This is free software; you can redistribute it and/or modify it under the 4 | terms of the MIT license. A copy of the license can be found in the file 5 | "LICENSE" at the root of this distribution. 6 | -----------------------------------------------------------------------------*/ 7 | #include "mimalloc.h" 8 | #include "mimalloc-internal.h" 9 | #include "mimalloc-atomic.h" 10 | 11 | #include // memset 12 | 13 | 14 | /* ----------------------------------------------------------- 15 | Merge thread statistics with the main one. 16 | ----------------------------------------------------------- */ 17 | 18 | static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src); 19 | 20 | void _mi_stats_done(mi_stats_t* stats) { 21 | if (stats == &_mi_stats_main) return; 22 | mi_stats_add(&_mi_stats_main, stats); 23 | memset(stats,0,sizeof(*stats)); 24 | } 25 | 26 | 27 | /* ----------------------------------------------------------- 28 | Statistics operations 29 | ----------------------------------------------------------- */ 30 | 31 | static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { 32 | if (amount == 0) return; 33 | bool in_main = ((uint8_t*)stat >= (uint8_t*)&_mi_stats_main 34 | && (uint8_t*)stat < ((uint8_t*)&_mi_stats_main + sizeof(mi_stats_t))); 35 | if (in_main) 36 | { 37 | // add atomically (for abandoned pages) 38 | int64_t current = mi_atomic_add(&stat->current,amount); 39 | if (current > stat->peak) stat->peak = stat->current; // racing.. it's ok 40 | if (amount > 0) { 41 | mi_atomic_add(&stat->allocated,amount); 42 | } 43 | else { 44 | mi_atomic_add(&stat->freed, -amount); 45 | } 46 | } 47 | else { 48 | // add thread local 49 | stat->current += amount; 50 | if (stat->current > stat->peak) stat->peak = stat->current; 51 | if (amount > 0) { 52 | stat->allocated += amount; 53 | } 54 | else { 55 | stat->freed += -amount; 56 | } 57 | } 58 | } 59 | 60 | void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { 61 | mi_atomic_add( &stat->count, 1 ); 62 | mi_atomic_add( &stat->total, (int64_t)amount ); 63 | } 64 | 65 | 66 | void _mi_stat_increase(mi_stat_count_t* stat, size_t amount) { 67 | mi_stat_update(stat, (int64_t)amount); 68 | } 69 | 70 | void _mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { 71 | mi_stat_update(stat, -((int64_t)amount)); 72 | } 73 | 74 | // must be thread safe as it is called from stats_merge 75 | static void mi_stat_add(mi_stat_count_t* stat, const mi_stat_count_t* src, int64_t unit) { 76 | if (stat==src) return; 77 | mi_atomic_add( &stat->allocated, src->allocated * unit); 78 | mi_atomic_add( &stat->current, src->current * unit); 79 | mi_atomic_add( &stat->freed, src->freed * unit); 80 | mi_atomic_add( &stat->peak, src->peak * unit); 81 | } 82 | 83 | static void mi_stat_counter_add(mi_stat_counter_t* stat, const mi_stat_counter_t* src, int64_t unit) { 84 | if (stat==src) return; 85 | mi_atomic_add( &stat->total, src->total * unit); 86 | mi_atomic_add( &stat->count, src->count * unit); 87 | } 88 | 89 | // must be thread safe as it is called from stats_merge 90 | static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { 91 | if (stats==src) return; 92 | mi_stat_add(&stats->segments, &src->segments,1); 93 | mi_stat_add(&stats->pages, &src->pages,1); 94 | mi_stat_add(&stats->reserved, &src->reserved, 1); 95 | mi_stat_add(&stats->committed, &src->committed, 1); 96 | mi_stat_add(&stats->reset, &src->reset, 1); 97 | mi_stat_add(&stats->page_committed, &src->page_committed, 1); 98 | 99 | mi_stat_add(&stats->pages_abandoned, &src->pages_abandoned, 1); 100 | mi_stat_add(&stats->segments_abandoned, &src->segments_abandoned, 1); 101 | mi_stat_add(&stats->mmap_calls, &src->mmap_calls, 1); 102 | mi_stat_add(&stats->mmap_ensure_aligned, &src->mmap_ensure_aligned, 1); 103 | mi_stat_add(&stats->mmap_right_align, &src->mmap_right_align, 1); 104 | mi_stat_add(&stats->commit_calls, &src->commit_calls, 1); 105 | mi_stat_add(&stats->threads, &src->threads, 1); 106 | mi_stat_add(&stats->pages_extended, &src->pages_extended, 1); 107 | 108 | mi_stat_add(&stats->malloc, &src->malloc, 1); 109 | mi_stat_add(&stats->huge, &src->huge, 1); 110 | mi_stat_counter_add(&stats->searches, &src->searches, 1); 111 | #if MI_STAT>1 112 | for (size_t i = 0; i <= MI_BIN_HUGE; i++) { 113 | if (src->normal[i].allocated > 0 || src->normal[i].freed > 0) { 114 | mi_stat_add(&stats->normal[i], &src->normal[i], 1); 115 | } 116 | } 117 | #endif 118 | } 119 | 120 | /* ----------------------------------------------------------- 121 | Display statistics 122 | ----------------------------------------------------------- */ 123 | 124 | static void mi_printf_amount(int64_t n, int64_t unit, FILE* out, const char* fmt) { 125 | char buf[32]; 126 | int len = 32; 127 | const char* suffix = (unit <= 0 ? " " : "b"); 128 | double base = (unit == 0 ? 1000.0 : 1024.0); 129 | if (unit>0) n *= unit; 130 | 131 | double pos = (double)(n < 0 ? -n : n); 132 | if (pos < base) 133 | snprintf(buf,len, "%d %s ", (int)n, suffix); 134 | else if (pos < base*base) 135 | snprintf(buf, len, "%.1f k%s", (double)n / base, suffix); 136 | else if (pos < base*base*base) 137 | snprintf(buf, len, "%.1f m%s", (double)n / (base*base), suffix); 138 | else 139 | snprintf(buf, len, "%.1f g%s", (double)n / (base*base*base), suffix); 140 | 141 | _mi_fprintf(out, (fmt==NULL ? "%11s" : fmt), buf); 142 | } 143 | 144 | 145 | static void mi_print_amount(int64_t n, int64_t unit, FILE* out) { 146 | mi_printf_amount(n,unit,out,NULL); 147 | } 148 | 149 | static void mi_print_count(int64_t n, int64_t unit, FILE* out) { 150 | if (unit==1) _mi_fprintf(out,"%11s"," "); 151 | else mi_print_amount(n,0,out); 152 | } 153 | 154 | static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, FILE* out ) { 155 | _mi_fprintf(out,"%10s:", msg); 156 | mi_print_amount(stat->peak, unit, out); 157 | if (unit!=0) { 158 | mi_print_amount(stat->allocated, unit, out); 159 | mi_print_amount(stat->freed, unit, out); 160 | } 161 | if (unit>0) { 162 | mi_print_amount(unit, (unit==0 ? 0 : 1), out); 163 | mi_print_count(stat->allocated, unit, out); 164 | if (stat->allocated > stat->freed) 165 | _mi_fprintf(out, " not all freed!\n"); 166 | else 167 | _mi_fprintf(out, " ok\n"); 168 | } 169 | else { 170 | _mi_fprintf(out, "\n"); 171 | } 172 | } 173 | 174 | static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, FILE* out ) { 175 | double avg = (stat->count == 0 ? 0.0 : (double)stat->total / (double)stat->count); 176 | _mi_fprintf(out,"%10s: %7.1f avg\n", msg, avg); 177 | } 178 | 179 | 180 | 181 | static void mi_print_header( FILE* out ) { 182 | _mi_fprintf(out,"%10s: %10s %10s %10s %10s %10s\n", "heap stats", "peak ", "total ", "freed ", "unit ", "count "); 183 | } 184 | 185 | #if MI_STAT>1 186 | static void mi_stats_print_bins(mi_stat_count_t* all, const mi_stat_count_t* bins, size_t max, const char* fmt, FILE* out) { 187 | bool found = false; 188 | char buf[64]; 189 | for (size_t i = 0; i <= max; i++) { 190 | if (bins[i].allocated > 0) { 191 | found = true; 192 | int64_t unit = _mi_bin_size((uint8_t)i); 193 | snprintf(buf, 64, "%s %3zu", fmt, i); 194 | mi_stat_add(all, &bins[i], unit); 195 | mi_stat_print(&bins[i], buf, unit, out); 196 | } 197 | } 198 | //snprintf(buf, 64, "%s all", fmt); 199 | //mi_stat_print(all, buf, 1); 200 | if (found) { 201 | _mi_fprintf(out, "\n"); 202 | mi_print_header(out); 203 | } 204 | } 205 | #endif 206 | 207 | 208 | static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit); 209 | 210 | static void _mi_stats_print(mi_stats_t* stats, double secs, FILE* out) mi_attr_noexcept { 211 | if (out == NULL) out = stderr; 212 | mi_print_header(out); 213 | #if MI_STAT>1 214 | mi_stat_count_t normal = { 0,0,0,0 }; 215 | mi_stats_print_bins(&normal, stats->normal, MI_BIN_HUGE, "normal",out); 216 | mi_stat_print(&normal, "normal", 1, out); 217 | mi_stat_print(&stats->huge, "huge", 1, out); 218 | mi_stat_count_t total = { 0,0,0,0 }; 219 | mi_stat_add(&total, &normal, 1); 220 | mi_stat_add(&total, &stats->huge, 1); 221 | mi_stat_print(&total, "total", 1, out); 222 | _mi_fprintf(out, "malloc requested: "); 223 | mi_print_amount(stats->malloc.allocated, 1, out); 224 | _mi_fprintf(out, "\n\n"); 225 | #endif 226 | mi_stat_print(&stats->reserved, "reserved", 1, out); 227 | mi_stat_print(&stats->committed, "committed", 1, out); 228 | mi_stat_print(&stats->reset, "reset", 1, out); 229 | mi_stat_print(&stats->page_committed, "touched", 1, out); 230 | mi_stat_print(&stats->segments, "segments", -1, out); 231 | mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out); 232 | mi_stat_print(&stats->pages, "pages", -1, out); 233 | mi_stat_print(&stats->pages_abandoned, "-abandoned", -1, out); 234 | mi_stat_print(&stats->pages_extended, "-extended", 0, out); 235 | mi_stat_print(&stats->mmap_calls, "mmaps", 0, out); 236 | mi_stat_print(&stats->mmap_right_align, "mmap fast", 0, out); 237 | mi_stat_print(&stats->mmap_ensure_aligned, "mmap slow", 0, out); 238 | mi_stat_print(&stats->commit_calls, "commits", 0, out); 239 | mi_stat_print(&stats->threads, "threads", 0, out); 240 | mi_stat_counter_print(&stats->searches, "searches", out); 241 | 242 | if (secs >= 0.0) _mi_fprintf(out, "%10s: %9.3f s\n", "elapsed", secs); 243 | 244 | double user_time; 245 | double sys_time; 246 | size_t peak_rss; 247 | size_t page_faults; 248 | size_t page_reclaim; 249 | size_t peak_commit; 250 | mi_process_info(&user_time, &sys_time, &peak_rss, &page_faults, &page_reclaim, &peak_commit); 251 | _mi_fprintf(out,"%10s: user: %.3f s, system: %.3f s, faults: %lu, reclaims: %lu, rss: ", "process", user_time, sys_time, (unsigned long)page_faults, (unsigned long)page_reclaim ); 252 | mi_printf_amount((int64_t)peak_rss, 1, out, "%s"); 253 | if (peak_commit > 0) { 254 | _mi_fprintf(out,", commit charge: "); 255 | mi_printf_amount((int64_t)peak_commit, 1, out, "%s"); 256 | } 257 | _mi_fprintf(out,"\n"); 258 | } 259 | 260 | static double mi_clock_end(double start); 261 | static double mi_clock_start(void); 262 | static double mi_time_start = 0.0; 263 | 264 | static mi_stats_t* mi_stats_get_default(void) { 265 | mi_heap_t* heap = mi_heap_get_default(); 266 | return &heap->tld->stats; 267 | } 268 | 269 | void mi_stats_reset(void) mi_attr_noexcept { 270 | mi_stats_t* stats = mi_stats_get_default(); 271 | if (stats != &_mi_stats_main) { memset(stats, 0, sizeof(mi_stats_t)); } 272 | memset(&_mi_stats_main, 0, sizeof(mi_stats_t)); 273 | mi_time_start = mi_clock_start(); 274 | } 275 | 276 | static void mi_stats_print_ex(mi_stats_t* stats, double secs, FILE* out) { 277 | if (stats != &_mi_stats_main) { 278 | mi_stats_add(&_mi_stats_main,stats); 279 | memset(stats,0,sizeof(mi_stats_t)); 280 | } 281 | _mi_stats_print(&_mi_stats_main, secs, out); 282 | } 283 | 284 | void mi_stats_print(FILE* out) mi_attr_noexcept { 285 | mi_stats_print_ex(mi_stats_get_default(),mi_clock_end(mi_time_start),out); 286 | } 287 | 288 | void mi_thread_stats_print(FILE* out) mi_attr_noexcept { 289 | _mi_stats_print(mi_stats_get_default(), mi_clock_end(mi_time_start), out); 290 | } 291 | 292 | 293 | 294 | // -------------------------------------------------------- 295 | // Basic timer for convenience 296 | // -------------------------------------------------------- 297 | 298 | #ifdef _WIN32 299 | #include 300 | static double mi_to_seconds(LARGE_INTEGER t) { 301 | static double freq = 0.0; 302 | if (freq <= 0.0) { 303 | LARGE_INTEGER f; 304 | QueryPerformanceFrequency(&f); 305 | freq = (double)(f.QuadPart); 306 | } 307 | return ((double)(t.QuadPart) / freq); 308 | } 309 | 310 | static double mi_clock_now(void) { 311 | LARGE_INTEGER t; 312 | QueryPerformanceCounter(&t); 313 | return mi_to_seconds(t); 314 | } 315 | #else 316 | #include 317 | #ifdef CLOCK_REALTIME 318 | static double mi_clock_now(void) { 319 | struct timespec t; 320 | clock_gettime(CLOCK_REALTIME, &t); 321 | return (double)t.tv_sec + (1.0e-9 * (double)t.tv_nsec); 322 | } 323 | #else 324 | // low resolution timer 325 | static double mi_clock_now(void) { 326 | return ((double)clock() / (double)CLOCKS_PER_SEC); 327 | } 328 | #endif 329 | #endif 330 | 331 | 332 | static double mi_clock_diff = 0.0; 333 | 334 | static double mi_clock_start(void) { 335 | if (mi_clock_diff == 0.0) { 336 | double t0 = mi_clock_now(); 337 | mi_clock_diff = mi_clock_now() - t0; 338 | } 339 | return mi_clock_now(); 340 | } 341 | 342 | static double mi_clock_end(double start) { 343 | double end = mi_clock_now(); 344 | return (end - start - mi_clock_diff); 345 | } 346 | 347 | 348 | // -------------------------------------------------------- 349 | // Basic process statistics 350 | // -------------------------------------------------------- 351 | 352 | #if defined(_WIN32) 353 | #include 354 | #include 355 | #pragma comment(lib,"psapi.lib") 356 | 357 | static double filetime_secs(const FILETIME* ftime) { 358 | ULARGE_INTEGER i; 359 | i.LowPart = ftime->dwLowDateTime; 360 | i.HighPart = ftime->dwHighDateTime; 361 | double secs = (double)(i.QuadPart) * 1.0e-7; // FILETIME is in 100 nano seconds 362 | return secs; 363 | } 364 | static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) { 365 | FILETIME ct; 366 | FILETIME ut; 367 | FILETIME st; 368 | FILETIME et; 369 | GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut); 370 | *utime = filetime_secs(&ut); 371 | *stime = filetime_secs(&st); 372 | 373 | #if 0 374 | PROCESS_MEMORY_COUNTERS info; 375 | GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); 376 | *peak_rss = (size_t)info.PeakWorkingSetSize; 377 | *page_faults = (size_t)info.PageFaultCount; 378 | *peak_commit = (size_t)info.PeakPagefileUsage; 379 | *page_reclaim = 0; 380 | #endif 381 | } 382 | 383 | #elif defined(__unix__) || defined(__unix) || defined(unix) || (defined(__APPLE__) && defined(__MACH__)) 384 | #include 385 | #include 386 | #include 387 | 388 | #if defined(__APPLE__) && defined(__MACH__) 389 | #include 390 | #endif 391 | 392 | static double timeval_secs(const struct timeval* tv) { 393 | return (double)tv->tv_sec + ((double)tv->tv_usec * 1.0e-6); 394 | } 395 | 396 | static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) { 397 | struct rusage rusage; 398 | getrusage(RUSAGE_SELF, &rusage); 399 | #if defined(__APPLE__) && defined(__MACH__) 400 | *peak_rss = rusage.ru_maxrss; 401 | #else 402 | *peak_rss = rusage.ru_maxrss * 1024; 403 | #endif 404 | *page_faults = rusage.ru_majflt; 405 | *page_reclaim = rusage.ru_minflt; 406 | *peak_commit = 0; 407 | *utime = timeval_secs(&rusage.ru_utime); 408 | *stime = timeval_secs(&rusage.ru_stime); 409 | } 410 | 411 | #else 412 | #ifndef __wasi__ 413 | // WebAssembly instances are not processes 414 | #pragma message("define a way to get process info") 415 | #endif 416 | 417 | static void mi_process_info(double* utime, double* stime, size_t* peak_rss, size_t* page_faults, size_t* page_reclaim, size_t* peak_commit) { 418 | *peak_rss = 0; 419 | *page_faults = 0; 420 | *page_reclaim = 0; 421 | *peak_commit = 0; 422 | *utime = 0.0; 423 | *stime = 0.0; 424 | } 425 | #endif 426 | -------------------------------------------------------------------------------- /tests/gcbench.nim: -------------------------------------------------------------------------------- 1 | discard """ 2 | outputsub: "Success!" 3 | """ 4 | 5 | # This is adapted from a benchmark written by John Ellis and Pete Kovac 6 | # of Post Communications. 7 | # It was modified by Hans Boehm of Silicon Graphics. 8 | # 9 | # This is no substitute for real applications. No actual application 10 | # is likely to behave in exactly this way. However, this benchmark was 11 | # designed to be more representative of real applications than other 12 | # Java GC benchmarks of which we are aware. 13 | # It attempts to model those properties of allocation requests that 14 | # are important to current GC techniques. 15 | # It is designed to be used either to obtain a single overall performance 16 | # number, or to give a more detailed estimate of how collector 17 | # performance varies with object lifetimes. It prints the time 18 | # required to allocate and collect balanced binary trees of various 19 | # sizes. Smaller trees result in shorter object lifetimes. Each cycle 20 | # allocates roughly the same amount of memory. 21 | # Two data structures are kept around during the entire process, so 22 | # that the measured performance is representative of applications 23 | # that maintain some live in-memory data. One of these is a tree 24 | # containing many pointers. The other is a large array containing 25 | # double precision floating point numbers. Both should be of comparable 26 | # size. 27 | # 28 | # The results are only really meaningful together with a specification 29 | # of how much memory was used. It is possible to trade memory for 30 | # better time performance. This benchmark should be run in a 32 MB 31 | # heap, though we don't currently know how to enforce that uniformly. 32 | # 33 | # Unlike the original Ellis and Kovac benchmark, we do not attempt 34 | # measure pause times. This facility should eventually be added back 35 | # in. There are several reasons for omitting it for now. The original 36 | # implementation depended on assumptions about the thread scheduler 37 | # that don't hold uniformly. The results really measure both the 38 | # scheduler and GC. Pause time measurements tend to not fit well with 39 | # current benchmark suites. As far as we know, none of the current 40 | # commercial Java implementations seriously attempt to minimize GC pause 41 | # times. 42 | # 43 | # Known deficiencies: 44 | # - No way to check on memory use 45 | # - No cyclic data structures 46 | # - No attempt to measure variation with object size 47 | # - Results are sensitive to locking cost, but we dont 48 | # check for proper locking 49 | # 50 | 51 | import ".." / araqsgc 52 | 53 | import allocators 54 | include system / ansi_c 55 | 56 | import 57 | strutils, times 58 | 59 | type 60 | PNode = ref TNode 61 | TNode = object 62 | left, right: PNode 63 | i, j: int 64 | s: string # this means we will have a destructor 65 | 66 | var xx = 0 67 | 68 | proc newNode(L, r: PNode): PNode = 69 | new(result) 70 | result.left = L 71 | result.right = r 72 | result.s = $xx # this definitely allocates! 73 | 74 | const 75 | kStretchTreeDepth = 18 # about 16Mb 76 | kLongLivedTreeDepth = 16 # about 4Mb 77 | kArraySize = 500000 # about 4Mb 78 | kMinTreeDepth = 4 79 | kMaxTreeDepth = 16 80 | 81 | when not declared(withScratchRegion): 82 | template withScratchRegion(body: untyped) = body 83 | 84 | # Nodes used by a tree of a given size 85 | proc treeSize(i: int): int = return ((1 shl (i + 1)) - 1) 86 | 87 | # Number of iterations to use for a given tree depth 88 | proc numIters(i: int): int = 89 | return 2 * treeSize(kStretchTreeDepth) div treeSize(i) 90 | 91 | # Build tree top down, assigning to older objects. 92 | proc populate(iDepth: int, thisNode: PNode) = 93 | if iDepth <= 0: 94 | return 95 | else: 96 | new(thisNode.left) 97 | new(thisNode.right) 98 | populate(iDepth-1, thisNode.left) 99 | populate(iDepth-1, thisNode.right) 100 | 101 | # Build tree bottom-up 102 | proc makeTree(iDepth: int): PNode = 103 | if iDepth <= 0: 104 | new(result) 105 | else: 106 | return newNode(makeTree(iDepth-1), makeTree(iDepth-1)) 107 | 108 | proc printDiagnostics() = 109 | echo("Total memory used: " & formatSize(usedMem)) 110 | #echo("Free memory: " & formatSize(getFreeMem())) 111 | 112 | when defined(m3): 113 | template myDispose(x) = deepDispose(x) 114 | else: 115 | template myDispose(x) = discard 116 | 117 | proc timeConstruction(depth: int) = 118 | var 119 | root, tempTree: PNode 120 | iNumIters: int 121 | 122 | iNumIters = numIters(depth) 123 | 124 | echo("Creating " & $iNumIters & " trees of depth " & $depth) 125 | var t = epochTime() 126 | for i in 0..iNumIters-1: 127 | new(tempTree) 128 | populate(depth, tempTree) 129 | myDispose(tempTree) 130 | tempTree = nil 131 | echo("\tTop down construction took " & $(epochTime() - t) & "msecs") 132 | t = epochTime() 133 | for i in 0..iNumIters-1: 134 | tempTree = makeTree(depth) 135 | myDispose(tempTree) 136 | tempTree = nil 137 | echo("\tBottom up construction took " & $(epochTime() - t) & "msecs") 138 | 139 | type 140 | tMyArray = seq[float] 141 | 142 | proc main() = 143 | var 144 | longLivedTree: PNode 145 | myarray: tMyArray 146 | 147 | echo("Garbage Collector Test") 148 | echo(" Stretching memory with a binary tree of depth " & $kStretchTreeDepth) 149 | printDiagnostics() 150 | var t = epochTime() 151 | 152 | # Stretch the memory space quickly 153 | withScratchRegion: 154 | var tempTree = makeTree(kStretchTreeDepth) 155 | myDispose(tempTree) 156 | tempTree = nil 157 | 158 | # Create a long lived object 159 | echo(" Creating a long-lived binary tree of depth " & 160 | $kLongLivedTreeDepth) 161 | new(longLivedTree) 162 | populate(kLongLivedTreeDepth, longLivedTree) 163 | 164 | # Create long-lived array, filling half of it 165 | echo(" Creating a long-lived array of " & $kArraySize & " doubles") 166 | withScratchRegion: 167 | newSeq(myarray, kArraySize) 168 | for i in 0..kArraySize div 2 - 1: 169 | myarray[i] = 1.0 / toFloat(i) 170 | 171 | printDiagnostics() 172 | 173 | var d = kMinTreeDepth 174 | while d <= kMaxTreeDepth: 175 | withScratchRegion: 176 | timeConstruction(d) 177 | inc(d, 2) 178 | 179 | if longLivedTree == nil or myarray[1000] != 1.0/1000.0: 180 | echo("Failed") 181 | # fake reference to LongLivedTree 182 | # and array to keep them from being optimized away 183 | 184 | myDispose(longLivedTree) 185 | 186 | var elapsed = epochTime() - t 187 | when not defined(m3): 188 | collect(0) 189 | printDiagnostics() 190 | echo("Completed in " & $elapsed & "s. Success!") 191 | 192 | when defined(GC_setMaxPause): 193 | GC_setMaxPause 2_000 194 | 195 | main() 196 | let (a, de) = allocCounters() 197 | discard cprintf("strings: %ld %ld\n", a, de) 198 | # strings: 3909446 3909446 199 | # wrong: 3909447 1977196 # little mor than the half?! 200 | # + 1 for the 'seq' that we use in the tracing mechanism 201 | --------------------------------------------------------------------------------